{ "title": "AI crawlers and the major websites that block them", "url": "https://scraperfinder.com/ai-crawlers/", "licence": "CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/): credit Scraper Finder with a link to the page", "updated": "2026-11-14", "method": "robots.txt read with RFC 9309 rules (packages/core/src/robots.ts); one plain HTTPS request a week per website", "registryChecked": "2026-11-14", "robotsCheckedFrom": "2026-11-14", "robotsCheckedTo": "2026-11-14", "websites": 172, "read": 137, "totals": { "crawlers": 34, "blockAnyTraining": 50, "blockAnyTrainingPct": 36, "gptbotNotSearch": 25, "blockEvery": 7, "blockNone": 81, "blockNonePct": 59, "nameAny": 67 }, "crawlers": [ { "token": "GPTBot", "operator": "OpenAI", "kind": "training", "purpose": "Crawls pages that may be used to train OpenAI's generative AI foundation models.", "robots": "OpenAI: disallowing GPTBot says the site's content should not be used to train its foundation models.", "source": "https://developers.openai.com/api/docs/bots", "read": 137, "named": 47, "blocked": 35, "blockedExcept": 3, "partial": 88, "allowed": 11, "blockedWhole": 38, "blockedByName": 32, "blockedByStar": 6, "blockedPct": 28 }, { "token": "OAI-SearchBot", "operator": "OpenAI", "kind": "search", "purpose": "Finds and surfaces websites in ChatGPT search results.", "robots": "OpenAI: sites that opt out of OAI-SearchBot are not shown in ChatGPT search answers.", "source": "https://developers.openai.com/api/docs/bots", "read": 137, "named": 32, "blocked": 10, "blockedExcept": 3, "partial": 110, "allowed": 14, "blockedWhole": 13, "blockedByName": 5, "blockedByStar": 8, "blockedPct": 9 }, { "token": "ChatGPT-User", "operator": "OpenAI", "kind": "user", "purpose": "Visits a page when a person's request in ChatGPT or a custom GPT needs it.", "robots": "OpenAI: because a user starts these visits, robots.txt rules may not apply.", "source": "https://developers.openai.com/api/docs/bots", "read": 137, "named": 37, "blocked": 19, "blockedExcept": 2, "partial": 103, "allowed": 13, "blockedWhole": 21, "blockedByName": 13, "blockedByStar": 8, "blockedPct": 15 }, { "token": "OAI-AdsBot", "operator": "OpenAI", "kind": "other", "purpose": "Checks the safety of web pages submitted as ads on ChatGPT.", "robots": "OpenAI's page gives no robots.txt guidance for it.", "source": "https://developers.openai.com/api/docs/bots", "read": 137, "named": 4, "blocked": 13, "blockedExcept": 2, "partial": 114, "allowed": 8, "blockedWhole": 15, "blockedByName": 2, "blockedByStar": 13, "blockedPct": 11 }, { "token": "ClaudeBot", "operator": "Anthropic", "kind": "training", "purpose": "Collects web content that may be used to train and develop Anthropic's models.", "robots": "Anthropic: its bots honour robots.txt directives.", "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "read": 137, "named": 44, "blocked": 34, "blockedExcept": 3, "partial": 92, "allowed": 8, "blockedWhole": 37, "blockedByName": 31, "blockedByStar": 6, "blockedPct": 27 }, { "token": "Claude-SearchBot", "operator": "Anthropic", "kind": "search", "purpose": "Indexes pages to improve the quality of Claude's search results.", "robots": "Anthropic: its bots honour robots.txt directives.", "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "read": 137, "named": 24, "blocked": 12, "blockedExcept": 2, "partial": 112, "allowed": 11, "blockedWhole": 14, "blockedByName": 6, "blockedByStar": 8, "blockedPct": 10 }, { "token": "Claude-User", "operator": "Anthropic", "kind": "user", "purpose": "Fetches a page when a Claude user asks something that needs it.", "robots": "Anthropic: its bots honour robots.txt directives, this one included.", "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "read": 137, "named": 24, "blocked": 15, "blockedExcept": 2, "partial": 109, "allowed": 11, "blockedWhole": 17, "blockedByName": 9, "blockedByStar": 8, "blockedPct": 12 }, { "token": "anthropic-ai", "operator": "Anthropic", "kind": "legacy", "purpose": "An older name many robots.txt files still carry. It is not on Anthropic's current crawler page.", "robots": "Not on the operator's current page.", "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "read": 137, "named": 23, "blocked": 23, "blockedExcept": 3, "partial": 103, "allowed": 8, "blockedWhole": 26, "blockedByName": 17, "blockedByStar": 9, "blockedPct": 19 }, { "token": "Claude-Web", "operator": "Anthropic", "kind": "legacy", "purpose": "An older name many robots.txt files still carry. It is not on Anthropic's current crawler page.", "robots": "Not on the operator's current page.", "source": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "read": 137, "named": 17, "blocked": 22, "blockedExcept": 2, "partial": 105, "allowed": 8, "blockedWhole": 24, "blockedByName": 14, "blockedByStar": 10, "blockedPct": 18 }, { "token": "Google-Extended", "operator": "Google", "kind": "control", "purpose": "A robots.txt token, not a crawler: it decides whether pages Google crawls may be used to train future Gemini models. It has no effect on Google Search.", "robots": "Google: crawling is done with Google's usual user agents; Google-Extended only reads as a robots.txt group.", "source": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "read": 137, "named": 35, "blocked": 26, "blockedExcept": 2, "partial": 99, "allowed": 10, "blockedWhole": 28, "blockedByName": 23, "blockedByStar": 5, "blockedPct": 20 }, { "token": "Google-CloudVertexBot", "operator": "Google", "kind": "other", "purpose": "Crawls sites whose owners asked for it, to build Vertex AI Agents. No effect on Google Search.", "robots": "Listed among Google's common crawlers, which follow robots.txt.", "source": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers", "read": 137, "named": 10, "blocked": 18, "blockedExcept": 2, "partial": 110, "allowed": 7, "blockedWhole": 20, "blockedByName": 9, "blockedByStar": 11, "blockedPct": 15 }, { "token": "Applebot-Extended", "operator": "Apple", "kind": "control", "purpose": "A robots.txt token, not a crawler: it decides whether pages Applebot collects may train Apple's foundation models. Blocking it keeps pages in Apple's search features.", "robots": "Apple: Applebot-Extended does not crawl; disallow it to opt out of model training.", "source": "https://support.apple.com/en-us/119829", "read": 137, "named": 26, "blocked": 25, "blockedExcept": 4, "partial": 99, "allowed": 9, "blockedWhole": 29, "blockedByName": 19, "blockedByStar": 10, "blockedPct": 21 }, { "token": "PerplexityBot", "operator": "Perplexity", "kind": "search", "purpose": "Surfaces and links websites in Perplexity search results. Perplexity says it is not used to crawl for AI model training.", "robots": "Perplexity: allow it in robots.txt for your site to appear in its search results.", "source": "https://docs.perplexity.ai/guides/bots", "read": 137, "named": 35, "blocked": 18, "blockedExcept": 3, "partial": 105, "allowed": 11, "blockedWhole": 21, "blockedByName": 15, "blockedByStar": 6, "blockedPct": 15 }, { "token": "Perplexity-User", "operator": "Perplexity", "kind": "user", "purpose": "Visits a page when a Perplexity user's question needs it.", "robots": "Perplexity: because a user asked for the page, this fetcher generally ignores robots.txt rules.", "source": "https://docs.perplexity.ai/guides/bots", "read": 137, "named": 16, "blocked": 15, "blockedExcept": 3, "partial": 109, "allowed": 10, "blockedWhole": 18, "blockedByName": 8, "blockedByStar": 10, "blockedPct": 13 }, { "token": "meta-externalagent", "operator": "Meta", "kind": "training", "purpose": "Crawls the web for training AI models and for improving products by indexing content.", "robots": "Meta documents blocking it with robots.txt disallow rules.", "source": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "read": 137, "named": 29, "blocked": 22, "blockedExcept": 4, "partial": 101, "allowed": 10, "blockedWhole": 26, "blockedByName": 19, "blockedByStar": 7, "blockedPct": 19 }, { "token": "meta-externalfetcher", "operator": "Meta", "kind": "user", "purpose": "Fetches individual links a user asks for, for Meta's agentic AI features.", "robots": "Meta: because the fetches are user requested, it may bypass robots.txt rules.", "source": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "read": 137, "named": 12, "blocked": 15, "blockedExcept": 1, "partial": 111, "allowed": 10, "blockedWhole": 16, "blockedByName": 8, "blockedByStar": 8, "blockedPct": 12 }, { "token": "meta-webindexer", "operator": "Meta", "kind": "search", "purpose": "Crawls the web to improve the quality of Meta AI search results.", "robots": "Meta: allowing it in robots.txt helps Meta cite and link your content.", "source": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "read": 137, "named": 8, "blocked": 14, "blockedExcept": 1, "partial": 113, "allowed": 9, "blockedWhole": 15, "blockedByName": 3, "blockedByStar": 12, "blockedPct": 11 }, { "token": "FacebookBot", "operator": "Meta", "kind": "legacy", "purpose": "An older Meta name many robots.txt files still carry. It is not on Meta's current crawler page.", "robots": "Not on the operator's current page.", "source": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers", "read": 137, "named": 16, "blocked": 22, "blockedExcept": 2, "partial": 106, "allowed": 7, "blockedWhole": 24, "blockedByName": 13, "blockedByStar": 11, "blockedPct": 18 }, { "token": "CCBot", "operator": "Common Crawl", "kind": "dataset", "purpose": "Builds Common Crawl, an open repository of web crawl data that anyone can download and analyse.", "robots": "Common Crawl: CCBot respects robots.txt.", "source": "https://commoncrawl.org/ccbot", "read": 137, "named": 36, "blocked": 32, "blockedExcept": 2, "partial": 94, "allowed": 9, "blockedWhole": 34, "blockedByName": 26, "blockedByStar": 8, "blockedPct": 25 }, { "token": "Amazonbot", "operator": "Amazon", "kind": "training", "purpose": "Improves Amazon's products and services. Amazon says what it collects may be used to train Amazon AI models.", "robots": "Amazon: honours the user-agent and allow/disallow rules in robots.txt.", "source": "https://developer.amazon.com/amazonbot", "read": 137, "named": 25, "blocked": 23, "blockedExcept": 5, "partial": 99, "allowed": 10, "blockedWhole": 28, "blockedByName": 17, "blockedByStar": 11, "blockedPct": 20 }, { "token": "Amzn-SearchBot", "operator": "Amazon", "kind": "search", "purpose": "Improves search experiences in Amazon products. Amazon says it does not crawl for generative AI training.", "robots": "Amazon: honours the user-agent and allow/disallow rules in robots.txt.", "source": "https://developer.amazon.com/amazonbot", "read": 137, "named": 3, "blocked": 13, "blockedExcept": 1, "partial": 114, "allowed": 9, "blockedWhole": 14, "blockedByName": 1, "blockedByStar": 13, "blockedPct": 10 }, { "token": "Amzn-User", "operator": "Amazon", "kind": "user", "purpose": "Supports user actions, such as Alexa answers that need current information. Amazon says it does not crawl for generative AI training.", "robots": "Amazon: honours the user-agent and allow/disallow rules in robots.txt.", "source": "https://developer.amazon.com/amazonbot", "read": 137, "named": 2, "blocked": 12, "blockedExcept": 1, "partial": 115, "allowed": 9, "blockedWhole": 13, "blockedByName": 0, "blockedByStar": 13, "blockedPct": 9 }, { "token": "MistralAI-User", "operator": "Mistral AI", "kind": "user", "purpose": "Visits a page when a user of Mistral's assistant asks for it. Not an automatic crawler.", "robots": "Mistral: robots.txt decides which sites these user requests may reach.", "source": "https://docs.mistral.ai/robots", "read": 137, "named": 4, "blocked": 15, "blockedExcept": 1, "partial": 113, "allowed": 8, "blockedWhole": 16, "blockedByName": 2, "blockedByStar": 14, "blockedPct": 12 }, { "token": "MistralAI-Index", "operator": "Mistral AI", "kind": "search", "purpose": "Indexes pages for Mistral's search. Mistral says the content is not used for AI training of any kind.", "robots": "Mistral's page says nothing more about robots.txt for this agent.", "source": "https://docs.mistral.ai/robots", "read": 137, "named": 1, "blocked": 13, "blockedExcept": 1, "partial": 115, "allowed": 8, "blockedWhole": 14, "blockedByName": 0, "blockedByStar": 14, "blockedPct": 10 }, { "token": "MistralAI-Training", "operator": "Mistral AI", "kind": "training", "purpose": "Collects web content for datasets that train Mistral's generative AI models.", "robots": "Mistral: webmasters can disallow it in robots.txt.", "source": "https://docs.mistral.ai/robots", "read": 137, "named": 3, "blocked": 16, "blockedExcept": 1, "partial": 113, "allowed": 7, "blockedWhole": 17, "blockedByName": 3, "blockedByStar": 14, "blockedPct": 12 }, { "token": "DuckAssistBot", "operator": "DuckDuckGo", "kind": "search", "purpose": "Fetches pages in real time for DuckDuckGo's AI-assisted answers, which cite their sources. DuckDuckGo says the data never trains AI models.", "robots": "DuckDuckGo: respects robots.txt; a change takes effect within 72 hours.", "source": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", "read": 137, "named": 11, "blocked": 13, "blockedExcept": 2, "partial": 114, "allowed": 8, "blockedWhole": 15, "blockedByName": 6, "blockedByStar": 9, "blockedPct": 11 }, { "token": "AI2Bot", "operator": "Allen Institute for AI", "kind": "training", "purpose": "Discovers web content used to train the Allen Institute's open language models.", "robots": "The operator's page does not say how it treats robots.txt.", "source": "https://allenai.org/crawler", "read": 137, "named": 10, "blocked": 21, "blockedExcept": 2, "partial": 107, "allowed": 7, "blockedWhole": 23, "blockedByName": 9, "blockedByStar": 14, "blockedPct": 17 }, { "token": "YouBot", "operator": "You.com", "kind": "search", "purpose": "Discovers and indexes pages for You.com search results.", "robots": "You.com: fully respects robots.txt and caches it for 30 minutes.", "source": "https://you.com/docs/youbot", "read": 137, "named": 10, "blocked": 19, "blockedExcept": 2, "partial": 108, "allowed": 8, "blockedWhole": 21, "blockedByName": 8, "blockedByStar": 13, "blockedPct": 15 }, { "token": "omgili", "operator": "Webz.io", "kind": "dataset", "purpose": "The crawler behind Webz.io, a web data service used by media monitors and research institutes.", "robots": "Webz.io: says it always complies with robots.txt requests.", "source": "https://webz.io/blog/web-data/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/", "read": 137, "named": 20, "blocked": 28, "blockedExcept": 1, "partial": 101, "allowed": 7, "blockedWhole": 29, "blockedByName": 19, "blockedByStar": 10, "blockedPct": 21 }, { "token": "omgilibot", "operator": "Webz.io", "kind": "legacy", "purpose": "An older name of Webz.io's crawler. Webz.io's own post gives the agent as omgili.", "robots": "Not on the operator's current page.", "source": "https://webz.io/blog/web-data/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/", "read": 137, "named": 15, "blocked": 23, "blockedExcept": 1, "partial": 105, "allowed": 8, "blockedWhole": 24, "blockedByName": 14, "blockedByStar": 10, "blockedPct": 18 }, { "token": "Bytespider", "operator": "ByteDance", "kind": "undocumented", "purpose": "A crawler third parties attribute to ByteDance. ByteDance publishes no page about it.", "robots": "No operator statement found.", "source": null, "read": 137, "named": 33, "blocked": 36, "blockedExcept": 4, "partial": 90, "allowed": 7, "blockedWhole": 40, "blockedByName": 30, "blockedByStar": 10, "blockedPct": 29 }, { "token": "cohere-ai", "operator": "Cohere", "kind": "undocumented", "purpose": "A name many block lists carry. Cohere's crawler page does not list it and says Cohere does not crawl to train generative AI models at this time.", "robots": "Cohere's page names no active agent and says its crawlers must respect robots.txt.", "source": "https://docs.cohere.com/docs/cohere-web-crawlers", "read": 137, "named": 16, "blocked": 22, "blockedExcept": 2, "partial": 105, "allowed": 8, "blockedWhole": 24, "blockedByName": 14, "blockedByStar": 10, "blockedPct": 18 }, { "token": "Diffbot", "operator": "Diffbot", "kind": "undocumented", "purpose": "The crawler of Diffbot, which sells web extraction and a knowledge graph. Diffbot's documentation has no crawler page.", "robots": "No operator statement found.", "source": "https://www.diffbot.com/docs/", "read": 137, "named": 18, "blocked": 25, "blockedExcept": 1, "partial": 105, "allowed": 6, "blockedWhole": 26, "blockedByName": 16, "blockedByStar": 10, "blockedPct": 19 }, { "token": "img2dataset", "operator": "Open-source tool", "kind": "other", "purpose": "The name of img2dataset, an open-source program that downloads large image collections for AI datasets. Anyone can run it; it is not one company's crawler.", "robots": "No operator page.", "source": null, "read": 137, "named": 8, "blocked": 20, "blockedExcept": 1, "partial": 109, "allowed": 7, "blockedWhole": 21, "blockedByName": 7, "blockedByStar": 14, "blockedPct": 15 } ], "rows": [ { "website": "2GIS", "domain": "2gis.com", "robotsUrl": "https://2gis.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Ahrefs", "domain": "ahrefs.com", "robotsUrl": "https://ahrefs.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Airbnb", "domain": "airbnb.com", "robotsUrl": "https://airbnb.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "partial", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "own" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "blocked", "own" ], "PerplexityBot": [ "partial", "own" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "own" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "blocked", "own" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "blocked", "own" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "own" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Alibaba.com", "domain": "alibaba.com", "robotsUrl": "https://alibaba.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "AliExpress", "domain": "aliexpress.com", "robotsUrl": "https://aliexpress.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "blocked", "own" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Allegro", "domain": "allegro.pl", "robotsUrl": "https://allegro.pl/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "own" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Amazon", "domain": "amazon.com", "robotsUrl": "https://amazon.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "blocked", "own" ], "ChatGPT-User": [ "blocked", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "blocked", "own" ], "Claude-User": [ "blocked", "own" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "blocked", "own" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "blocked", "own" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "blocked", "own" ], "Perplexity-User": [ "blocked", "own" ], "meta-externalagent": [ "blocked", "own" ], "meta-externalfetcher": [ "blocked", "own" ], "meta-webindexer": [ "blocked", "own" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "blocked", "own" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "blocked", "own" ], "AI2Bot": [ "blocked", "own" ], "YouBot": [ "blocked", "own" ], "omgili": [ "blocked", "own" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "blocked", "own" ], "Diffbot": [ "blocked", "own" ], "img2dataset": [ "blocked", "own" ] } }, { "website": "Apple App Store", "domain": "apps.apple.com", "robotsUrl": "https://apps.apple.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Ashby job boards", "domain": "jobs.ashbyhq.com", "robotsUrl": "https://jobs.ashbyhq.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "AutoScout24", "domain": "autoscout24.com", "robotsUrl": "https://autoscout24.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "blocked", "own" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Baidu", "domain": "baidu.com", "robotsUrl": "https://baidu.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "star" ], "OAI-SearchBot": [ "blocked", "star" ], "ChatGPT-User": [ "blocked", "star" ], "OAI-AdsBot": [ "blocked", "star" ], "ClaudeBot": [ "blocked", "star" ], "Claude-SearchBot": [ "blocked", "star" ], "Claude-User": [ "blocked", "star" ], "anthropic-ai": [ "blocked", "star" ], "Claude-Web": [ "blocked", "star" ], "Google-Extended": [ "blocked", "star" ], "Google-CloudVertexBot": [ "blocked", "star" ], "Applebot-Extended": [ "blocked", "star" ], "PerplexityBot": [ "blocked", "star" ], "Perplexity-User": [ "blocked", "star" ], "meta-externalagent": [ "blocked", "star" ], "meta-externalfetcher": [ "blocked", "star" ], "meta-webindexer": [ "blocked", "star" ], "FacebookBot": [ "blocked", "star" ], "CCBot": [ "blocked", "star" ], "Amazonbot": [ "blocked", "star" ], "Amzn-SearchBot": [ "blocked", "star" ], "Amzn-User": [ "blocked", "star" ], "MistralAI-User": [ "blocked", "star" ], "MistralAI-Index": [ "blocked", "star" ], "MistralAI-Training": [ "blocked", "star" ], "DuckAssistBot": [ "blocked", "star" ], "AI2Bot": [ "blocked", "star" ], "YouBot": [ "blocked", "star" ], "omgili": [ "blocked", "star" ], "omgilibot": [ "blocked", "star" ], "Bytespider": [ "blocked", "star" ], "cohere-ai": [ "blocked", "star" ], "Diffbot": [ "blocked", "star" ], "img2dataset": [ "blocked", "star" ] } }, { "website": "Bayt", "domain": "bayt.com", "robotsUrl": "https://bayt.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Bayut", "domain": "bayut.com", "robotsUrl": "https://bayut.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Bilibili", "domain": "bilibili.com", "robotsUrl": "https://bilibili.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Bing", "domain": "bing.com", "robotsUrl": "https://bing.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Bing Maps", "domain": "bing.com/maps", "robotsUrl": "https://bing.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Bluesky", "domain": "bsky.app", "robotsUrl": "https://bsky.app/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Bol.com", "domain": "bol.com", "robotsUrl": "https://bol.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "blocked", "own" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Booking.com", "domain": "booking.com", "robotsUrl": "https://booking.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Brazil CNPJ registry", "domain": "gov.br", "robotsUrl": "https://gov.br/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Built In", "domain": "builtin.com", "robotsUrl": "https://builtin.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "allowed", "own" ], "OAI-SearchBot": [ "allowed", "own" ], "ChatGPT-User": [ "allowed", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "allowed", "own" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Capterra", "domain": "capterra.com", "robotsUrl": "https://capterra.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Coursera", "domain": "coursera.org", "robotsUrl": "https://coursera.org/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "own" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "partial", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "own" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "own" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "blocked", "own" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "own" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "own" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Craigslist", "domain": "craigslist.org", "robotsUrl": "https://craigslist.org/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Crexi", "domain": "crexi.com", "robotsUrl": "https://crexi.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Crunchbase", "domain": "crunchbase.com", "robotsUrl": "https://crunchbase.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "allowed", "own" ], "ChatGPT-User": [ "allowed", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "allowed", "own" ], "Claude-User": [ "allowed", "own" ], "anthropic-ai": [ "blocked", "own" ], "Claude-Web": [ "blocked", "own" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "blocked", "own" ], "PerplexityBot": [ "allowed", "own" ], "Perplexity-User": [ "allowed", "own" ], "meta-externalagent": [ "blocked", "own" ], "meta-externalfetcher": [ "allowed", "own" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "blocked", "own" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "blocked", "own" ], "YouBot": [ "partial", "star" ], "omgili": [ "blocked", "own" ], "omgilibot": [ "blocked", "own" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "blocked", "own" ], "Diffbot": [ "blocked", "own" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Dailymotion", "domain": "dailymotion.com", "robotsUrl": "https://dailymotion.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "DeviantArt", "domain": "deviantart.com", "robotsUrl": "https://deviantart.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "eBay", "domain": "ebay.com", "robotsUrl": "https://ebay.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked-except", "own" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "partial", "own" ], "OAI-AdsBot": [ "blocked-except", "own" ], "ClaudeBot": [ "blocked-except", "own" ], "Claude-SearchBot": [ "partial", "own" ], "Claude-User": [ "partial", "own" ], "anthropic-ai": [ "blocked-except", "own" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "blocked-except", "own" ], "PerplexityBot": [ "blocked-except", "own" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "blocked-except", "own" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "blocked-except", "own" ], "Amazonbot": [ "blocked-except", "own" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "blocked-except", "own" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "ESPN", "domain": "espn.com", "robotsUrl": "https://espn.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "blocked", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "blocked", "own" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "blocked", "own" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "blocked", "own" ], "omgilibot": [ "blocked", "own" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Etsy", "domain": "etsy.com", "robotsUrl": "https://etsy.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Eventbrite", "domain": "eventbrite.com", "robotsUrl": "https://eventbrite.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "allowed", "own" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Expedia", "domain": "expedia.com", "robotsUrl": "https://expedia.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "partial", "own" ], "OAI-AdsBot": [ "partial", "own" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "own" ], "Claude-User": [ "partial", "own" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "own" ], "Perplexity-User": [ "partial", "own" ], "meta-externalagent": [ "partial", "own" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "own" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Facebook", "domain": "facebook.com", "robotsUrl": "https://facebook.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "blocked", "star" ], "ChatGPT-User": [ "blocked", "star" ], "OAI-AdsBot": [ "blocked", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "blocked", "star" ], "Claude-User": [ "blocked", "star" ], "anthropic-ai": [ "blocked", "star" ], "Claude-Web": [ "blocked", "star" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "blocked", "star" ], "Applebot-Extended": [ "blocked", "own" ], "PerplexityBot": [ "blocked", "own" ], "Perplexity-User": [ "blocked", "star" ], "meta-externalagent": [ "blocked", "star" ], "meta-externalfetcher": [ "blocked", "star" ], "meta-webindexer": [ "blocked", "star" ], "FacebookBot": [ "blocked", "star" ], "CCBot": [ "blocked", "star" ], "Amazonbot": [ "blocked", "own" ], "Amzn-SearchBot": [ "blocked", "star" ], "Amzn-User": [ "blocked", "star" ], "MistralAI-User": [ "blocked", "star" ], "MistralAI-Index": [ "blocked", "star" ], "MistralAI-Training": [ "blocked", "star" ], "DuckAssistBot": [ "blocked", "star" ], "AI2Bot": [ "blocked", "star" ], "YouBot": [ "blocked", "star" ], "omgili": [ "blocked", "star" ], "omgilibot": [ "blocked", "star" ], "Bytespider": [ "blocked", "star" ], "cohere-ai": [ "blocked", "star" ], "Diffbot": [ "blocked", "star" ], "img2dataset": [ "blocked", "star" ] } }, { "website": "FINN.no", "domain": "finn.no", "robotsUrl": "https://finn.no/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "own" ], "OAI-SearchBot": [ "allowed", "own" ], "ChatGPT-User": [ "allowed", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "own" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "own" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "own" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "own" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "own" ], "Amazonbot": [ "partial", "own" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "own" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Fiverr", "domain": "fiverr.com", "robotsUrl": "https://fiverr.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "partial", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "own" ], "Claude-User": [ "partial", "own" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "own" ], "PerplexityBot": [ "partial", "own" ], "Perplexity-User": [ "partial", "own" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "own" ], "MistralAI-Index": [ "partial", "own" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "own" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "own" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Flipkart", "domain": "flipkart.com", "robotsUrl": "https://www.flipkart.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "blocked", "own" ], "Amzn-SearchBot": [ "blocked", "own" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Fotocasa", "domain": "fotocasa.es", "robotsUrl": "https://fotocasa.es/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Freelancer.com", "domain": "freelancer.com", "robotsUrl": "https://freelancer.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "G2", "domain": "g2.com", "robotsUrl": "https://g2.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "own" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "own" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "own" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "own" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "own" ], "Amazonbot": [ "partial", "own" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "own" ], "img2dataset": [ "partial", "star" ] } }, { "website": "GitHub", "domain": "github.com", "robotsUrl": "https://github.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "own" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "partial", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "own" ], "Claude-SearchBot": [ "partial", "own" ], "Claude-User": [ "partial", "own" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "own" ], "Perplexity-User": [ "partial", "own" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Glassdoor", "domain": "glassdoor.com", "robotsUrl": "https://glassdoor.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked-except", "own" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked-except", "own" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "blocked-except", "own" ], "Claude-Web": [ "blocked-except", "own" ], "Google-Extended": [ "blocked-except", "own" ], "Google-CloudVertexBot": [ "blocked-except", "own" ], "Applebot-Extended": [ "blocked-except", "own" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "blocked", "own" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "blocked-except", "own" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "blocked", "own" ], "omgili": [ "blocked", "own" ], "omgilibot": [ "blocked", "own" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "blocked-except", "own" ], "Diffbot": [ "blocked", "own" ], "img2dataset": [ "blocked", "own" ] } }, { "website": "Goodreads", "domain": "goodreads.com", "robotsUrl": "https://goodreads.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Google Flights", "domain": "google.com/travel/flights", "robotsUrl": "https://google.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Google Maps", "domain": "google.com/maps", "robotsUrl": "https://google.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Google News", "domain": "news.google.com", "robotsUrl": "https://news.google.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "allowed", "star" ], "ChatGPT-User": [ "blocked", "own" ], "OAI-AdsBot": [ "allowed", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "allowed", "star" ], "Claude-User": [ "allowed", "star" ], "anthropic-ai": [ "blocked", "own" ], "Claude-Web": [ "blocked", "own" ], "Google-Extended": [ "allowed", "star" ], "Google-CloudVertexBot": [ "allowed", "star" ], "Applebot-Extended": [ "allowed", "star" ], "PerplexityBot": [ "blocked", "own" ], "Perplexity-User": [ "allowed", "star" ], "meta-externalagent": [ "allowed", "star" ], "meta-externalfetcher": [ "allowed", "star" ], "meta-webindexer": [ "allowed", "star" ], "FacebookBot": [ "allowed", "star" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "allowed", "star" ], "Amzn-SearchBot": [ "allowed", "star" ], "Amzn-User": [ "allowed", "star" ], "MistralAI-User": [ "allowed", "star" ], "MistralAI-Index": [ "allowed", "star" ], "MistralAI-Training": [ "allowed", "star" ], "DuckAssistBot": [ "allowed", "star" ], "AI2Bot": [ "allowed", "star" ], "YouBot": [ "allowed", "star" ], "omgili": [ "allowed", "star" ], "omgilibot": [ "allowed", "star" ], "Bytespider": [ "allowed", "star" ], "cohere-ai": [ "allowed", "star" ], "Diffbot": [ "allowed", "star" ], "img2dataset": [ "allowed", "star" ] } }, { "website": "Google Patents", "domain": "patents.google.com", "robotsUrl": "https://patents.google.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "allowed", "star" ], "OAI-SearchBot": [ "allowed", "star" ], "ChatGPT-User": [ "allowed", "star" ], "OAI-AdsBot": [ "allowed", "star" ], "ClaudeBot": [ "allowed", "star" ], "Claude-SearchBot": [ "allowed", "star" ], "Claude-User": [ "allowed", "star" ], "anthropic-ai": [ "allowed", "star" ], "Claude-Web": [ "allowed", "star" ], "Google-Extended": [ "allowed", "star" ], "Google-CloudVertexBot": [ "allowed", "star" ], "Applebot-Extended": [ "allowed", "star" ], "PerplexityBot": [ "allowed", "star" ], "Perplexity-User": [ "allowed", "star" ], "meta-externalagent": [ "allowed", "star" ], "meta-externalfetcher": [ "allowed", "star" ], "meta-webindexer": [ "allowed", "star" ], "FacebookBot": [ "allowed", "star" ], "CCBot": [ "allowed", "star" ], "Amazonbot": [ "allowed", "star" ], "Amzn-SearchBot": [ "allowed", "star" ], "Amzn-User": [ "allowed", "star" ], "MistralAI-User": [ "allowed", "star" ], "MistralAI-Index": [ "allowed", "star" ], "MistralAI-Training": [ "allowed", "star" ], "DuckAssistBot": [ "allowed", "star" ], "AI2Bot": [ "allowed", "star" ], "YouBot": [ "allowed", "star" ], "omgili": [ "allowed", "star" ], "omgilibot": [ "allowed", "star" ], "Bytespider": [ "allowed", "star" ], "cohere-ai": [ "allowed", "star" ], "Diffbot": [ "allowed", "star" ], "img2dataset": [ "allowed", "star" ] } }, { "website": "Google Play", "domain": "play.google.com", "robotsUrl": "https://play.google.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Google Scholar", "domain": "scholar.google.com", "robotsUrl": "https://scholar.google.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Google Search", "domain": "google.com", "robotsUrl": "https://google.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Google Trends", "domain": "trends.google.com", "robotsUrl": "https://trends.google.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Google Workspace Marketplace", "domain": "workspace.google.com/marketplace", "robotsUrl": "https://workspace.google.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Greenhouse job boards", "domain": "boards.greenhouse.io", "robotsUrl": "https://boards.greenhouse.io/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "GulfTalent", "domain": "gulftalent.com", "robotsUrl": "https://gulftalent.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "partial", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "blocked", "own" ], "Claude-Web": [ "blocked", "own" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "blocked", "own" ], "Applebot-Extended": [ "blocked", "own" ], "PerplexityBot": [ "partial", "own" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "blocked", "own" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "blocked", "own" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "blocked", "own" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "blocked", "own" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "blocked", "own" ], "YouBot": [ "partial", "star" ], "omgili": [ "blocked", "own" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "blocked", "own" ], "Diffbot": [ "blocked", "own" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Hacker News", "domain": "news.ycombinator.com", "robotsUrl": "https://news.ycombinator.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Handshake", "domain": "joinhandshake.com", "robotsUrl": "https://joinhandshake.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "allowed", "star" ], "OAI-SearchBot": [ "allowed", "star" ], "ChatGPT-User": [ "allowed", "star" ], "OAI-AdsBot": [ "allowed", "star" ], "ClaudeBot": [ "allowed", "star" ], "Claude-SearchBot": [ "allowed", "star" ], "Claude-User": [ "allowed", "star" ], "anthropic-ai": [ "allowed", "star" ], "Claude-Web": [ "allowed", "star" ], "Google-Extended": [ "allowed", "star" ], "Google-CloudVertexBot": [ "allowed", "star" ], "Applebot-Extended": [ "allowed", "star" ], "PerplexityBot": [ "allowed", "star" ], "Perplexity-User": [ "allowed", "star" ], "meta-externalagent": [ "allowed", "star" ], "meta-externalfetcher": [ "allowed", "star" ], "meta-webindexer": [ "allowed", "star" ], "FacebookBot": [ "blocked", "own" ], "CCBot": [ "allowed", "star" ], "Amazonbot": [ "allowed", "star" ], "Amzn-SearchBot": [ "allowed", "star" ], "Amzn-User": [ "allowed", "star" ], "MistralAI-User": [ "allowed", "star" ], "MistralAI-Index": [ "allowed", "star" ], "MistralAI-Training": [ "allowed", "star" ], "DuckAssistBot": [ "allowed", "star" ], "AI2Bot": [ "blocked", "own" ], "YouBot": [ "allowed", "star" ], "omgili": [ "blocked", "own" ], "omgilibot": [ "blocked", "own" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "allowed", "star" ], "Diffbot": [ "blocked", "own" ], "img2dataset": [ "blocked", "own" ] } }, { "website": "Himalayas", "domain": "himalayas.app", "robotsUrl": "https://himalayas.app/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Hostelworld", "domain": "hostelworld.com", "robotsUrl": "https://hostelworld.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Houzz", "domain": "houzz.com", "robotsUrl": "https://houzz.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "own" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "partial", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "own" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "own" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "own" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Idealista", "domain": "idealista.com", "robotsUrl": "https://idealista.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "IKEA", "domain": "ikea.com", "robotsUrl": "https://ikea.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "blocked", "own" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "IMDb", "domain": "imdb.com", "robotsUrl": "https://imdb.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "star" ], "OAI-SearchBot": [ "blocked", "star" ], "ChatGPT-User": [ "blocked", "star" ], "OAI-AdsBot": [ "blocked", "star" ], "ClaudeBot": [ "blocked", "star" ], "Claude-SearchBot": [ "blocked", "star" ], "Claude-User": [ "blocked", "star" ], "anthropic-ai": [ "blocked", "star" ], "Claude-Web": [ "blocked", "star" ], "Google-Extended": [ "blocked", "star" ], "Google-CloudVertexBot": [ "blocked", "star" ], "Applebot-Extended": [ "blocked", "star" ], "PerplexityBot": [ "blocked", "star" ], "Perplexity-User": [ "blocked", "star" ], "meta-externalagent": [ "blocked", "star" ], "meta-externalfetcher": [ "blocked", "star" ], "meta-webindexer": [ "blocked", "star" ], "FacebookBot": [ "blocked", "star" ], "CCBot": [ "blocked", "star" ], "Amazonbot": [ "partial", "own" ], "Amzn-SearchBot": [ "partial", "own" ], "Amzn-User": [ "partial", "own" ], "MistralAI-User": [ "blocked", "star" ], "MistralAI-Index": [ "blocked", "star" ], "MistralAI-Training": [ "blocked", "star" ], "DuckAssistBot": [ "partial", "own" ], "AI2Bot": [ "blocked", "star" ], "YouBot": [ "blocked", "star" ], "omgili": [ "blocked", "star" ], "omgilibot": [ "blocked", "star" ], "Bytespider": [ "blocked", "star" ], "cohere-ai": [ "blocked", "star" ], "Diffbot": [ "blocked", "star" ], "img2dataset": [ "blocked", "star" ] } }, { "website": "Immowelt", "domain": "immowelt.de", "robotsUrl": "https://immowelt.de/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "own" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "partial", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "own" ], "Claude-SearchBot": [ "partial", "own" ], "Claude-User": [ "partial", "own" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "own" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "ImportYeti", "domain": "importyeti.com", "robotsUrl": "https://importyeti.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Indeed", "domain": "indeed.com", "robotsUrl": "https://indeed.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "own" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "partial", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "own" ], "Claude-SearchBot": [ "partial", "own" ], "Claude-User": [ "partial", "own" ], "anthropic-ai": [ "partial", "own" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "own" ], "Google-CloudVertexBot": [ "partial", "own" ], "Applebot-Extended": [ "partial", "own" ], "PerplexityBot": [ "partial", "own" ], "Perplexity-User": [ "partial", "own" ], "meta-externalagent": [ "partial", "own" ], "meta-externalfetcher": [ "partial", "own" ], "meta-webindexer": [ "partial", "own" ], "FacebookBot": [ "partial", "own" ], "CCBot": [ "partial", "own" ], "Amazonbot": [ "partial", "own" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "own" ], "AI2Bot": [ "partial", "own" ], "YouBot": [ "partial", "own" ], "omgili": [ "blocked", "own" ], "omgilibot": [ "blocked", "own" ], "Bytespider": [ "partial", "own" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "own" ], "img2dataset": [ "partial", "own" ] } }, { "website": "IndiaMART", "domain": "indiamart.com", "robotsUrl": "https://indiamart.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "allowed", "own" ], "OAI-SearchBot": [ "allowed", "own" ], "ChatGPT-User": [ "allowed", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "allowed", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "allowed", "own" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "blocked", "own" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "blocked", "own" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Kayak", "domain": "kayak.com", "robotsUrl": "https://kayak.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Lever job boards", "domain": "jobs.lever.co", "robotsUrl": "https://jobs.lever.co/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "allowed", "star" ], "ChatGPT-User": [ "allowed", "star" ], "OAI-AdsBot": [ "allowed", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "allowed", "star" ], "Claude-User": [ "allowed", "star" ], "anthropic-ai": [ "blocked", "own" ], "Claude-Web": [ "blocked", "own" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "blocked", "own" ], "Applebot-Extended": [ "blocked", "own" ], "PerplexityBot": [ "allowed", "star" ], "Perplexity-User": [ "allowed", "star" ], "meta-externalagent": [ "blocked", "own" ], "meta-externalfetcher": [ "allowed", "star" ], "meta-webindexer": [ "allowed", "star" ], "FacebookBot": [ "allowed", "star" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "blocked", "own" ], "Amzn-SearchBot": [ "allowed", "star" ], "Amzn-User": [ "allowed", "star" ], "MistralAI-User": [ "allowed", "star" ], "MistralAI-Index": [ "allowed", "star" ], "MistralAI-Training": [ "blocked", "own" ], "DuckAssistBot": [ "allowed", "star" ], "AI2Bot": [ "allowed", "star" ], "YouBot": [ "allowed", "star" ], "omgili": [ "blocked", "own" ], "omgilibot": [ "allowed", "star" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "blocked", "own" ], "Diffbot": [ "blocked", "own" ], "img2dataset": [ "allowed", "star" ] } }, { "website": "LinkedIn", "domain": "linkedin.com", "robotsUrl": "https://www.linkedin.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "blocked", "own" ], "OAI-AdsBot": [ "blocked", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "partial", "own" ], "Claude-User": [ "blocked", "own" ], "anthropic-ai": [ "blocked", "own" ], "Claude-Web": [ "blocked", "own" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "blocked", "own" ], "Applebot-Extended": [ "blocked", "star" ], "PerplexityBot": [ "blocked", "own" ], "Perplexity-User": [ "blocked", "own" ], "meta-externalagent": [ "blocked", "own" ], "meta-externalfetcher": [ "blocked", "own" ], "meta-webindexer": [ "blocked", "star" ], "FacebookBot": [ "blocked", "star" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "blocked", "star" ], "Amzn-SearchBot": [ "blocked", "star" ], "Amzn-User": [ "blocked", "star" ], "MistralAI-User": [ "blocked", "star" ], "MistralAI-Index": [ "blocked", "star" ], "MistralAI-Training": [ "blocked", "star" ], "DuckAssistBot": [ "blocked", "own" ], "AI2Bot": [ "blocked", "star" ], "YouBot": [ "blocked", "star" ], "omgili": [ "blocked", "own" ], "omgilibot": [ "blocked", "own" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "blocked", "own" ], "Diffbot": [ "blocked", "own" ], "img2dataset": [ "blocked", "star" ] } }, { "website": "LinkedIn Ad Library", "domain": "linkedin.com/ad-library", "robotsUrl": "https://www.linkedin.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "blocked", "own" ], "OAI-AdsBot": [ "blocked", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "partial", "own" ], "Claude-User": [ "blocked", "own" ], "anthropic-ai": [ "blocked", "own" ], "Claude-Web": [ "blocked", "own" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "blocked", "own" ], "Applebot-Extended": [ "blocked", "star" ], "PerplexityBot": [ "blocked", "own" ], "Perplexity-User": [ "blocked", "own" ], "meta-externalagent": [ "blocked", "own" ], "meta-externalfetcher": [ "blocked", "own" ], "meta-webindexer": [ "blocked", "star" ], "FacebookBot": [ "blocked", "star" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "blocked", "star" ], "Amzn-SearchBot": [ "blocked", "star" ], "Amzn-User": [ "blocked", "star" ], "MistralAI-User": [ "blocked", "star" ], "MistralAI-Index": [ "blocked", "star" ], "MistralAI-Training": [ "blocked", "star" ], "DuckAssistBot": [ "blocked", "own" ], "AI2Bot": [ "blocked", "star" ], "YouBot": [ "blocked", "star" ], "omgili": [ "blocked", "own" ], "omgilibot": [ "blocked", "own" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "blocked", "own" ], "Diffbot": [ "blocked", "own" ], "img2dataset": [ "blocked", "star" ] } }, { "website": "LinkedIn Jobs", "domain": "linkedin.com/jobs", "robotsUrl": "https://www.linkedin.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "blocked", "own" ], "OAI-AdsBot": [ "blocked", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "partial", "own" ], "Claude-User": [ "blocked", "own" ], "anthropic-ai": [ "blocked", "own" ], "Claude-Web": [ "blocked", "own" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "blocked", "own" ], "Applebot-Extended": [ "blocked", "star" ], "PerplexityBot": [ "blocked", "own" ], "Perplexity-User": [ "blocked", "own" ], "meta-externalagent": [ "blocked", "own" ], "meta-externalfetcher": [ "blocked", "own" ], "meta-webindexer": [ "blocked", "star" ], "FacebookBot": [ "blocked", "star" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "blocked", "star" ], "Amzn-SearchBot": [ "blocked", "star" ], "Amzn-User": [ "blocked", "star" ], "MistralAI-User": [ "blocked", "star" ], "MistralAI-Index": [ "blocked", "star" ], "MistralAI-Training": [ "blocked", "star" ], "DuckAssistBot": [ "blocked", "own" ], "AI2Bot": [ "blocked", "star" ], "YouBot": [ "blocked", "star" ], "omgili": [ "blocked", "own" ], "omgilibot": [ "blocked", "own" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "blocked", "own" ], "Diffbot": [ "blocked", "own" ], "img2dataset": [ "blocked", "star" ] } }, { "website": "Medium", "domain": "medium.com", "robotsUrl": "https://medium.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked-except", "own" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked-except", "own" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "blocked-except", "own" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "blocked-except", "own" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "blocked-except", "own" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "blocked-except", "own" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "blocked-except", "own" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Meetup", "domain": "meetup.com", "robotsUrl": "https://meetup.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "MercadoLibre", "domain": "mercadolibre.com", "robotsUrl": "https://mercadolibre.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "blocked", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "blocked", "own" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "blocked", "own" ], "Perplexity-User": [ "blocked", "own" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "blocked", "own" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Mercari Japan", "domain": "jp.mercari.com", "robotsUrl": "https://jp.mercari.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Meta Ad Library", "domain": "facebook.com/ads/library", "robotsUrl": "https://facebook.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "blocked", "star" ], "ChatGPT-User": [ "blocked", "star" ], "OAI-AdsBot": [ "blocked", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "blocked", "star" ], "Claude-User": [ "blocked", "star" ], "anthropic-ai": [ "blocked", "star" ], "Claude-Web": [ "blocked", "star" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "blocked", "star" ], "Applebot-Extended": [ "blocked", "own" ], "PerplexityBot": [ "blocked", "own" ], "Perplexity-User": [ "blocked", "star" ], "meta-externalagent": [ "blocked", "star" ], "meta-externalfetcher": [ "blocked", "star" ], "meta-webindexer": [ "blocked", "star" ], "FacebookBot": [ "blocked", "star" ], "CCBot": [ "blocked", "star" ], "Amazonbot": [ "blocked", "own" ], "Amzn-SearchBot": [ "blocked", "star" ], "Amzn-User": [ "blocked", "star" ], "MistralAI-User": [ "blocked", "star" ], "MistralAI-Index": [ "blocked", "star" ], "MistralAI-Training": [ "blocked", "star" ], "DuckAssistBot": [ "blocked", "star" ], "AI2Bot": [ "blocked", "star" ], "YouBot": [ "blocked", "star" ], "omgili": [ "blocked", "star" ], "omgilibot": [ "blocked", "star" ], "Bytespider": [ "blocked", "star" ], "cohere-ai": [ "blocked", "star" ], "Diffbot": [ "blocked", "star" ], "img2dataset": [ "blocked", "star" ] } }, { "website": "MLB.com", "domain": "mlb.com", "robotsUrl": "https://mlb.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Naukri", "domain": "naukri.com", "robotsUrl": "https://naukri.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "blocked-except", "own" ], "ChatGPT-User": [ "blocked-except", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "blocked-except", "own" ], "Claude-User": [ "blocked-except", "own" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "blocked-except", "own" ], "Perplexity-User": [ "blocked-except", "own" ], "meta-externalagent": [ "blocked-except", "own" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "blocked-except", "own" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "blocked-except", "own" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "blocked-except", "own" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Naver", "domain": "naver.com", "robotsUrl": "https://naver.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "allowed", "star" ], "OAI-SearchBot": [ "allowed", "star" ], "ChatGPT-User": [ "allowed", "star" ], "OAI-AdsBot": [ "allowed", "star" ], "ClaudeBot": [ "allowed", "star" ], "Claude-SearchBot": [ "allowed", "star" ], "Claude-User": [ "allowed", "star" ], "anthropic-ai": [ "allowed", "star" ], "Claude-Web": [ "allowed", "star" ], "Google-Extended": [ "allowed", "star" ], "Google-CloudVertexBot": [ "allowed", "star" ], "Applebot-Extended": [ "allowed", "star" ], "PerplexityBot": [ "allowed", "star" ], "Perplexity-User": [ "allowed", "star" ], "meta-externalagent": [ "allowed", "star" ], "meta-externalfetcher": [ "allowed", "star" ], "meta-webindexer": [ "allowed", "star" ], "FacebookBot": [ "allowed", "star" ], "CCBot": [ "allowed", "star" ], "Amazonbot": [ "allowed", "star" ], "Amzn-SearchBot": [ "allowed", "star" ], "Amzn-User": [ "allowed", "star" ], "MistralAI-User": [ "allowed", "star" ], "MistralAI-Index": [ "allowed", "star" ], "MistralAI-Training": [ "allowed", "star" ], "DuckAssistBot": [ "allowed", "star" ], "AI2Bot": [ "allowed", "star" ], "YouBot": [ "allowed", "star" ], "omgili": [ "allowed", "star" ], "omgilibot": [ "allowed", "star" ], "Bytespider": [ "allowed", "star" ], "cohere-ai": [ "allowed", "star" ], "Diffbot": [ "allowed", "star" ], "img2dataset": [ "allowed", "star" ] } }, { "website": "Nike", "domain": "nike.com", "robotsUrl": "https://nike.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "OP.GG", "domain": "op.gg", "robotsUrl": "https://op.gg/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "allowed", "star" ], "OAI-SearchBot": [ "allowed", "star" ], "ChatGPT-User": [ "allowed", "star" ], "OAI-AdsBot": [ "allowed", "star" ], "ClaudeBot": [ "allowed", "star" ], "Claude-SearchBot": [ "allowed", "star" ], "Claude-User": [ "allowed", "star" ], "anthropic-ai": [ "allowed", "star" ], "Claude-Web": [ "allowed", "star" ], "Google-Extended": [ "allowed", "star" ], "Google-CloudVertexBot": [ "allowed", "star" ], "Applebot-Extended": [ "allowed", "star" ], "PerplexityBot": [ "allowed", "star" ], "Perplexity-User": [ "allowed", "star" ], "meta-externalagent": [ "allowed", "star" ], "meta-externalfetcher": [ "allowed", "star" ], "meta-webindexer": [ "allowed", "star" ], "FacebookBot": [ "allowed", "star" ], "CCBot": [ "allowed", "star" ], "Amazonbot": [ "allowed", "star" ], "Amzn-SearchBot": [ "allowed", "star" ], "Amzn-User": [ "allowed", "star" ], "MistralAI-User": [ "allowed", "star" ], "MistralAI-Index": [ "allowed", "star" ], "MistralAI-Training": [ "allowed", "star" ], "DuckAssistBot": [ "allowed", "star" ], "AI2Bot": [ "allowed", "star" ], "YouBot": [ "allowed", "star" ], "omgili": [ "allowed", "star" ], "omgilibot": [ "allowed", "star" ], "Bytespider": [ "allowed", "star" ], "cohere-ai": [ "allowed", "star" ], "Diffbot": [ "allowed", "star" ], "img2dataset": [ "allowed", "star" ] } }, { "website": "Pexels", "domain": "pexels.com", "robotsUrl": "https://pexels.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "blocked", "own" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Pinterest", "domain": "pinterest.com", "robotsUrl": "https://pinterest.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "star" ], "OAI-SearchBot": [ "blocked", "star" ], "ChatGPT-User": [ "blocked", "star" ], "OAI-AdsBot": [ "blocked", "star" ], "ClaudeBot": [ "blocked", "star" ], "Claude-SearchBot": [ "blocked", "star" ], "Claude-User": [ "blocked", "star" ], "anthropic-ai": [ "blocked", "star" ], "Claude-Web": [ "blocked", "star" ], "Google-Extended": [ "blocked", "star" ], "Google-CloudVertexBot": [ "blocked", "star" ], "Applebot-Extended": [ "blocked", "star" ], "PerplexityBot": [ "blocked", "star" ], "Perplexity-User": [ "blocked", "star" ], "meta-externalagent": [ "blocked", "star" ], "meta-externalfetcher": [ "blocked", "star" ], "meta-webindexer": [ "blocked", "star" ], "FacebookBot": [ "partial", "own" ], "CCBot": [ "blocked", "star" ], "Amazonbot": [ "blocked", "star" ], "Amzn-SearchBot": [ "blocked", "star" ], "Amzn-User": [ "blocked", "star" ], "MistralAI-User": [ "blocked", "star" ], "MistralAI-Index": [ "blocked", "star" ], "MistralAI-Training": [ "blocked", "star" ], "DuckAssistBot": [ "partial", "own" ], "AI2Bot": [ "blocked", "star" ], "YouBot": [ "blocked", "star" ], "omgili": [ "blocked", "star" ], "omgilibot": [ "blocked", "star" ], "Bytespider": [ "blocked", "star" ], "cohere-ai": [ "blocked", "star" ], "Diffbot": [ "blocked", "star" ], "img2dataset": [ "blocked", "star" ] } }, { "website": "Product Hunt", "domain": "producthunt.com", "robotsUrl": "https://producthunt.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Property Finder", "domain": "propertyfinder.ae", "robotsUrl": "https://propertyfinder.ae/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "QuintoAndar", "domain": "quintoandar.com.br", "robotsUrl": "https://www.quintoandar.com.br/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Quora", "domain": "quora.com", "robotsUrl": "https://quora.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "blocked-except", "own" ], "ChatGPT-User": [ "blocked", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "blocked", "own" ], "Claude-User": [ "blocked", "own" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "blocked", "own" ], "PerplexityBot": [ "blocked", "own" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "own" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "blocked", "own" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Realtor.com", "domain": "realtor.com", "robotsUrl": "https://realtor.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "blocked-except", "own" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "blocked", "own" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "blocked", "own" ], "meta-externalfetcher": [ "blocked", "own" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "blocked", "own" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Reddit", "domain": "reddit.com", "robotsUrl": "https://reddit.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "star" ], "OAI-SearchBot": [ "blocked", "star" ], "ChatGPT-User": [ "blocked", "star" ], "OAI-AdsBot": [ "blocked", "star" ], "ClaudeBot": [ "blocked", "star" ], "Claude-SearchBot": [ "blocked", "star" ], "Claude-User": [ "blocked", "star" ], "anthropic-ai": [ "blocked", "star" ], "Claude-Web": [ "blocked", "star" ], "Google-Extended": [ "blocked", "star" ], "Google-CloudVertexBot": [ "blocked", "star" ], "Applebot-Extended": [ "blocked", "star" ], "PerplexityBot": [ "blocked", "star" ], "Perplexity-User": [ "blocked", "star" ], "meta-externalagent": [ "blocked", "star" ], "meta-externalfetcher": [ "blocked", "star" ], "meta-webindexer": [ "blocked", "star" ], "FacebookBot": [ "blocked", "star" ], "CCBot": [ "blocked", "star" ], "Amazonbot": [ "blocked", "star" ], "Amzn-SearchBot": [ "blocked", "star" ], "Amzn-User": [ "blocked", "star" ], "MistralAI-User": [ "blocked", "star" ], "MistralAI-Index": [ "blocked", "star" ], "MistralAI-Training": [ "blocked", "star" ], "DuckAssistBot": [ "blocked", "star" ], "AI2Bot": [ "blocked", "star" ], "YouBot": [ "blocked", "star" ], "omgili": [ "blocked", "star" ], "omgilibot": [ "blocked", "star" ], "Bytespider": [ "blocked", "star" ], "cohere-ai": [ "blocked", "star" ], "Diffbot": [ "blocked", "star" ], "img2dataset": [ "blocked", "star" ] } }, { "website": "Reed", "domain": "reed.co.uk", "robotsUrl": "https://reed.co.uk/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "own" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "partial", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "own" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Shein", "domain": "shein.com", "robotsUrl": "https://shein.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Shopify stores", "domain": "shopify.com", "robotsUrl": "https://shopify.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Similarweb", "domain": "similarweb.com", "robotsUrl": "https://similarweb.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Skool", "domain": "skool.com", "robotsUrl": "https://skool.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Snapchat", "domain": "snapchat.com", "robotsUrl": "https://snapchat.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "partial", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "partial", "own" ], "Claude-User": [ "partial", "own" ], "anthropic-ai": [ "blocked", "own" ], "Claude-Web": [ "blocked", "own" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "blocked", "own" ], "PerplexityBot": [ "partial", "own" ], "Perplexity-User": [ "partial", "own" ], "meta-externalagent": [ "blocked", "own" ], "meta-externalfetcher": [ "blocked", "own" ], "meta-webindexer": [ "allowed", "own" ], "FacebookBot": [ "blocked", "own" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "blocked", "own" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "own" ], "AI2Bot": [ "blocked", "own" ], "YouBot": [ "blocked", "own" ], "omgili": [ "blocked", "own" ], "omgilibot": [ "blocked", "own" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "blocked", "own" ], "Diffbot": [ "blocked", "own" ], "img2dataset": [ "blocked", "own" ] } }, { "website": "SoundCloud", "domain": "soundcloud.com", "robotsUrl": "https://soundcloud.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "allowed", "own" ], "OAI-SearchBot": [ "allowed", "own" ], "ChatGPT-User": [ "allowed", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "allowed", "own" ], "Claude-SearchBot": [ "allowed", "own" ], "Claude-User": [ "allowed", "own" ], "anthropic-ai": [ "allowed", "own" ], "Claude-Web": [ "allowed", "own" ], "Google-Extended": [ "allowed", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "allowed", "own" ], "PerplexityBot": [ "allowed", "own" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "allowed", "own" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "allowed", "own" ], "Amazonbot": [ "allowed", "own" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "allowed", "own" ], "omgilibot": [ "allowed", "own" ], "Bytespider": [ "allowed", "own" ], "cohere-ai": [ "allowed", "own" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Spotify", "domain": "open.spotify.com", "robotsUrl": "https://open.spotify.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "allowed", "own" ], "ChatGPT-User": [ "allowed", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "allowed", "own" ], "Claude-User": [ "allowed", "own" ], "anthropic-ai": [ "blocked", "own" ], "Claude-Web": [ "blocked", "own" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "blocked", "own" ], "PerplexityBot": [ "allowed", "own" ], "Perplexity-User": [ "allowed", "own" ], "meta-externalagent": [ "blocked", "own" ], "meta-externalfetcher": [ "allowed", "own" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "blocked", "own" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "allowed", "own" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "blocked", "own" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Steam", "domain": "store.steampowered.com", "robotsUrl": "https://store.steampowered.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "StockX", "domain": "stockx.com", "robotsUrl": "https://stockx.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Substack", "domain": "substack.com", "robotsUrl": "https://substack.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Target", "domain": "target.com", "robotsUrl": "https://target.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Ticketmaster", "domain": "ticketmaster.com", "robotsUrl": "https://ticketmaster.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "TikTok", "domain": "tiktok.com", "robotsUrl": "https://tiktok.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "blocked", "own" ], "ChatGPT-User": [ "blocked", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "blocked", "own" ], "Claude-User": [ "blocked", "own" ], "anthropic-ai": [ "blocked", "own" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "blocked", "own" ], "PerplexityBot": [ "blocked", "own" ], "Perplexity-User": [ "blocked", "own" ], "meta-externalagent": [ "blocked", "own" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "blocked", "own" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "blocked", "own" ], "AI2Bot": [ "blocked", "own" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "TikTok Ads Library", "domain": "library.tiktok.com", "robotsUrl": "https://library.tiktok.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "star" ], "OAI-SearchBot": [ "blocked", "star" ], "ChatGPT-User": [ "blocked", "star" ], "OAI-AdsBot": [ "blocked", "star" ], "ClaudeBot": [ "blocked", "star" ], "Claude-SearchBot": [ "blocked", "star" ], "Claude-User": [ "blocked", "star" ], "anthropic-ai": [ "blocked", "star" ], "Claude-Web": [ "blocked", "star" ], "Google-Extended": [ "blocked", "star" ], "Google-CloudVertexBot": [ "blocked", "star" ], "Applebot-Extended": [ "blocked", "star" ], "PerplexityBot": [ "blocked", "star" ], "Perplexity-User": [ "blocked", "star" ], "meta-externalagent": [ "blocked", "star" ], "meta-externalfetcher": [ "blocked", "star" ], "meta-webindexer": [ "blocked", "star" ], "FacebookBot": [ "blocked", "star" ], "CCBot": [ "blocked", "star" ], "Amazonbot": [ "blocked", "star" ], "Amzn-SearchBot": [ "blocked", "star" ], "Amzn-User": [ "blocked", "star" ], "MistralAI-User": [ "blocked", "star" ], "MistralAI-Index": [ "blocked", "star" ], "MistralAI-Training": [ "blocked", "star" ], "DuckAssistBot": [ "blocked", "star" ], "AI2Bot": [ "blocked", "star" ], "YouBot": [ "blocked", "star" ], "omgili": [ "blocked", "star" ], "omgilibot": [ "blocked", "star" ], "Bytespider": [ "blocked", "star" ], "cohere-ai": [ "blocked", "star" ], "Diffbot": [ "blocked", "star" ], "img2dataset": [ "blocked", "star" ] } }, { "website": "Tokopedia", "domain": "tokopedia.com", "robotsUrl": "https://tokopedia.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Totaljobs", "domain": "totaljobs.com", "robotsUrl": "https://totaljobs.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "blocked", "own" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Trading Economics", "domain": "tradingeconomics.com", "robotsUrl": "https://tradingeconomics.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "allowed", "none" ], "OAI-SearchBot": [ "allowed", "none" ], "ChatGPT-User": [ "allowed", "none" ], "OAI-AdsBot": [ "allowed", "none" ], "ClaudeBot": [ "allowed", "none" ], "Claude-SearchBot": [ "allowed", "none" ], "Claude-User": [ "allowed", "none" ], "anthropic-ai": [ "allowed", "none" ], "Claude-Web": [ "allowed", "none" ], "Google-Extended": [ "allowed", "none" ], "Google-CloudVertexBot": [ "allowed", "none" ], "Applebot-Extended": [ "allowed", "none" ], "PerplexityBot": [ "allowed", "none" ], "Perplexity-User": [ "allowed", "none" ], "meta-externalagent": [ "allowed", "none" ], "meta-externalfetcher": [ "allowed", "none" ], "meta-webindexer": [ "allowed", "none" ], "FacebookBot": [ "allowed", "none" ], "CCBot": [ "allowed", "none" ], "Amazonbot": [ "allowed", "none" ], "Amzn-SearchBot": [ "allowed", "none" ], "Amzn-User": [ "allowed", "none" ], "MistralAI-User": [ "allowed", "none" ], "MistralAI-Index": [ "allowed", "none" ], "MistralAI-Training": [ "allowed", "none" ], "DuckAssistBot": [ "allowed", "none" ], "AI2Bot": [ "allowed", "none" ], "YouBot": [ "allowed", "none" ], "omgili": [ "allowed", "none" ], "omgilibot": [ "allowed", "none" ], "Bytespider": [ "allowed", "none" ], "cohere-ai": [ "allowed", "none" ], "Diffbot": [ "allowed", "none" ], "img2dataset": [ "allowed", "none" ] } }, { "website": "Transfermarkt", "domain": "transfermarkt.com", "robotsUrl": "https://transfermarkt.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "allowed", "star" ], "OAI-SearchBot": [ "allowed", "star" ], "ChatGPT-User": [ "allowed", "star" ], "OAI-AdsBot": [ "allowed", "star" ], "ClaudeBot": [ "allowed", "star" ], "Claude-SearchBot": [ "allowed", "star" ], "Claude-User": [ "allowed", "star" ], "anthropic-ai": [ "allowed", "star" ], "Claude-Web": [ "allowed", "star" ], "Google-Extended": [ "allowed", "star" ], "Google-CloudVertexBot": [ "allowed", "star" ], "Applebot-Extended": [ "allowed", "star" ], "PerplexityBot": [ "allowed", "star" ], "Perplexity-User": [ "allowed", "star" ], "meta-externalagent": [ "allowed", "star" ], "meta-externalfetcher": [ "allowed", "star" ], "meta-webindexer": [ "allowed", "star" ], "FacebookBot": [ "allowed", "star" ], "CCBot": [ "allowed", "star" ], "Amazonbot": [ "allowed", "star" ], "Amzn-SearchBot": [ "allowed", "star" ], "Amzn-User": [ "allowed", "star" ], "MistralAI-User": [ "allowed", "star" ], "MistralAI-Index": [ "allowed", "star" ], "MistralAI-Training": [ "allowed", "star" ], "DuckAssistBot": [ "allowed", "star" ], "AI2Bot": [ "allowed", "star" ], "YouBot": [ "allowed", "star" ], "omgili": [ "allowed", "star" ], "omgilibot": [ "allowed", "star" ], "Bytespider": [ "allowed", "star" ], "cohere-ai": [ "allowed", "star" ], "Diffbot": [ "allowed", "star" ], "img2dataset": [ "allowed", "star" ] } }, { "website": "Trendyol", "domain": "trendyol.com", "robotsUrl": "https://trendyol.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Trip.com", "domain": "trip.com", "robotsUrl": "https://trip.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Tripadvisor", "domain": "tripadvisor.com", "robotsUrl": "https://tripadvisor.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "partial", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "blocked-except", "own" ], "Google-CloudVertexBot": [ "blocked", "own" ], "Applebot-Extended": [ "blocked", "own" ], "PerplexityBot": [ "partial", "own" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "blocked", "own" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "blocked", "own" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "blocked", "own" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "blocked", "own" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "blocked", "own" ] } }, { "website": "Trulia", "domain": "trulia.com", "robotsUrl": "https://trulia.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Trustpilot", "domain": "trustpilot.com", "robotsUrl": "https://trustpilot.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "own" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "partial", "own" ], "OAI-AdsBot": [ "blocked", "star" ], "ClaudeBot": [ "partial", "own" ], "Claude-SearchBot": [ "partial", "own" ], "Claude-User": [ "partial", "own" ], "anthropic-ai": [ "blocked", "own" ], "Claude-Web": [ "blocked", "star" ], "Google-Extended": [ "partial", "own" ], "Google-CloudVertexBot": [ "blocked", "star" ], "Applebot-Extended": [ "blocked", "own" ], "PerplexityBot": [ "partial", "own" ], "Perplexity-User": [ "blocked", "star" ], "meta-externalagent": [ "partial", "own" ], "meta-externalfetcher": [ "partial", "own" ], "meta-webindexer": [ "partial", "own" ], "FacebookBot": [ "partial", "own" ], "CCBot": [ "partial", "own" ], "Amazonbot": [ "blocked", "star" ], "Amzn-SearchBot": [ "blocked", "star" ], "Amzn-User": [ "blocked", "star" ], "MistralAI-User": [ "blocked", "star" ], "MistralAI-Index": [ "blocked", "star" ], "MistralAI-Training": [ "blocked", "star" ], "DuckAssistBot": [ "blocked", "star" ], "AI2Bot": [ "blocked", "star" ], "YouBot": [ "blocked", "own" ], "omgili": [ "blocked", "own" ], "omgilibot": [ "blocked", "own" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "blocked", "own" ], "Diffbot": [ "blocked", "own" ], "img2dataset": [ "blocked", "star" ] } }, { "website": "Twitch", "domain": "twitch.tv", "robotsUrl": "https://twitch.tv/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "allowed", "own" ], "Amzn-SearchBot": [ "allowed", "own" ], "Amzn-User": [ "allowed", "own" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Uber Eats", "domain": "ubereats.com", "robotsUrl": "https://ubereats.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Ubersuggest", "domain": "ubersuggest.com", "robotsUrl": "https://ubersuggest.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "own" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "partial", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "own" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "own" ], "Claude-Web": [ "partial", "own" ], "Google-Extended": [ "partial", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "own" ], "PerplexityBot": [ "partial", "own" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "own" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Unsplash", "domain": "unsplash.com", "robotsUrl": "https://unsplash.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "blocked", "own" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Vimeo", "domain": "vimeo.com", "robotsUrl": "https://vimeo.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "allowed", "own" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "allowed", "own" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "allowed", "own" ], "Claude-Web": [ "allowed", "own" ], "Google-Extended": [ "allowed", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "blocked", "own" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "blocked", "own" ], "omgilibot": [ "blocked", "own" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "blocked", "own" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Vinted", "domain": "vinted.com", "robotsUrl": "https://vinted.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "blocked", "own" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Vivino", "domain": "vivino.com", "robotsUrl": "https://vivino.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "blocked", "own" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "blocked", "own" ], "YouBot": [ "partial", "star" ], "omgili": [ "blocked", "own" ], "omgilibot": [ "blocked", "own" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "blocked", "own" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Vrbo", "domain": "vrbo.com", "robotsUrl": "https://vrbo.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "partial", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "own" ], "Claude-User": [ "partial", "own" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "own" ], "Perplexity-User": [ "partial", "own" ], "meta-externalagent": [ "partial", "own" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "own" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Walk Score", "domain": "walkscore.com", "robotsUrl": "https://walkscore.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Walmart", "domain": "walmart.com", "robotsUrl": "https://walmart.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "We Work Remotely", "domain": "weworkremotely.com", "robotsUrl": "https://weworkremotely.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Welcome to the Jungle", "domain": "welcometothejungle.com", "robotsUrl": "https://welcometothejungle.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Wellfound", "domain": "wellfound.com", "robotsUrl": "https://wellfound.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Wikipedia", "domain": "wikipedia.org", "robotsUrl": "https://wikipedia.org/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Willhaben", "domain": "willhaben.at", "robotsUrl": "https://willhaben.at/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "X (Twitter)", "domain": "x.com", "robotsUrl": "https://x.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "star" ], "OAI-SearchBot": [ "blocked", "star" ], "ChatGPT-User": [ "blocked", "star" ], "OAI-AdsBot": [ "blocked", "star" ], "ClaudeBot": [ "blocked", "star" ], "Claude-SearchBot": [ "blocked", "star" ], "Claude-User": [ "blocked", "star" ], "anthropic-ai": [ "blocked", "star" ], "Claude-Web": [ "blocked", "star" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "blocked", "star" ], "Applebot-Extended": [ "blocked", "star" ], "PerplexityBot": [ "blocked", "star" ], "Perplexity-User": [ "blocked", "star" ], "meta-externalagent": [ "blocked", "own" ], "meta-externalfetcher": [ "blocked", "own" ], "meta-webindexer": [ "blocked", "own" ], "FacebookBot": [ "blocked", "own" ], "CCBot": [ "blocked", "star" ], "Amazonbot": [ "blocked", "star" ], "Amzn-SearchBot": [ "blocked", "star" ], "Amzn-User": [ "blocked", "star" ], "MistralAI-User": [ "blocked", "star" ], "MistralAI-Index": [ "blocked", "star" ], "MistralAI-Training": [ "blocked", "star" ], "DuckAssistBot": [ "blocked", "star" ], "AI2Bot": [ "blocked", "star" ], "YouBot": [ "blocked", "star" ], "omgili": [ "blocked", "star" ], "omgilibot": [ "blocked", "star" ], "Bytespider": [ "blocked", "star" ], "cohere-ai": [ "blocked", "star" ], "Diffbot": [ "blocked", "star" ], "img2dataset": [ "blocked", "star" ] } }, { "website": "Y Combinator", "domain": "ycombinator.com", "robotsUrl": "https://ycombinator.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Yahoo Finance", "domain": "finance.yahoo.com", "robotsUrl": "https://finance.yahoo.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "blocked", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "blocked", "own" ], "Claude-Web": [ "blocked", "own" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "blocked", "own" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "blocked", "own" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "blocked", "own" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "blocked", "own" ], "omgili": [ "blocked", "own" ], "omgilibot": [ "blocked", "own" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "blocked", "own" ], "Diffbot": [ "blocked", "own" ], "img2dataset": [ "blocked", "own" ] } }, { "website": "Yahoo News", "domain": "news.yahoo.com", "robotsUrl": "https://news.yahoo.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "blocked", "own" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "partial", "own" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "blocked", "own" ], "Claude-Web": [ "blocked", "own" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "blocked", "own" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "blocked", "own" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "blocked", "own" ], "omgili": [ "blocked", "own" ], "omgilibot": [ "blocked", "own" ], "Bytespider": [ "blocked", "own" ], "cohere-ai": [ "blocked", "own" ], "Diffbot": [ "blocked", "own" ], "img2dataset": [ "blocked", "own" ] } }, { "website": "Yandex", "domain": "yandex.com", "robotsUrl": "https://yandex.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Yelp", "domain": "yelp.com", "robotsUrl": "https://yelp.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "blocked", "own" ], "OAI-SearchBot": [ "blocked-except", "own" ], "ChatGPT-User": [ "blocked-except", "own" ], "OAI-AdsBot": [ "blocked", "star" ], "ClaudeBot": [ "blocked", "own" ], "Claude-SearchBot": [ "blocked-except", "own" ], "Claude-User": [ "blocked-except", "own" ], "anthropic-ai": [ "blocked", "own" ], "Claude-Web": [ "blocked", "own" ], "Google-Extended": [ "blocked", "own" ], "Google-CloudVertexBot": [ "blocked", "star" ], "Applebot-Extended": [ "blocked", "star" ], "PerplexityBot": [ "blocked-except", "own" ], "Perplexity-User": [ "blocked-except", "own" ], "meta-externalagent": [ "blocked", "own" ], "meta-externalfetcher": [ "blocked", "own" ], "meta-webindexer": [ "blocked", "star" ], "FacebookBot": [ "blocked", "star" ], "CCBot": [ "blocked", "own" ], "Amazonbot": [ "blocked", "star" ], "Amzn-SearchBot": [ "blocked", "star" ], "Amzn-User": [ "blocked", "star" ], "MistralAI-User": [ "blocked", "star" ], "MistralAI-Index": [ "blocked", "star" ], "MistralAI-Training": [ "blocked", "star" ], "DuckAssistBot": [ "blocked", "star" ], "AI2Bot": [ "blocked", "star" ], "YouBot": [ "blocked", "star" ], "omgili": [ "blocked", "star" ], "omgilibot": [ "blocked", "star" ], "Bytespider": [ "blocked", "star" ], "cohere-ai": [ "blocked", "star" ], "Diffbot": [ "blocked", "star" ], "img2dataset": [ "blocked", "star" ] } }, { "website": "YouTube", "domain": "youtube.com", "robotsUrl": "https://youtube.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "star" ], "Claude-SearchBot": [ "partial", "star" ], "Claude-User": [ "partial", "star" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "partial", "star" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "partial", "star" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "partial", "star" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "partial", "star" ], "YouBot": [ "partial", "star" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "Zillow", "domain": "zillow.com", "robotsUrl": "https://zillow.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "partial", "star" ], "OAI-SearchBot": [ "partial", "star" ], "ChatGPT-User": [ "partial", "star" ], "OAI-AdsBot": [ "partial", "star" ], "ClaudeBot": [ "partial", "own" ], "Claude-SearchBot": [ "partial", "own" ], "Claude-User": [ "partial", "own" ], "anthropic-ai": [ "partial", "star" ], "Claude-Web": [ "partial", "star" ], "Google-Extended": [ "partial", "star" ], "Google-CloudVertexBot": [ "partial", "star" ], "Applebot-Extended": [ "blocked-except", "own" ], "PerplexityBot": [ "partial", "star" ], "Perplexity-User": [ "partial", "star" ], "meta-externalagent": [ "blocked-except", "own" ], "meta-externalfetcher": [ "partial", "star" ], "meta-webindexer": [ "partial", "star" ], "FacebookBot": [ "partial", "star" ], "CCBot": [ "partial", "star" ], "Amazonbot": [ "blocked-except", "own" ], "Amzn-SearchBot": [ "partial", "star" ], "Amzn-User": [ "partial", "star" ], "MistralAI-User": [ "partial", "star" ], "MistralAI-Index": [ "partial", "star" ], "MistralAI-Training": [ "partial", "star" ], "DuckAssistBot": [ "partial", "star" ], "AI2Bot": [ "blocked-except", "own" ], "YouBot": [ "blocked-except", "own" ], "omgili": [ "partial", "star" ], "omgilibot": [ "partial", "star" ], "Bytespider": [ "partial", "star" ], "cohere-ai": [ "partial", "star" ], "Diffbot": [ "partial", "star" ], "img2dataset": [ "partial", "star" ] } }, { "website": "ZipRecruiter", "domain": "ziprecruiter.com", "robotsUrl": "https://ziprecruiter.com/robots.txt", "checked": "2026-11-14", "access": { "GPTBot": [ "allowed", "own" ], "OAI-SearchBot": [ "partial", "own" ], "ChatGPT-User": [ "partial", "own" ], "OAI-AdsBot": [ "partial", "own" ], "ClaudeBot": [ "partial", "own" ], "Claude-SearchBot": [ "partial", "own" ], "Claude-User": [ "partial", "own" ], "anthropic-ai": [ "blocked-except", "star" ], "Claude-Web": [ "blocked-except", "star" ], "Google-Extended": [ "partial", "own" ], "Google-CloudVertexBot": [ "blocked-except", "star" ], "Applebot-Extended": [ "allowed", "own" ], "PerplexityBot": [ "partial", "own" ], "Perplexity-User": [ "blocked-except", "star" ], "meta-externalagent": [ "allowed", "own" ], "meta-externalfetcher": [ "blocked-except", "star" ], "meta-webindexer": [ "blocked-except", "star" ], "FacebookBot": [ "blocked-except", "star" ], "CCBot": [ "allowed", "own" ], "Amazonbot": [ "blocked-except", "star" ], "Amzn-SearchBot": [ "blocked-except", "star" ], "Amzn-User": [ "blocked-except", "star" ], "MistralAI-User": [ "blocked-except", "star" ], "MistralAI-Index": [ "blocked-except", "star" ], "MistralAI-Training": [ "blocked-except", "star" ], "DuckAssistBot": [ "blocked-except", "star" ], "AI2Bot": [ "blocked-except", "star" ], "YouBot": [ "blocked-except", "star" ], "omgili": [ "blocked-except", "star" ], "omgilibot": [ "blocked-except", "star" ], "Bytespider": [ "blocked-except", "star" ], "cohere-ai": [ "blocked-except", "star" ], "Diffbot": [ "blocked-except", "star" ], "img2dataset": [ "blocked-except", "star" ] } } ] }