{
 "key": "a/what-it-takes",
 "path": "/what-it-takes/",
 "title": "What it takes to get data from 172 major websites",
 "checkedFrom": "2026-11-09",
 "checkedTo": "2026-11-13",
 "updated": "2026-11-13",
 "hash": "453c019a5a975ec0",
 "totals": {
  "websites": 172,
  "apiPublic": 39,
  "apiLimited": 47,
  "apiPartner": 36,
  "apiNone": 45,
  "apiRetired": 5,
  "apiPublicPct": 23,
  "apiNoOpenPct": 77,
  "robotsRead": 130,
  "robotsBlocksAll": 14,
  "robotsSomePaths": 111,
  "robotsNoRules": 5,
  "robotsCrawlDelay": 5,
  "robotsNamesAi": 63,
  "robotsRefused": 22,
  "termsRefused": 52,
  "plainRefusedEither": 59,
  "plainRefusedEitherPct": 34,
  "courtCases": 8,
  "openSource": 88
 },
 "aiCrawlers": [
  {
   "name": "GPTBot",
   "websites": 44
  },
  {
   "name": "ClaudeBot",
   "websites": 41
  },
  {
   "name": "ChatGPT-User",
   "websites": 35
  },
  {
   "name": "CCBot",
   "websites": 34
  },
  {
   "name": "Google-Extended",
   "websites": 32
  },
  {
   "name": "PerplexityBot",
   "websites": 32
  },
  {
   "name": "Bytespider",
   "websites": 31
  },
  {
   "name": "OAI-SearchBot",
   "websites": 30
  },
  {
   "name": "meta-externalagent",
   "websites": 27
  },
  {
   "name": "Applebot-Extended",
   "websites": 25
  },
  {
   "name": "Amazonbot",
   "websites": 24
  },
  {
   "name": "Claude-SearchBot",
   "websites": 22
  },
  {
   "name": "Claude-User",
   "websites": 22
  },
  {
   "name": "anthropic-ai",
   "websites": 21
  },
  {
   "name": "Diffbot",
   "websites": 16
  },
  {
   "name": "FacebookBot",
   "websites": 16
  },
  {
   "name": "Claude-Web",
   "websites": 15
  },
  {
   "name": "cohere-ai",
   "websites": 14
  },
  {
   "name": "Perplexity-User",
   "websites": 14
  },
  {
   "name": "omgilibot",
   "websites": 13
  },
  {
   "name": "AI2Bot",
   "websites": 10
  },
  {
   "name": "meta-externalfetcher",
   "websites": 10
  },
  {
   "name": "YouBot",
   "websites": 10
  },
  {
   "name": "DuckAssistBot",
   "websites": 9
  },
  {
   "name": "img2dataset",
   "websites": 8
  },
  {
   "name": "MistralAI-User",
   "websites": 4
  }
 ],
 "robotsAnswers": {
  "answered": 137,
  "refused": 22,
  "not-found": 4,
  "no-answer": 9,
  "not-read": 0
 },
 "termsAnswers": {
  "answered": 111,
  "refused": 52,
  "not-found": 0,
  "no-answer": 4,
  "not-read": 5
 },
 "closedBoth": [
  {
   "name": "Facebook",
   "href": "/sites/facebook/",
   "api": "limited"
  },
  {
   "name": "Google News",
   "href": "/sites/google-news/",
   "api": "none"
  },
  {
   "name": "IMDb",
   "href": "/sites/imdb/",
   "api": "limited"
  },
  {
   "name": "LinkedIn",
   "href": "/sites/linkedin/",
   "api": "partner"
  },
  {
   "name": "Naver",
   "href": "/sites/naver/",
   "api": "limited"
  },
  {
   "name": "Quora",
   "href": "/sites/quora/",
   "api": "none"
  },
  {
   "name": "Reddit",
   "href": "/sites/reddit/",
   "api": "limited"
  },
  {
   "name": "TikTok Ads Library",
   "href": "/sites/tiktok-ads-library/",
   "api": "limited"
  },
  {
   "name": "Trustpilot",
   "href": "/sites/trustpilot/",
   "api": "limited"
  },
  {
   "name": "ZipRecruiter",
   "href": "/sites/ziprecruiter/",
   "api": "partner"
  }
 ],
 "byCategory": [
  {
   "name": "Ecommerce",
   "websites": 36,
   "apiPublic": 4,
   "apiLimited": 9,
   "apiPartner": 13,
   "apiNone": 9,
   "apiRetired": 1,
   "apiPublicPct": 11,
   "apiNoOpenPct": 89,
   "robotsRead": 20,
   "robotsBlocksAll": 0,
   "robotsSomePaths": 20,
   "robotsNoRules": 0,
   "robotsCrawlDelay": 0,
   "robotsNamesAi": 8,
   "robotsRefused": 7,
   "termsRefused": 12,
   "plainRefusedEither": 13,
   "plainRefusedEitherPct": 36,
   "courtCases": 2,
   "openSource": 22
  },
  {
   "name": "Jobs",
   "websites": 20,
   "apiPublic": 6,
   "apiLimited": 4,
   "apiPartner": 2,
   "apiNone": 8,
   "apiRetired": 0,
   "apiPublicPct": 30,
   "apiNoOpenPct": 70,
   "robotsRead": 17,
   "robotsBlocksAll": 1,
   "robotsSomePaths": 14,
   "robotsNoRules": 2,
   "robotsCrawlDelay": 1,
   "robotsNamesAi": 10,
   "robotsRefused": 1,
   "termsRefused": 7,
   "plainRefusedEither": 8,
   "plainRefusedEitherPct": 40,
   "courtCases": 0,
   "openSource": 4
  },
  {
   "name": "Real estate",
   "websites": 13,
   "apiPublic": 0,
   "apiLimited": 1,
   "apiPartner": 3,
   "apiNone": 9,
   "apiRetired": 0,
   "apiPublicPct": 0,
   "apiNoOpenPct": 100,
   "robotsRead": 9,
   "robotsBlocksAll": 0,
   "robotsSomePaths": 9,
   "robotsNoRules": 0,
   "robotsCrawlDelay": 0,
   "robotsNamesAi": 3,
   "robotsRefused": 4,
   "termsRefused": 6,
   "plainRefusedEither": 7,
   "plainRefusedEitherPct": 54,
   "courtCases": 0,
   "openSource": 7
  },
  {
   "name": "Social network",
   "websites": 10,
   "apiPublic": 3,
   "apiLimited": 6,
   "apiPartner": 0,
   "apiNone": 1,
   "apiRetired": 0,
   "apiPublicPct": 30,
   "apiNoOpenPct": 70,
   "robotsRead": 8,
   "robotsBlocksAll": 4,
   "robotsSomePaths": 4,
   "robotsNoRules": 0,
   "robotsCrawlDelay": 1,
   "robotsNamesAi": 5,
   "robotsRefused": 0,
   "termsRefused": 3,
   "plainRefusedEither": 3,
   "plainRefusedEitherPct": 30,
   "courtCases": 4,
   "openSource": 7
  },
  {
   "name": "Travel",
   "websites": 10,
   "apiPublic": 0,
   "apiLimited": 0,
   "apiPartner": 9,
   "apiNone": 1,
   "apiRetired": 0,
   "apiPublicPct": 0,
   "apiNoOpenPct": 100,
   "robotsRead": 8,
   "robotsBlocksAll": 0,
   "robotsSomePaths": 8,
   "robotsNoRules": 0,
   "robotsCrawlDelay": 0,
   "robotsNamesAi": 5,
   "robotsRefused": 1,
   "termsRefused": 4,
   "plainRefusedEither": 5,
   "plainRefusedEitherPct": 50,
   "courtCases": 0,
   "openSource": 4
  },
  {
   "name": "Sports and gaming",
   "websites": 6,
   "apiPublic": 1,
   "apiLimited": 2,
   "apiPartner": 0,
   "apiNone": 3,
   "apiRetired": 0,
   "apiPublicPct": 17,
   "apiNoOpenPct": 83,
   "robotsRead": 5,
   "robotsBlocksAll": 0,
   "robotsSomePaths": 3,
   "robotsNoRules": 2,
   "robotsCrawlDelay": 0,
   "robotsNamesAi": 1,
   "robotsRefused": 1,
   "termsRefused": 2,
   "plainRefusedEither": 2,
   "plainRefusedEitherPct": 33,
   "courtCases": 0,
   "openSource": 3
  },
  {
   "name": "Local and place data",
   "websites": 5,
   "apiPublic": 3,
   "apiLimited": 0,
   "apiPartner": 1,
   "apiNone": 0,
   "apiRetired": 1,
   "apiPublicPct": 60,
   "apiNoOpenPct": 40,
   "robotsRead": 3,
   "robotsBlocksAll": 0,
   "robotsSomePaths": 3,
   "robotsNoRules": 0,
   "robotsCrawlDelay": 0,
   "robotsNamesAi": 1,
   "robotsRefused": 0,
   "termsRefused": 0,
   "plainRefusedEither": 0,
   "plainRefusedEitherPct": 0,
   "courtCases": 0,
   "openSource": 1
  },
  {
   "name": "News and publishing",
   "websites": 5,
   "apiPublic": 1,
   "apiLimited": 2,
   "apiPartner": 0,
   "apiNone": 2,
   "apiRetired": 0,
   "apiPublicPct": 20,
   "apiNoOpenPct": 80,
   "robotsRead": 5,
   "robotsBlocksAll": 1,
   "robotsSomePaths": 4,
   "robotsNoRules": 0,
   "robotsCrawlDelay": 1,
   "robotsNamesAi": 3,
   "robotsRefused": 0,
   "termsRefused": 1,
   "plainRefusedEither": 1,
   "plainRefusedEitherPct": 20,
   "courtCases": 0,
   "openSource": 3
  },
  {
   "name": "Video",
   "websites": 5,
   "apiPublic": 3,
   "apiLimited": 2,
   "apiPartner": 0,
   "apiNone": 0,
   "apiRetired": 0,
   "apiPublicPct": 60,
   "apiNoOpenPct": 40,
   "robotsRead": 5,
   "robotsBlocksAll": 0,
   "robotsSomePaths": 5,
   "robotsNoRules": 0,
   "robotsCrawlDelay": 1,
   "robotsNamesAi": 2,
   "robotsRefused": 0,
   "termsRefused": 0,
   "plainRefusedEither": 0,
   "plainRefusedEitherPct": 0,
   "courtCases": 0,
   "openSource": 5
  },
  {
   "name": "Ads and transparency",
   "websites": 4,
   "apiPublic": 0,
   "apiLimited": 2,
   "apiPartner": 0,
   "apiNone": 2,
   "apiRetired": 0,
   "apiPublicPct": 0,
   "apiNoOpenPct": 100,
   "robotsRead": 1,
   "robotsBlocksAll": 1,
   "robotsSomePaths": 0,
   "robotsNoRules": 0,
   "robotsCrawlDelay": 0,
   "robotsNamesAi": 0,
   "robotsRefused": 2,
   "termsRefused": 1,
   "plainRefusedEither": 2,
   "plainRefusedEitherPct": 50,
   "courtCases": 0,
   "openSource": 1
  },
  {
   "name": "Finance and markets",
   "websites": 4,
   "apiPublic": 2,
   "apiLimited": 0,
   "apiPartner": 1,
   "apiNone": 1,
   "apiRetired": 0,
   "apiPublicPct": 50,
   "apiNoOpenPct": 50,
   "robotsRead": 2,
   "robotsBlocksAll": 0,
   "robotsSomePaths": 1,
   "robotsNoRules": 1,
   "robotsCrawlDelay": 0,
   "robotsNamesAi": 1,
   "robotsRefused": 2,
   "termsRefused": 1,
   "plainRefusedEither": 2,
   "plainRefusedEitherPct": 50,
   "courtCases": 0,
   "openSource": 2
  },
  {
   "name": "Media and entertainment",
   "websites": 4,
   "apiPublic": 0,
   "apiLimited": 3,
   "apiPartner": 0,
   "apiNone": 0,
   "apiRetired": 1,
   "apiPublicPct": 0,
   "apiNoOpenPct": 100,
   "robotsRead": 4,
   "robotsBlocksAll": 1,
   "robotsSomePaths": 3,
   "robotsNoRules": 0,
   "robotsCrawlDelay": 0,
   "robotsNamesAi": 4,
   "robotsRefused": 0,
   "termsRefused": 0,
   "plainRefusedEither": 0,
   "plainRefusedEitherPct": 0,
   "courtCases": 0,
   "openSource": 4
  },
  {
   "name": "Research and reference",
   "websites": 4,
   "apiPublic": 3,
   "apiLimited": 0,
   "apiPartner": 0,
   "apiNone": 1,
   "apiRetired": 0,
   "apiPublicPct": 75,
   "apiNoOpenPct": 25,
   "robotsRead": 3,
   "robotsBlocksAll": 0,
   "robotsSomePaths": 3,
   "robotsNoRules": 0,
   "robotsCrawlDelay": 0,
   "robotsNamesAi": 0,
   "robotsRefused": 0,
   "termsRefused": 0,
   "plainRefusedEither": 0,
   "plainRefusedEitherPct": 0,
   "courtCases": 0,
   "openSource": 4
  },
  {
   "name": "Search results",
   "websites": 4,
   "apiPublic": 1,
   "apiLimited": 2,
   "apiPartner": 0,
   "apiNone": 0,
   "apiRetired": 1,
   "apiPublicPct": 25,
   "apiNoOpenPct": 75,
   "robotsRead": 4,
   "robotsBlocksAll": 1,
   "robotsSomePaths": 3,
   "robotsNoRules": 0,
   "robotsCrawlDelay": 0,
   "robotsNamesAi": 0,
   "robotsRefused": 0,
   "termsRefused": 0,
   "plainRefusedEither": 0,
   "plainRefusedEitherPct": 0,
   "courtCases": 1,
   "openSource": 3
  },
  {
   "name": "SEO and web analytics",
   "websites": 4,
   "apiPublic": 1,
   "apiLimited": 3,
   "apiPartner": 0,
   "apiNone": 0,
   "apiRetired": 0,
   "apiPublicPct": 25,
   "apiNoOpenPct": 75,
   "robotsRead": 4,
   "robotsBlocksAll": 0,
   "robotsSomePaths": 4,
   "robotsNoRules": 0,
   "robotsCrawlDelay": 0,
   "robotsNamesAi": 1,
   "robotsRefused": 0,
   "termsRefused": 1,
   "plainRefusedEither": 1,
   "plainRefusedEitherPct": 25,
   "courtCases": 0,
   "openSource": 2
  },
  {
   "name": "Other",
   "websites": 38,
   "apiPublic": 11,
   "apiLimited": 11,
   "apiPartner": 7,
   "apiNone": 8,
   "apiRetired": 1,
   "apiPublicPct": 29,
   "apiNoOpenPct": 71,
   "robotsRead": 32,
   "robotsBlocksAll": 5,
   "robotsSomePaths": 27,
   "robotsNoRules": 0,
   "robotsCrawlDelay": 1,
   "robotsNamesAi": 19,
   "robotsRefused": 4,
   "termsRefused": 14,
   "plainRefusedEither": 15,
   "plainRefusedEitherPct": 39,
   "courtCases": 1,
   "openSource": 16
  }
 ],
 "rows": [
  {
   "slug": "2gis",
   "name": "2GIS",
   "domain": "2gis.com",
   "category": "Local and place data",
   "href": "/sites/2gis/",
   "api": "public",
   "apiName": "2GIS Places API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://2gis.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 13,
    "crawlDelay": null,
    "namedAgents": 3,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://law.2gis.ru/licensing-agreement",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "99acres",
   "name": "99acres",
   "domain": "99acres.com",
   "category": "Real estate",
   "href": "/sites/99acres/",
   "api": "none",
   "apiName": "No public listing API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.99acres.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.99acres.com/terms-and-conditions",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 15,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "ahrefs",
   "name": "Ahrefs",
   "domain": "ahrefs.com",
   "category": "SEO and web analytics",
   "href": "/sites/ahrefs/",
   "api": "public",
   "apiName": "Ahrefs API v3",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://ahrefs.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 24,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://ahrefs.com/legal/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "airbnb",
   "name": "Airbnb",
   "domain": "airbnb.com",
   "category": "Travel",
   "href": "/sites/airbnb/",
   "api": "none",
   "apiName": "No public listing API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://airbnb.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 61,
    "crawlDelay": null,
    "namedAgents": 17,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "anthropic-ai",
     "PerplexityBot",
     "Applebot-Extended",
     "meta-externalagent",
     "cohere-ai",
     "AI2Bot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.airbnb.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "alibaba",
   "name": "Alibaba.com",
   "domain": "alibaba.com",
   "category": "B2B and trade data",
   "href": "/sites/alibaba/",
   "api": "partner",
   "apiName": "Alibaba.com Open Platform",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://alibaba.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 43,
    "crawlDelay": null,
    "namedAgents": 5,
    "aiCrawlersNamed": [
     "GPTBot",
     "Google-Extended"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://rule.alibaba.com/rule/detail/2041.htm",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "aliexpress",
   "name": "AliExpress",
   "domain": "aliexpress.com",
   "category": "Ecommerce",
   "href": "/sites/aliexpress/",
   "api": "partner",
   "apiName": "AliExpress Open Platform / Affiliate API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://aliexpress.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 35,
    "crawlDelay": null,
    "namedAgents": 8,
    "aiCrawlersNamed": [
     "GPTBot",
     "ClaudeBot",
     "Google-Extended",
     "CCBot",
     "Bytespider",
     "Amazonbot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.aliexpress.com/p/terms-conditions.html",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 5,
    "topStars": 329,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "allegro",
   "name": "Allegro",
   "domain": "allegro.pl",
   "category": "Ecommerce",
   "href": "/sites/allegro/",
   "api": "limited",
   "apiName": "Allegro REST API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://allegro.pl/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 33,
    "crawlDelay": null,
    "namedAgents": 2,
    "aiCrawlersNamed": [
     "CCBot"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://allegro.pl/regulamin",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "amazon",
   "name": "Amazon",
   "domain": "amazon.com",
   "category": "Ecommerce",
   "href": "/sites/amazon/",
   "api": "limited",
   "apiName": "Amazon Creators API (replaced the Product Advertising API) and Selling Partner API",
   "apiChecked": "2026-11-12",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://amazon.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 120,
    "crawlDelay": null,
    "namedAgents": 99,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-User",
     "Claude-SearchBot",
     "Claude-Web",
     "Google-Extended",
     "PerplexityBot",
     "Perplexity-User",
     "CCBot",
     "Bytespider",
     "meta-externalagent",
     "meta-externalfetcher",
     "cohere-ai",
     "Diffbot",
     "YouBot",
     "MistralAI-User",
     "DuckAssistBot",
     "AI2Bot",
     "img2dataset"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.amazon.com/gp/help/customer/display.html?nodeId=508088",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 1,
   "openSource": {
    "repos": 4,
    "topStars": 3419,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "apartments-com",
   "name": "Apartments.com",
   "domain": "apartments.com",
   "category": "Real estate",
   "href": "/sites/apartments-com/",
   "api": "none",
   "apiName": "No public listing API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.apartments.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.apartments.com/about/terms-of-service",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "app-store",
   "name": "Apple App Store",
   "domain": "apps.apple.com",
   "category": "App stores",
   "href": "/sites/app-store/",
   "api": "limited",
   "apiName": "iTunes Search API and App Store Connect API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://apps.apple.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 5,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.apple.com/legal/internet-services/itunes/",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "ashby",
   "name": "Ashby job boards",
   "domain": "jobs.ashbyhq.com",
   "category": "Jobs",
   "href": "/sites/ashby/",
   "api": "public",
   "apiName": "Ashby Public Job Posting API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://jobs.ashbyhq.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 3,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.ashbyhq.com/resources/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "autoscout24",
   "name": "AutoScout24",
   "domain": "autoscout24.com",
   "category": "Automotive",
   "href": "/sites/autoscout24/",
   "api": "partner",
   "apiName": "AutoScout24 Listing Creation API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://autoscout24.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 36,
    "crawlDelay": null,
    "namedAgents": 15,
    "aiCrawlersNamed": [
     "GPTBot",
     "ClaudeBot",
     "Google-Extended",
     "CCBot",
     "Applebot-Extended"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.autoscout24.com/company/agb/",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 25,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "baidu",
   "name": "Baidu",
   "domain": "baidu.com",
   "category": "Search results",
   "href": "/sites/baidu/",
   "api": "public",
   "apiName": "Baidu AI Search API (Qianfan web_search)",
   "apiChecked": "2026-11-10",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://baidu.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "all",
    "disallowRules": 1,
    "crawlDelay": null,
    "namedAgents": 15,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.baidu.com/duty/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "bayt",
   "name": "Bayt",
   "domain": "bayt.com",
   "category": "Jobs",
   "href": "/sites/bayt/",
   "api": "none",
   "apiName": "No public jobs API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://bayt.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 25,
    "crawlDelay": null,
    "namedAgents": 3,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.bayt.com/en/terms/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "bayut",
   "name": "Bayut",
   "domain": "bayut.com",
   "category": "Real estate",
   "href": "/sites/bayut/",
   "api": "none",
   "apiName": "No public listing API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://bayut.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 231,
    "crawlDelay": null,
    "namedAgents": 9,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 503,
    "url": "https://www.bayut.com/terms-and-conditions/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "best-buy",
   "name": "Best Buy",
   "domain": "bestbuy.com",
   "category": "Ecommerce",
   "href": "/sites/best-buy/",
   "api": "public",
   "apiName": "Best Buy Developer API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "no-answer",
    "status": 0,
    "url": "https://www.bestbuy.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://developer.bestbuy.com/legal",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "bilibili",
   "name": "Bilibili",
   "domain": "bilibili.com",
   "category": "Video",
   "href": "/sites/bilibili/",
   "api": "limited",
   "apiName": "Bilibili Open Platform",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://bilibili.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 10,
    "crawlDelay": null,
    "namedAgents": 8,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.bilibili.com/protocal/licence.html",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 3,
    "topStars": 5366,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "bing",
   "name": "Bing",
   "domain": "bing.com",
   "category": "Search results",
   "href": "/sites/bing/",
   "api": "retired",
   "apiName": "Bing Search APIs (retired)",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://bing.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 95,
    "crawlDelay": null,
    "namedAgents": 2,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "not-read",
    "status": null,
    "url": "https://www.microsoft.com/en-us/servicesagreement",
    "checked": "2026-11-13",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 375,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "bing-maps",
   "name": "Bing Maps",
   "domain": "bing.com/maps",
   "category": "Local and place data",
   "href": "/sites/bing-maps/",
   "api": "retired",
   "apiName": "Bing Maps for Enterprise (successor: Azure Maps)",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "not-found",
    "status": 404,
    "url": "https://www.bing.com/maps/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "not-read",
    "status": null,
    "url": "https://www.microsoft.com/en-us/servicesagreement",
    "checked": "2026-11-13",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "bizbuysell",
   "name": "BizBuySell",
   "domain": "bizbuysell.com",
   "category": "Ecommerce",
   "href": "/sites/bizbuysell/",
   "api": "none",
   "apiName": "No official BizBuySell data API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.bizbuysell.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.bizbuysell.com/terms-of-use/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 25,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "bluesky",
   "name": "Bluesky",
   "domain": "bsky.app",
   "category": "Social network",
   "href": "/sites/bluesky/",
   "api": "public",
   "apiName": "Bluesky API (AT Protocol)",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://bsky.app/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 1,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://bsky.social/about/support/tos",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "bol-com",
   "name": "Bol.com",
   "domain": "bol.com",
   "category": "Ecommerce",
   "href": "/sites/bol-com/",
   "api": "partner",
   "apiName": "bol Marketing Catalog API (affiliates) and Retailer API (sellers)",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://bol.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 55,
    "crawlDelay": null,
    "namedAgents": 6,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.bol.com/nl/nl/tc/algemene-voorwaarden-thuiswinkel",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "booking",
   "name": "Booking.com",
   "domain": "booking.com",
   "category": "Travel",
   "href": "/sites/booking/",
   "api": "partner",
   "apiName": "Booking.com Demand API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://booking.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 78,
    "crawlDelay": null,
    "namedAgents": 12,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.booking.com/content/terms.html",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 18,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "cnpj",
   "name": "Brazil CNPJ registry",
   "domain": "gov.br",
   "category": "Business registries",
   "href": "/sites/cnpj/",
   "api": "public",
   "apiName": "Receita Federal CNPJ open data (Dados Abertos)",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://gov.br/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 53,
    "crawlDelay": null,
    "namedAgents": 1,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.gov.br/pt-br/termos-de-uso",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "builtin",
   "name": "Built In",
   "domain": "builtin.com",
   "category": "Jobs",
   "href": "/sites/builtin/",
   "api": "none",
   "apiName": "No public Built In API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://builtin.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 119,
    "crawlDelay": null,
    "namedAgents": 4,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "PerplexityBot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://builtin.com/legal/community-terms-of-use",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": null
  },
  {
   "slug": "capterra",
   "name": "Capterra",
   "domain": "capterra.com",
   "category": "Software reviews",
   "href": "/sites/capterra/",
   "api": "none",
   "apiName": "No public Capterra data API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://capterra.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 19,
    "crawlDelay": null,
    "namedAgents": 44,
    "aiCrawlersNamed": [
     "Bytespider"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.capterra.com/legal/terms-of-use/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "coupang",
   "name": "Coupang",
   "domain": "coupang.com",
   "category": "Ecommerce",
   "href": "/sites/coupang/",
   "api": "partner",
   "apiName": "Coupang Open API (sellers) and Coupang Partners API (affiliates)",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.coupang.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.coupang.com/np/policies/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 3,
    "topStars": 51,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "coursera",
   "name": "Coursera",
   "domain": "coursera.org",
   "category": "Education",
   "href": "/sites/coursera/",
   "api": "limited",
   "apiName": "Coursera Developer Portal (Coursera for Business APIs)",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://coursera.org/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 25,
    "crawlDelay": null,
    "namedAgents": 12,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Google-Extended",
     "PerplexityBot",
     "CCBot",
     "meta-externalagent",
     "MistralAI-User"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.coursera.org/about/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 9672,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "craigslist",
   "name": "Craigslist",
   "domain": "craigslist.org",
   "category": "Ecommerce",
   "href": "/sites/craigslist/",
   "api": "none",
   "apiName": "No public listings API (bulk posting interface only)",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://craigslist.org/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 8,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.craigslist.org/about/terms.of.use",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 1,
   "openSource": {
    "repos": 7,
    "topStars": 81,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "crexi",
   "name": "Crexi",
   "domain": "crexi.com",
   "category": "Commercial real estate",
   "href": "/sites/crexi/",
   "api": "none",
   "apiName": "No public listing API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://crexi.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 34,
    "crawlDelay": null,
    "namedAgents": 8,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.crexi.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "crunchbase",
   "name": "Crunchbase",
   "domain": "crunchbase.com",
   "category": "Startups and products",
   "href": "/sites/crunchbase/",
   "api": "limited",
   "apiName": "Crunchbase API (v4)",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://crunchbase.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 17,
    "crawlDelay": null,
    "namedAgents": 25,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-User",
     "Claude-SearchBot",
     "Claude-Web",
     "anthropic-ai",
     "Google-Extended",
     "PerplexityBot",
     "Perplexity-User",
     "CCBot",
     "Bytespider",
     "Applebot-Extended",
     "meta-externalagent",
     "meta-externalfetcher",
     "FacebookBot",
     "cohere-ai",
     "Diffbot",
     "omgilibot",
     "AI2Bot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://about.crunchbase.com/terms-of-service/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 2,
    "topStars": 67,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "dailymotion",
   "name": "Dailymotion",
   "domain": "dailymotion.com",
   "category": "Video",
   "href": "/sites/dailymotion/",
   "api": "limited",
   "apiName": "Dailymotion API v2",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://dailymotion.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 107,
    "crawlDelay": null,
    "namedAgents": 6,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "not-read",
    "status": null,
    "url": "https://legal.dailymotion.com/en/terms-of-use/",
    "checked": "2026-11-13",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 10,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "deviantart",
   "name": "DeviantArt",
   "domain": "deviantart.com",
   "category": "Images and media",
   "href": "/sites/deviantart/",
   "api": "public",
   "apiName": "DeviantArt API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://deviantart.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 12,
    "crawlDelay": 1,
    "namedAgents": 1,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "no-answer",
    "status": 0,
    "url": "https://www.deviantart.com/about/policy/service/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 4,
    "topStars": 69,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "dexscreener",
   "name": "DEX Screener",
   "domain": "dexscreener.com",
   "category": "Finance and markets",
   "href": "/sites/dexscreener/",
   "api": "public",
   "apiName": "DEX Screener API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.dexscreener.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://docs.dexscreener.com/api/api-terms-and-conditions",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 3,
    "topStars": 80,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "doordash",
   "name": "DoorDash",
   "domain": "doordash.com",
   "category": "Food and drink",
   "href": "/sites/doordash/",
   "api": "partner",
   "apiName": "DoorDash Marketplace API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.doordash.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://help.doordash.com/en-us/legal/article/cx-terms-and-conditions",
    "checked": "2026-11-13",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "ebay",
   "name": "eBay",
   "domain": "ebay.com",
   "category": "Ecommerce",
   "href": "/sites/ebay/",
   "api": "public",
   "apiName": "eBay Browse API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://ebay.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 216,
    "crawlDelay": null,
    "namedAgents": 20,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-User",
     "Claude-SearchBot",
     "anthropic-ai",
     "PerplexityBot",
     "CCBot",
     "Bytespider",
     "Amazonbot",
     "Applebot-Extended",
     "meta-externalagent"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.ebay.com/help/policies/member-behaviour-policies/user-agreement?id=4259",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 2,
    "topStars": 557,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "espn",
   "name": "ESPN",
   "domain": "espn.com",
   "category": "Sports and gaming",
   "href": "/sites/espn/",
   "api": "none",
   "apiName": "No public ESPN API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://espn.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 103,
    "crawlDelay": null,
    "namedAgents": 10,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "anthropic-ai",
     "Google-Extended",
     "CCBot",
     "Bytespider",
     "FacebookBot",
     "omgilibot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://disneytermsofuse.com/english/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "etsy",
   "name": "Etsy",
   "domain": "etsy.com",
   "category": "Ecommerce",
   "href": "/sites/etsy/",
   "api": "public",
   "apiName": "Etsy Open API v3",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://etsy.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 1680,
    "crawlDelay": null,
    "namedAgents": 2,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.etsy.com/legal/terms-of-use",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 138,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "eventbrite",
   "name": "Eventbrite",
   "domain": "eventbrite.com",
   "category": "Events",
   "href": "/sites/eventbrite/",
   "api": "limited",
   "apiName": "Eventbrite API v3",
   "apiChecked": "2026-11-12",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://eventbrite.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 51,
    "crawlDelay": null,
    "namedAgents": 138,
    "aiCrawlersNamed": [
     "CCBot"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://support.eventbrite.com/hc/en-us/articles/34852909920018-Eventbrite-Terms-of-Service",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "expedia",
   "name": "Expedia",
   "domain": "expedia.com",
   "category": "Travel",
   "href": "/sites/expedia/",
   "api": "partner",
   "apiName": "Expedia Partner Solutions APIs",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://expedia.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 52,
    "crawlDelay": null,
    "namedAgents": 15,
    "aiCrawlersNamed": [
     "ChatGPT-User",
     "OAI-SearchBot",
     "Claude-User",
     "Claude-SearchBot",
     "PerplexityBot",
     "Perplexity-User",
     "meta-externalagent"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 429,
    "url": "https://www.expedia.com/p/info-other/legal.htm",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 23,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "facebook",
   "name": "Facebook",
   "domain": "facebook.com",
   "category": "Social network",
   "href": "/sites/facebook/",
   "api": "limited",
   "apiName": "Meta Graph API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://facebook.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "all",
    "disallowRules": 1,
    "crawlDelay": null,
    "namedAgents": 36,
    "aiCrawlersNamed": [
     "GPTBot",
     "ClaudeBot",
     "Google-Extended",
     "PerplexityBot",
     "Amazonbot",
     "Applebot-Extended"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 400,
    "url": "https://www.facebook.com/legal/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 4,
   "openSource": {
    "repos": 1,
    "topStars": 3327,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "finn-no",
   "name": "FINN.no",
   "domain": "finn.no",
   "category": "Real estate and marketplace",
   "href": "/sites/finn-no/",
   "api": "partner",
   "apiName": "FINN API",
   "apiChecked": "2026-11-12",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://finn.no/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 74,
    "crawlDelay": null,
    "namedAgents": 17,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "anthropic-ai",
     "Google-Extended",
     "CCBot",
     "Bytespider",
     "Amazonbot",
     "Applebot-Extended",
     "meta-externalagent"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.finn.no/vilkar/brukervilkar-finn",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "fiverr",
   "name": "Fiverr",
   "domain": "fiverr.com",
   "category": "Freelance marketplaces",
   "href": "/sites/fiverr/",
   "api": "limited",
   "apiName": "Fiverr MCP",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://fiverr.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 53,
    "crawlDelay": null,
    "namedAgents": 13,
    "aiCrawlersNamed": [
     "ChatGPT-User",
     "OAI-SearchBot",
     "Claude-User",
     "Claude-SearchBot",
     "Google-Extended",
     "PerplexityBot",
     "Perplexity-User",
     "Applebot-Extended",
     "YouBot",
     "MistralAI-User",
     "DuckAssistBot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.fiverr.com/legal-portal/legal-terms/terms-of-service",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 6,
    "topStars": 88,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "flipkart",
   "name": "Flipkart",
   "domain": "flipkart.com",
   "category": "Ecommerce",
   "href": "/sites/flipkart/",
   "api": "partner",
   "apiName": "Flipkart Affiliate API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.flipkart.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 67,
    "crawlDelay": null,
    "namedAgents": 6,
    "aiCrawlersNamed": [
     "Amazonbot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.flipkart.com/pages/terms",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 69,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "florida-sunbiz",
   "name": "Florida Sunbiz",
   "domain": "sunbiz.org",
   "category": "Business registries",
   "href": "/sites/florida-sunbiz/",
   "api": "public",
   "apiName": "Florida Division of Corporations data downloads",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.sunbiz.org/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://dos.fl.gov/privacy-policy/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "fotocasa",
   "name": "Fotocasa",
   "domain": "fotocasa.es",
   "category": "Real estate",
   "href": "/sites/fotocasa/",
   "api": "none",
   "apiName": "No public listing API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://fotocasa.es/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 295,
    "crawlDelay": null,
    "namedAgents": 40,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.fotocasa.es/aviso-legal",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "freelancer",
   "name": "Freelancer.com",
   "domain": "freelancer.com",
   "category": "Freelance marketplaces",
   "href": "/sites/freelancer/",
   "api": "public",
   "apiName": "Freelancer API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://freelancer.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 38,
    "crawlDelay": null,
    "namedAgents": 5,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.freelancer.com/about/terms",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "g2",
   "name": "G2",
   "domain": "g2.com",
   "category": "Software reviews",
   "href": "/sites/g2/",
   "api": "none",
   "apiName": "No public reviews API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://g2.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 91,
    "crawlDelay": null,
    "namedAgents": 45,
    "aiCrawlersNamed": [
     "GPTBot",
     "ClaudeBot",
     "CCBot",
     "Bytespider",
     "Amazonbot",
     "Applebot-Extended",
     "meta-externalagent",
     "Diffbot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://legal.g2.com/terms-of-use",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "getyourguide",
   "name": "GetYourGuide",
   "domain": "getyourguide.com",
   "category": "Travel",
   "href": "/sites/getyourguide/",
   "api": "partner",
   "apiName": "GetYourGuide Partner API",
   "apiChecked": "2026-11-12",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.getyourguide.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.getyourguide.com/c/general-terms-and-conditions/",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "github",
   "name": "GitHub",
   "domain": "github.com",
   "category": "Developer platforms",
   "href": "/sites/github/",
   "api": "public",
   "apiName": "GitHub REST API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://github.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 282,
    "crawlDelay": null,
    "namedAgents": 15,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-User",
     "Claude-SearchBot",
     "PerplexityBot",
     "Perplexity-User",
     "Bytespider"
    ]
   },
   "terms": {
    "answer": "not-read",
    "status": null,
    "url": "https://docs.github.com/en/site-policy/github-terms/github-terms-of-service",
    "checked": "2026-11-13",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": null
  },
  {
   "slug": "glassdoor",
   "name": "Glassdoor",
   "domain": "glassdoor.com",
   "category": "Jobs",
   "href": "/sites/glassdoor/",
   "api": "none",
   "apiName": "No public API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://glassdoor.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 142,
    "crawlDelay": null,
    "namedAgents": 25,
    "aiCrawlersNamed": [
     "GPTBot",
     "ClaudeBot",
     "Claude-Web",
     "anthropic-ai",
     "Google-Extended",
     "CCBot",
     "Bytespider",
     "Amazonbot",
     "Applebot-Extended",
     "FacebookBot",
     "cohere-ai",
     "Diffbot",
     "omgilibot",
     "YouBot",
     "img2dataset"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.glassdoor.com/about/terms/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 5,
    "topStars": 193,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "goodreads",
   "name": "Goodreads",
   "domain": "goodreads.com",
   "category": "Media and entertainment",
   "href": "/sites/goodreads/",
   "api": "retired",
   "apiName": "Goodreads public developer API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://goodreads.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 27,
    "crawlDelay": null,
    "namedAgents": 7,
    "aiCrawlersNamed": [
     "GPTBot",
     "CCBot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.goodreads.com/about/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 2,
    "topStars": 310,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "google-ads-transparency",
   "name": "Google Ads Transparency Center",
   "domain": "adstransparency.google.com",
   "category": "Ads and transparency",
   "href": "/sites/google-ads-transparency/",
   "api": "none",
   "apiName": "No public ad-library API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "no-answer",
    "status": 0,
    "url": "https://www.adstransparency.google.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://policies.google.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "google-flights",
   "name": "Google Flights",
   "domain": "google.com/travel/flights",
   "category": "Travel",
   "href": "/sites/google-flights/",
   "api": "partner",
   "apiName": "Google Flights Search partner integration",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.google.com/travel/flights/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://policies.google.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 4,
    "topStars": 2107,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "google-maps",
   "name": "Google Maps",
   "domain": "google.com/maps",
   "category": "Local and place data",
   "href": "/sites/google-maps/",
   "api": "public",
   "apiName": "Google Maps Platform Places API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "not-found",
    "status": 404,
    "url": "https://www.google.com/maps/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://cloud.google.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 3,
    "topStars": 7931,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "google-news",
   "name": "Google News",
   "domain": "news.google.com",
   "category": "News and publishing",
   "href": "/sites/google-news/",
   "api": "none",
   "apiName": "No official Google News API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://news.google.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "all",
    "disallowRules": 1,
    "crawlDelay": null,
    "namedAgents": 8,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "ClaudeBot",
     "Claude-Web",
     "anthropic-ai",
     "PerplexityBot",
     "CCBot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://policies.google.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 7,
    "topStars": 3400,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "google-patents",
   "name": "Google Patents",
   "domain": "patents.google.com",
   "category": "Research and reference",
   "href": "/sites/google-patents/",
   "api": "public",
   "apiName": "Google Patents Public Datasets on BigQuery",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://patents.google.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 1,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://policies.google.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 2,
    "topStars": 77,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "google-play",
   "name": "Google Play",
   "domain": "play.google.com",
   "category": "App stores",
   "href": "/sites/google-play/",
   "api": "limited",
   "apiName": "Google Play Developer API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://play.google.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 97,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://play.google.com/about/play-terms/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 5,
    "topStars": 3001,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "google-scholar",
   "name": "Google Scholar",
   "domain": "scholar.google.com",
   "category": "Research and reference",
   "href": "/sites/google-scholar/",
   "api": "none",
   "apiName": "No official Google Scholar API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://scholar.google.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 7,
    "crawlDelay": null,
    "namedAgents": 3,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://policies.google.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 7,
    "topStars": 2102,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "google-search",
   "name": "Google Search",
   "domain": "google.com",
   "category": "Search results",
   "href": "/sites/google-search/",
   "api": "limited",
   "apiName": "Programmable Search Engine JSON API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://google.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 170,
    "crawlDelay": null,
    "namedAgents": 4,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://policies.google.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 2,
   "openSource": {
    "repos": 4,
    "topStars": 650,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "google-shopping",
   "name": "Google Shopping",
   "domain": "shopping.google.com",
   "category": "Ecommerce",
   "href": "/sites/google-shopping/",
   "api": "limited",
   "apiName": "Google Merchant API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "no-answer",
    "status": 0,
    "url": "https://www.shopping.google.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://policies.google.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 2,
    "topStars": 33,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "google-trends",
   "name": "Google Trends",
   "domain": "trends.google.com",
   "category": "SEO and web analytics",
   "href": "/sites/google-trends/",
   "api": "limited",
   "apiName": "Google Trends API (alpha)",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://trends.google.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 2,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://policies.google.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 4,
    "topStars": 2915,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "google-workspace-marketplace",
   "name": "Google Workspace Marketplace",
   "domain": "workspace.google.com/marketplace",
   "category": "App stores",
   "href": "/sites/google-workspace-marketplace/",
   "api": "limited",
   "apiName": "Google Workspace Marketplace SDK",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "no-answer",
    "status": 0,
    "url": "https://www.workspace.google.com/marketplace/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://developers.google.com/workspace/marketplace/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "greenhouse",
   "name": "Greenhouse job boards",
   "domain": "boards.greenhouse.io",
   "category": "Jobs",
   "href": "/sites/greenhouse/",
   "api": "public",
   "apiName": "Greenhouse Job Board API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://boards.greenhouse.io/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 1,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.greenhouse.com/legal",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "gulf-talent",
   "name": "GulfTalent",
   "domain": "gulftalent.com",
   "category": "Jobs",
   "href": "/sites/gulf-talent/",
   "api": "none",
   "apiName": "No public jobs API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://gulftalent.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 3,
    "crawlDelay": null,
    "namedAgents": 57,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-Web",
     "anthropic-ai",
     "Google-Extended",
     "PerplexityBot",
     "CCBot",
     "Bytespider",
     "Amazonbot",
     "Applebot-Extended",
     "meta-externalagent",
     "cohere-ai",
     "Diffbot",
     "AI2Bot"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.gulftalent.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "hacker-news",
   "name": "Hacker News",
   "domain": "news.ycombinator.com",
   "category": "News and publishing",
   "href": "/sites/hacker-news/",
   "api": "public",
   "apiName": "Hacker News API (Firebase)",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://news.ycombinator.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 12,
    "crawlDelay": 30,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.ycombinator.com/legal",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 127,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "handshake",
   "name": "Handshake",
   "domain": "joinhandshake.com",
   "category": "Jobs",
   "href": "/sites/handshake/",
   "api": "limited",
   "apiName": "Handshake EDU API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://joinhandshake.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "none",
    "disallowRules": 0,
    "crawlDelay": null,
    "namedAgents": 20,
    "aiCrawlersNamed": [
     "Bytespider",
     "FacebookBot",
     "Diffbot",
     "omgilibot",
     "AI2Bot",
     "img2dataset"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://joinhandshake.com/legal/tos/",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "hepsiburada",
   "name": "Hepsiburada",
   "domain": "hepsiburada.com",
   "category": "Ecommerce",
   "href": "/sites/hepsiburada/",
   "api": "partner",
   "apiName": "Hepsiburada Marketplace service integration",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.hepsiburada.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.hepsiburada.com/kullanim-kosullari",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "himalayas",
   "name": "Himalayas",
   "domain": "himalayas.app",
   "category": "Jobs",
   "href": "/sites/himalayas/",
   "api": "public",
   "apiName": "Himalayas Remote Jobs API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://himalayas.app/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 25,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://himalayas.app/terms",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "hostelworld",
   "name": "Hostelworld",
   "domain": "hostelworld.com",
   "category": "Travel",
   "href": "/sites/hostelworld/",
   "api": "partner",
   "apiName": "Hostelworld Affiliate Programme (feed and Partner API)",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://hostelworld.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 38,
    "crawlDelay": null,
    "namedAgents": 2,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.hostelworld.com/legal/hostel-terms-and-conditions/",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "houzz",
   "name": "Houzz",
   "domain": "houzz.com",
   "category": "Local and place data",
   "href": "/sites/houzz/",
   "api": "partner",
   "apiName": "Houzz Pro API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://houzz.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 144,
    "crawlDelay": null,
    "namedAgents": 10,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-User",
     "Google-Extended",
     "PerplexityBot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.houzz.com/termsOfUse",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "idealista",
   "name": "Idealista",
   "domain": "idealista.com",
   "category": "Real estate",
   "href": "/sites/idealista/",
   "api": "partner",
   "apiName": "Idealista API for professionals",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://idealista.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 186,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.idealista.com/en/legal/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 4,
    "topStars": 187,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "ikea",
   "name": "IKEA",
   "domain": "ikea.com",
   "category": "Ecommerce",
   "href": "/sites/ikea/",
   "api": "none",
   "apiName": "No public product API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://ikea.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 92,
    "crawlDelay": null,
    "namedAgents": 4,
    "aiCrawlersNamed": [
     "Amazonbot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.ikea.com/us/en/customer-service/terms-conditions/",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "imdb",
   "name": "IMDb",
   "domain": "imdb.com",
   "category": "Media and entertainment",
   "href": "/sites/imdb/",
   "api": "limited",
   "apiName": "IMDb API (AWS Data Exchange) and IMDb non-commercial datasets",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://imdb.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "all",
    "disallowRules": 1,
    "crawlDelay": null,
    "namedAgents": 37,
    "aiCrawlersNamed": [
     "Amazonbot",
     "DuckAssistBot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.imdb.com/conditions",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 3,
    "topStars": 340,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "immowelt",
   "name": "Immowelt",
   "domain": "immowelt.de",
   "category": "Real estate",
   "href": "/sites/immowelt/",
   "api": "none",
   "apiName": "No public listing API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://immowelt.de/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 72,
    "crawlDelay": null,
    "namedAgents": 10,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-User",
     "Claude-SearchBot",
     "PerplexityBot"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.immowelt.de/agb",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "importyeti",
   "name": "ImportYeti",
   "domain": "importyeti.com",
   "category": "B2B and trade data",
   "href": "/sites/importyeti/",
   "api": "public",
   "apiName": "ImportYeti API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://importyeti.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 7,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.importyeti.com/policies/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "indeed",
   "name": "Indeed",
   "domain": "indeed.com",
   "category": "Jobs",
   "href": "/sites/indeed/",
   "api": "partner",
   "apiName": "Indeed partner APIs",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://indeed.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 169,
    "crawlDelay": null,
    "namedAgents": 64,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-User",
     "Claude-SearchBot",
     "anthropic-ai",
     "Google-Extended",
     "PerplexityBot",
     "Perplexity-User",
     "CCBot",
     "Bytespider",
     "Amazonbot",
     "Applebot-Extended",
     "meta-externalagent",
     "meta-externalfetcher",
     "FacebookBot",
     "Diffbot",
     "omgilibot",
     "YouBot",
     "DuckAssistBot",
     "AI2Bot",
     "img2dataset"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.indeed.com/legal",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 7,
    "topStars": 146,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "indiamart",
   "name": "IndiaMART",
   "domain": "indiamart.com",
   "category": "B2B and trade data",
   "href": "/sites/indiamart/",
   "api": "limited",
   "apiName": "IndiaMART Lead Manager CRM API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://indiamart.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 31,
    "crawlDelay": null,
    "namedAgents": 56,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Google-Extended",
     "Bytespider",
     "meta-externalagent",
     "FacebookBot"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.indiamart.com/terms-of-use.html",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "instagram",
   "name": "Instagram",
   "domain": "instagram.com",
   "category": "Social network",
   "href": "/sites/instagram/",
   "api": "limited",
   "apiName": "Instagram Platform API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.instagram.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 400,
    "url": "https://help.instagram.com/581066165581870",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 3,
   "openSource": {
    "repos": 2,
    "topStars": 3365,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "jobstreet",
   "name": "JobStreet",
   "domain": "jobstreet.com",
   "category": "Jobs",
   "href": "/sites/jobstreet/",
   "api": "limited",
   "apiName": "SEEK / JobStreet partner APIs",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "not-found",
    "status": 404,
    "url": "https://www.jobstreet.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://my.jobstreet.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 10,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "kayak",
   "name": "Kayak",
   "domain": "kayak.com",
   "category": "Travel",
   "href": "/sites/kayak/",
   "api": "partner",
   "apiName": "KAYAK Affiliate Network APIs",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://kayak.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 115,
    "crawlDelay": null,
    "namedAgents": 21,
    "aiCrawlersNamed": [
     "ChatGPT-User"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.kayak.com/terms-of-use",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "lever",
   "name": "Lever job boards",
   "domain": "jobs.lever.co",
   "category": "Jobs",
   "href": "/sites/lever/",
   "api": "public",
   "apiName": "Lever Postings API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://jobs.lever.co/robots.txt",
    "checked": "2026-11-13",
    "rule": "none",
    "disallowRules": 0,
    "crawlDelay": 1,
    "namedAgents": 32,
    "aiCrawlersNamed": [
     "GPTBot",
     "ClaudeBot",
     "Claude-Web",
     "anthropic-ai",
     "Google-Extended",
     "CCBot",
     "Bytespider",
     "Amazonbot",
     "Applebot-Extended",
     "meta-externalagent",
     "cohere-ai",
     "Diffbot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.lever.co/legal/terms-of-service",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "linkedin",
   "name": "LinkedIn",
   "domain": "linkedin.com",
   "category": "Professional network",
   "href": "/sites/linkedin/",
   "api": "partner",
   "apiName": "LinkedIn APIs (partner programs)",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.linkedin.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "all",
    "disallowRules": 1,
    "crawlDelay": null,
    "namedAgents": 77,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-User",
     "Claude-SearchBot",
     "Claude-Web",
     "anthropic-ai",
     "Google-Extended",
     "PerplexityBot",
     "Perplexity-User",
     "CCBot",
     "Bytespider",
     "meta-externalagent",
     "meta-externalfetcher",
     "cohere-ai",
     "Diffbot",
     "omgilibot",
     "DuckAssistBot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.linkedin.com/legal/user-agreement",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 3,
   "openSource": {
    "repos": 1,
    "topStars": 4674,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "linkedin-ad-library",
   "name": "LinkedIn Ad Library",
   "domain": "linkedin.com/ad-library",
   "category": "Ads and transparency",
   "href": "/sites/linkedin-ad-library/",
   "api": "none",
   "apiName": "No public ad-library API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.linkedin.com/ad-library/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.linkedin.com/legal/user-agreement",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "linkedin-jobs",
   "name": "LinkedIn Jobs",
   "domain": "linkedin.com/jobs",
   "category": "Jobs and hiring",
   "href": "/sites/linkedin-jobs/",
   "api": "partner",
   "apiName": "LinkedIn Talent APIs",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "not-found",
    "status": 404,
    "url": "https://www.linkedin.com/jobs/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.linkedin.com/legal/user-agreement",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 5,
    "topStars": 498,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "lowes",
   "name": "Lowe's",
   "domain": "lowes.com",
   "category": "Ecommerce",
   "href": "/sites/lowes/",
   "api": "none",
   "apiName": "No public product API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.lowes.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.lowes.com/l/about/terms-and-conditions-of-use",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "medium",
   "name": "Medium",
   "domain": "medium.com",
   "category": "News and publishing",
   "href": "/sites/medium/",
   "api": "limited",
   "apiName": "Medium API (unsupported) and Medium RSS feeds",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://medium.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 15,
    "crawlDelay": null,
    "namedAgents": 8,
    "aiCrawlersNamed": [
     "GPTBot",
     "ClaudeBot",
     "Bytespider",
     "Amazonbot",
     "Applebot-Extended",
     "meta-externalagent",
     "FacebookBot"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://policy.medium.com/medium-terms-of-service-9db0094a1e0f",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": null
  },
  {
   "slug": "meetup",
   "name": "Meetup",
   "domain": "meetup.com",
   "category": "Events",
   "href": "/sites/meetup/",
   "api": "limited",
   "apiName": "Meetup GraphQL API",
   "apiChecked": "2026-11-12",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://meetup.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 115,
    "crawlDelay": null,
    "namedAgents": 1,
    "aiCrawlersNamed": [
     "GPTBot"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://help.meetup.com/hc/en-us/articles/360027447252-Terms-of-Service",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "mercadolibre",
   "name": "MercadoLibre",
   "domain": "mercadolibre.com",
   "category": "Ecommerce",
   "href": "/sites/mercadolibre/",
   "api": "public",
   "apiName": "Mercado Libre Developers API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://mercadolibre.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 5,
    "crawlDelay": null,
    "namedAgents": 9,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "ClaudeBot",
     "Claude-User",
     "PerplexityBot",
     "Perplexity-User",
     "Amazonbot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.mercadolibre.com.ar/ayuda/terminos-y-condiciones_299",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 2,
    "topStars": 60,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "mercari-japan",
   "name": "Mercari Japan",
   "domain": "jp.mercari.com",
   "category": "Ecommerce",
   "href": "/sites/mercari-japan/",
   "api": "limited",
   "apiName": "Mercari Shops API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://jp.mercari.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 14,
    "crawlDelay": null,
    "namedAgents": 3,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://static.jp.mercari.com/tos",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "meta-ad-library",
   "name": "Meta Ad Library",
   "domain": "facebook.com/ads/library",
   "category": "Ads and transparency",
   "href": "/sites/meta-ad-library/",
   "api": "limited",
   "apiName": "Meta Ad Library API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.facebook.com/ads/library/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 400,
    "url": "https://www.facebook.com/policies_center/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 146,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "mlb",
   "name": "MLB.com",
   "domain": "mlb.com",
   "category": "Sports and gaming",
   "href": "/sites/mlb/",
   "api": "limited",
   "apiName": "MLB Stats API (statsapi.mlb.com)",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://mlb.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 76,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.mlb.com/official-information/terms-of-use",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "myntra",
   "name": "Myntra",
   "domain": "myntra.com",
   "category": "Ecommerce",
   "href": "/sites/myntra/",
   "api": "partner",
   "apiName": "Myntra Developer Centre seller APIs",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.myntra.com/robots.txt",
    "checked": "2026-11-13",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "no-answer",
    "status": 0,
    "url": "https://www.myntra.com/termsofuse",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "naukri",
   "name": "Naukri",
   "domain": "naukri.com",
   "category": "Jobs",
   "href": "/sites/naukri/",
   "api": "none",
   "apiName": "No public jobs API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://naukri.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 27,
    "crawlDelay": null,
    "namedAgents": 12,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-User",
     "Claude-SearchBot",
     "PerplexityBot",
     "Perplexity-User",
     "CCBot",
     "Bytespider",
     "meta-externalagent",
     "DuckAssistBot"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.naukri.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 2,
    "topStars": 17,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "naver",
   "name": "Naver",
   "domain": "naver.com",
   "category": "Search and local data",
   "href": "/sites/naver/",
   "api": "limited",
   "apiName": "Naver Open API and Maps API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://naver.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "all",
    "disallowRules": 1,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://policy.naver.com/policy/service.html",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "nike",
   "name": "Nike",
   "domain": "nike.com",
   "category": "Ecommerce",
   "href": "/sites/nike/",
   "api": "none",
   "apiName": "No public product API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://nike.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 18,
    "crawlDelay": null,
    "namedAgents": 5,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://agreementservice.svs.nike.com/us/en_us/rest/agreement?agreementType=termsOfUse&uxId=com.nike.commerce.nikedotcom.web&country=US&language=en&requestType=redirect",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "noon",
   "name": "Noon",
   "domain": "noon.com",
   "category": "Ecommerce",
   "href": "/sites/noon/",
   "api": "none",
   "apiName": "No public noon data API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "no-answer",
    "status": 0,
    "url": "https://www.noon.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "no-answer",
    "status": 0,
    "url": "https://www.noon.com/uae-en/terms-of-use/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": null
  },
  {
   "slug": "op-gg",
   "name": "OP.GG",
   "domain": "op.gg",
   "category": "Sports and gaming",
   "href": "/sites/op-gg/",
   "api": "limited",
   "apiName": "OP.GG MCP Server",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://op.gg/robots.txt",
    "checked": "2026-11-13",
    "rule": "none",
    "disallowRules": 0,
    "crawlDelay": null,
    "namedAgents": 3,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://op.gg/lol/policies/agreement",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 15,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "ozon",
   "name": "Ozon",
   "domain": "ozon.ru",
   "category": "Ecommerce",
   "href": "/sites/ozon/",
   "api": "partner",
   "apiName": "Ozon Seller API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 307,
    "url": "https://www.ozon.ru/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 307,
    "url": "https://docs.ozon.ru/legal/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 2,
    "topStars": 130,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "pexels",
   "name": "Pexels",
   "domain": "pexels.com",
   "category": "Images and media",
   "href": "/sites/pexels/",
   "api": "public",
   "apiName": "Pexels API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://pexels.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 110,
    "crawlDelay": null,
    "namedAgents": 10,
    "aiCrawlersNamed": [
     "GPTBot",
     "ClaudeBot",
     "CCBot",
     "Bytespider",
     "Applebot-Extended"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.pexels.com/terms-of-service/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 3,
    "topStars": 61,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "pinterest",
   "name": "Pinterest",
   "domain": "pinterest.com",
   "category": "Social network",
   "href": "/sites/pinterest/",
   "api": "public",
   "apiName": "Pinterest API v5",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://pinterest.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "all",
    "disallowRules": 1,
    "crawlDelay": null,
    "namedAgents": 313,
    "aiCrawlersNamed": [
     "FacebookBot",
     "DuckAssistBot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.pinterest.com/terms/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 456,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "product-hunt",
   "name": "Product Hunt",
   "domain": "producthunt.com",
   "category": "Startups and products",
   "href": "/sites/product-hunt/",
   "api": "public",
   "apiName": "Product Hunt API v2",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://producthunt.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 14,
    "crawlDelay": null,
    "namedAgents": 2,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.producthunt.com/legal",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 2,
    "topStars": 40,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "propertyfinder",
   "name": "Property Finder",
   "domain": "propertyfinder.ae",
   "category": "Real estate",
   "href": "/sites/propertyfinder/",
   "api": "none",
   "apiName": "No public listing API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://propertyfinder.ae/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 57,
    "crawlDelay": null,
    "namedAgents": 22,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.propertyfinder.com/ae/terms-and-conditions/",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "quintoandar",
   "name": "QuintoAndar",
   "domain": "quintoandar.com.br",
   "category": "Real estate",
   "href": "/sites/quintoandar/",
   "api": "none",
   "apiName": "No official QuintoAndar data API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://quintoandar.com.br/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 100,
    "crawlDelay": null,
    "namedAgents": 1,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.quintoandar.com.br/termos",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 12,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "quora",
   "name": "Quora",
   "domain": "quora.com",
   "category": "Q&A and social",
   "href": "/sites/quora/",
   "api": "none",
   "apiName": "No public content API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://quora.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "all",
    "disallowRules": 63,
    "crawlDelay": null,
    "namedAgents": 28,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-User",
     "Claude-SearchBot",
     "Google-Extended",
     "PerplexityBot",
     "Bytespider",
     "Amazonbot",
     "Applebot-Extended",
     "Diffbot"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.quora.com/about/tos",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 3,
    "topStars": 126,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "realtor-com",
   "name": "Realtor.com",
   "domain": "realtor.com",
   "category": "Real estate",
   "href": "/sites/realtor-com/",
   "api": "partner",
   "apiName": "Realtor.com data programs",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://realtor.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 319,
    "crawlDelay": null,
    "namedAgents": 15,
    "aiCrawlersNamed": [
     "meta-externalagent",
     "meta-externalfetcher",
     "FacebookBot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.realtor.com/terms-of-use",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "reddit",
   "name": "Reddit",
   "domain": "reddit.com",
   "category": "Social network",
   "href": "/sites/reddit/",
   "api": "limited",
   "apiName": "Reddit Data API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://reddit.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "all",
    "disallowRules": 1,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.redditinc.com/policies/user-agreement",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 2,
   "openSource": {
    "repos": 2,
    "topStars": 2629,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "redfin",
   "name": "Redfin",
   "domain": "redfin.com",
   "category": "Real estate",
   "href": "/sites/redfin/",
   "api": "limited",
   "apiName": "Redfin Data Center (aggregates only)",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.redfin.com/robots.txt",
    "checked": "2026-11-13",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.redfin.com/about/terms-of-use",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 4,
    "topStars": 100,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "reed",
   "name": "Reed",
   "domain": "reed.co.uk",
   "category": "Jobs",
   "href": "/sites/reed/",
   "api": "public",
   "apiName": "Reed Jobseeker API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://reed.co.uk/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 30,
    "crawlDelay": null,
    "namedAgents": 11,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "PerplexityBot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.reed.co.uk/policies",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": null
  },
  {
   "slug": "sephora",
   "name": "Sephora",
   "domain": "sephora.com",
   "category": "Ecommerce",
   "href": "/sites/sephora/",
   "api": "none",
   "apiName": "No public product API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.sephora.com/robots.txt",
    "checked": "2026-11-13",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.sephora.com/beauty/terms-of-use",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 3,
    "topStars": 33,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "shein",
   "name": "Shein",
   "domain": "shein.com",
   "category": "Ecommerce",
   "href": "/sites/shein/",
   "api": "partner",
   "apiName": "SHEIN Open Platform",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://shein.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 7,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.shein.com/Terms-and-Conditions-a-399.html",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 4,
    "topStars": 505,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "shopee",
   "name": "Shopee",
   "domain": "shopee.com",
   "category": "Ecommerce",
   "href": "/sites/shopee/",
   "api": "partner",
   "apiName": "Shopee Open Platform",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.shopee.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://shopee.sg/docs/3612",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 2,
    "topStars": 333,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "shopify-stores",
   "name": "Shopify stores",
   "domain": "shopify.com",
   "category": "Ecommerce",
   "href": "/sites/shopify-stores/",
   "api": "limited",
   "apiName": "Shopify Storefront API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://shopify.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 2,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.shopify.com/legal/api-terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "similarweb",
   "name": "Similarweb",
   "domain": "similarweb.com",
   "category": "SEO and web analytics",
   "href": "/sites/similarweb/",
   "api": "limited",
   "apiName": "Similarweb API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://similarweb.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 16,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.similarweb.com/corp/legal/terms/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 2,
    "topStars": 21,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "simplyhired",
   "name": "SimplyHired",
   "domain": "simplyhired.com",
   "category": "Jobs",
   "href": "/sites/simplyhired/",
   "api": "none",
   "apiName": "No public jobs API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.simplyhired.com/robots.txt",
    "checked": "2026-11-13",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.simplyhired.com/about/terms-of-service",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "skool",
   "name": "Skool",
   "domain": "skool.com",
   "category": "Social network",
   "href": "/sites/skool/",
   "api": "none",
   "apiName": "No public Skool API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://skool.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 1,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.skool.com/legal",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 4,
    "topStars": 443,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "snapchat",
   "name": "Snapchat",
   "domain": "snapchat.com",
   "category": "Social network",
   "href": "/sites/snapchat/",
   "api": "limited",
   "apiName": "Snapchat APIs",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://snapchat.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 4,
    "crawlDelay": null,
    "namedAgents": 50,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-User",
     "Claude-SearchBot",
     "Claude-Web",
     "anthropic-ai",
     "Google-Extended",
     "PerplexityBot",
     "Perplexity-User",
     "CCBot",
     "Bytespider",
     "Amazonbot",
     "Applebot-Extended",
     "meta-externalagent",
     "meta-externalfetcher",
     "FacebookBot",
     "cohere-ai",
     "Diffbot",
     "omgilibot",
     "YouBot",
     "DuckAssistBot",
     "AI2Bot",
     "img2dataset"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.snap.com/en-US/terms",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 3,
    "topStars": 727,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "sofascore",
   "name": "SofaScore",
   "domain": "sofascore.com",
   "category": "Sports and gaming",
   "href": "/sites/sofascore/",
   "api": "none",
   "apiName": "No public SofaScore data API (free widgets only)",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.sofascore.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.sofascore.com/terms-and-conditions",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 2,
    "topStars": 77,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "soundcloud",
   "name": "SoundCloud",
   "domain": "soundcloud.com",
   "category": "Media and entertainment",
   "href": "/sites/soundcloud/",
   "api": "limited",
   "apiName": "SoundCloud API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://soundcloud.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 7,
    "crawlDelay": null,
    "namedAgents": 19,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-User",
     "Claude-SearchBot",
     "Claude-Web",
     "anthropic-ai",
     "Google-Extended",
     "PerplexityBot",
     "CCBot",
     "Bytespider",
     "Amazonbot",
     "Applebot-Extended",
     "meta-externalagent",
     "cohere-ai",
     "omgilibot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://soundcloud.com/terms-of-use",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 332,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "spotify",
   "name": "Spotify",
   "domain": "open.spotify.com",
   "category": "Media and entertainment",
   "href": "/sites/spotify/",
   "api": "limited",
   "apiName": "Spotify Web API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://open.spotify.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 3,
    "crawlDelay": null,
    "namedAgents": 26,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-User",
     "Claude-SearchBot",
     "Claude-Web",
     "anthropic-ai",
     "Google-Extended",
     "PerplexityBot",
     "Perplexity-User",
     "CCBot",
     "Bytespider",
     "Amazonbot",
     "Applebot-Extended",
     "meta-externalagent",
     "meta-externalfetcher",
     "FacebookBot",
     "cohere-ai"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.spotify.com/us/legal/end-user-agreement/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 3,
    "topStars": 26791,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "steam",
   "name": "Steam",
   "domain": "store.steampowered.com",
   "category": "Sports and gaming",
   "href": "/sites/steam/",
   "api": "public",
   "apiName": "Steam Web API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://store.steampowered.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 9,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://store.steampowered.com/subscriber_agreement/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "stocktwits",
   "name": "StockTwits",
   "domain": "stocktwits.com",
   "category": "Finance and markets",
   "href": "/sites/stocktwits/",
   "api": "partner",
   "apiName": "Stocktwits API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.stocktwits.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://stocktwits.com/about/legal/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "stockx",
   "name": "StockX",
   "domain": "stockx.com",
   "category": "Ecommerce",
   "href": "/sites/stockx/",
   "api": "limited",
   "apiName": "StockX API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://stockx.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 173,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://stockx.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 3,
    "topStars": 20,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "substack",
   "name": "Substack",
   "domain": "substack.com",
   "category": "News and publishing",
   "href": "/sites/substack/",
   "api": "limited",
   "apiName": "Substack Developer API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://substack.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 14,
    "crawlDelay": null,
    "namedAgents": 3,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://substack.com/tos",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 2,
    "topStars": 546,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "target",
   "name": "Target",
   "domain": "target.com",
   "category": "Ecommerce",
   "href": "/sites/target/",
   "api": "none",
   "apiName": "No public product API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://target.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 114,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.target.com/c/terms-conditions/-/N-4sr7l",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": null
  },
  {
   "slug": "home-depot",
   "name": "The Home Depot",
   "domain": "homedepot.com",
   "category": "Ecommerce",
   "href": "/sites/home-depot/",
   "api": "none",
   "apiName": "No public product API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.homedepot.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.homedepot.com/c/Terms_of_Use",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 11,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "threads",
   "name": "Threads",
   "domain": "threads.net",
   "category": "Social network",
   "href": "/sites/threads/",
   "api": "limited",
   "apiName": "Threads API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.threads.net/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 400,
    "url": "https://help.instagram.com/769983657850450",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": null
  },
  {
   "slug": "ticketmaster",
   "name": "Ticketmaster",
   "domain": "ticketmaster.com",
   "category": "Events",
   "href": "/sites/ticketmaster/",
   "api": "public",
   "apiName": "Ticketmaster Discovery API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://ticketmaster.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 70,
    "crawlDelay": null,
    "namedAgents": 3,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://help.ticketmaster.com/hc/en-us/articles/10468830739345-Terms-of-Use",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 16,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "tiktok",
   "name": "TikTok",
   "domain": "tiktok.com",
   "category": "Social network",
   "href": "/sites/tiktok/",
   "api": "limited",
   "apiName": "TikTok APIs",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://tiktok.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 16,
    "crawlDelay": null,
    "namedAgents": 26,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-User",
     "Claude-SearchBot",
     "anthropic-ai",
     "Google-Extended",
     "PerplexityBot",
     "Perplexity-User",
     "CCBot",
     "Bytespider",
     "Applebot-Extended",
     "meta-externalagent",
     "MistralAI-User",
     "DuckAssistBot",
     "AI2Bot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.tiktok.com/legal/page/us/terms-of-service/en",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 4,
    "topStars": 21357,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "tiktok-ads-library",
   "name": "TikTok Ads Library",
   "domain": "library.tiktok.com",
   "category": "Ads and transparency",
   "href": "/sites/tiktok-ads-library/",
   "api": "limited",
   "apiName": "TikTok Commercial Content Library / Research API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://library.tiktok.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "all",
    "disallowRules": 4,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.tiktok.com/legal/page/us/terms-of-service/en",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "tiktok-shop",
   "name": "TikTok Shop",
   "domain": "shop.tiktok.com",
   "category": "Ecommerce",
   "href": "/sites/tiktok-shop/",
   "api": "partner",
   "apiName": "TikTok Shop Partner APIs",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "no-answer",
    "status": 0,
    "url": "https://www.shop.tiktok.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.tiktok.com/legal/page/us/terms-of-service/en",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 3,
    "topStars": 19,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "tokopedia",
   "name": "Tokopedia",
   "domain": "tokopedia.com",
   "category": "Ecommerce",
   "href": "/sites/tokopedia/",
   "api": "partner",
   "apiName": "Tokopedia & Shop APIs (TikTok Shop Partner Center)",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://tokopedia.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 110,
    "crawlDelay": null,
    "namedAgents": 1,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "no-answer",
    "status": 0,
    "url": "https://www.tokopedia.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 3,
    "topStars": 18,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "totaljobs",
   "name": "Totaljobs",
   "domain": "totaljobs.com",
   "category": "Jobs",
   "href": "/sites/totaljobs/",
   "api": "none",
   "apiName": "No public jobs API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://totaljobs.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 139,
    "crawlDelay": null,
    "namedAgents": 5,
    "aiCrawlersNamed": [
     "Amazonbot"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.totaljobs.com/termsandconditions",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "trading-economics",
   "name": "Trading Economics",
   "domain": "tradingeconomics.com",
   "category": "Finance and markets",
   "href": "/sites/trading-economics/",
   "api": "public",
   "apiName": "Trading Economics API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://tradingeconomics.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "none",
    "disallowRules": 0,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://tradingeconomics.com/terms.aspx",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "transfermarkt",
   "name": "Transfermarkt",
   "domain": "transfermarkt.com",
   "category": "Sports and gaming",
   "href": "/sites/transfermarkt/",
   "api": "none",
   "apiName": "No public Transfermarkt API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://transfermarkt.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "none",
    "disallowRules": 0,
    "crawlDelay": null,
    "namedAgents": 3,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 405,
    "url": "https://www.transfermarkt.com/intern/anb",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 3,
    "topStars": 527,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "trendyol",
   "name": "Trendyol",
   "domain": "trendyol.com",
   "category": "Ecommerce",
   "href": "/sites/trendyol/",
   "api": "partner",
   "apiName": "Trendyol Marketplace Integration API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://trendyol.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 567,
    "crawlDelay": null,
    "namedAgents": 8,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.trendyol.com/kullanim_kosullari",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "trip-com",
   "name": "Trip.com",
   "domain": "trip.com",
   "category": "Travel",
   "href": "/sites/trip-com/",
   "api": "partner",
   "apiName": "Trip.com Affiliate Programme and Connectivity platform",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://trip.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 88,
    "crawlDelay": null,
    "namedAgents": 4,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 432,
    "url": "https://www.trip.com/contents/service-guideline/terms.html",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "tripadvisor",
   "name": "Tripadvisor",
   "domain": "tripadvisor.com",
   "category": "Travel",
   "href": "/sites/tripadvisor/",
   "api": "partner",
   "apiName": "Tripadvisor Content API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://tripadvisor.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 606,
    "crawlDelay": null,
    "namedAgents": 24,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Google-Extended",
     "PerplexityBot",
     "CCBot",
     "Bytespider",
     "Amazonbot",
     "Applebot-Extended",
     "meta-externalagent",
     "cohere-ai",
     "YouBot",
     "img2dataset"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.tripadvisor.com/pages/terms.html",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 6,
    "topStars": 214,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "trulia",
   "name": "Trulia",
   "domain": "trulia.com",
   "category": "Real estate",
   "href": "/sites/trulia/",
   "api": "none",
   "apiName": "No official Trulia data API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://trulia.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 138,
    "crawlDelay": null,
    "namedAgents": 4,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.trulia.com/info/terms/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 20,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "trustpilot",
   "name": "Trustpilot",
   "domain": "trustpilot.com",
   "category": "Reviews",
   "href": "/sites/trustpilot/",
   "api": "limited",
   "apiName": "Trustpilot business APIs",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://trustpilot.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "all",
    "disallowRules": 1,
    "crawlDelay": null,
    "namedAgents": 44,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-User",
     "Claude-SearchBot",
     "anthropic-ai",
     "Google-Extended",
     "PerplexityBot",
     "CCBot",
     "Bytespider",
     "Applebot-Extended",
     "meta-externalagent",
     "meta-externalfetcher",
     "FacebookBot",
     "cohere-ai",
     "Diffbot",
     "omgilibot",
     "YouBot"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.trustpilot.com/legal/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 2,
    "topStars": 29,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "trustradius",
   "name": "TrustRadius",
   "domain": "trustradius.com",
   "category": "Software reviews",
   "href": "/sites/trustradius/",
   "api": "none",
   "apiName": "No public TrustRadius data API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.trustradius.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.trustradius.com/static/terms-of-use",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "twitch",
   "name": "Twitch",
   "domain": "twitch.tv",
   "category": "Video",
   "href": "/sites/twitch/",
   "api": "public",
   "apiName": "Twitch API (Helix)",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://twitch.tv/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 68,
    "crawlDelay": null,
    "namedAgents": 4,
    "aiCrawlersNamed": [
     "Amazonbot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://legal.twitch.com/legal/terms-of-service/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 4,
    "topStars": 4017,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "uber-eats",
   "name": "Uber Eats",
   "domain": "ubereats.com",
   "category": "Food and drink",
   "href": "/sites/uber-eats/",
   "api": "partner",
   "apiName": "Uber Eats Marketplace APIs",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://ubereats.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 4,
    "crawlDelay": null,
    "namedAgents": 1,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.uber.com/legal/en/document/?name=general-terms-of-use&country=united-states&lang=en",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 2,
    "topStars": 33,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "ubersuggest",
   "name": "Ubersuggest",
   "domain": "ubersuggest.com",
   "category": "SEO and web analytics",
   "href": "/sites/ubersuggest/",
   "api": "limited",
   "apiName": "Ubersuggest MCP Server",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://ubersuggest.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 280,
    "crawlDelay": null,
    "namedAgents": 10,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-Web",
     "anthropic-ai",
     "Google-Extended",
     "PerplexityBot",
     "CCBot",
     "Applebot-Extended"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://neilpatel.com/terms/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "udemy",
   "name": "Udemy",
   "domain": "udemy.com",
   "category": "Education",
   "href": "/sites/udemy/",
   "api": "retired",
   "apiName": "Udemy Affiliate API v2.0",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.udemy.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.udemy.com/terms/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 2096,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "unsplash",
   "name": "Unsplash",
   "domain": "unsplash.com",
   "category": "Images and media",
   "href": "/sites/unsplash/",
   "api": "public",
   "apiName": "Unsplash API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://unsplash.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 168,
    "crawlDelay": null,
    "namedAgents": 21,
    "aiCrawlersNamed": [
     "ClaudeBot",
     "Claude-SearchBot",
     "Bytespider"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 401,
    "url": "https://unsplash.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "vimeo",
   "name": "Vimeo",
   "domain": "vimeo.com",
   "category": "Video",
   "href": "/sites/vimeo/",
   "api": "public",
   "apiName": "Vimeo API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://vimeo.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 330,
    "crawlDelay": 2,
    "namedAgents": 25,
    "aiCrawlersNamed": [
     "GPTBot",
     "ClaudeBot",
     "Claude-Web",
     "anthropic-ai",
     "Google-Extended",
     "CCBot",
     "Bytespider",
     "FacebookBot",
     "Diffbot",
     "omgilibot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://vimeo.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 3,
    "topStars": 442,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "vinted",
   "name": "Vinted",
   "domain": "vinted.com",
   "category": "Ecommerce",
   "href": "/sites/vinted/",
   "api": "limited",
   "apiName": "Vinted Pro Integrations API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://vinted.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 6,
    "crawlDelay": null,
    "namedAgents": 6,
    "aiCrawlersNamed": [
     "GPTBot",
     "ClaudeBot",
     "Google-Extended",
     "CCBot",
     "Amazonbot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.vinted.com/terms_and_conditions",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 4,
    "topStars": 347,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "vivareal",
   "name": "VivaReal",
   "domain": "vivareal.com.br",
   "category": "Real estate",
   "href": "/sites/vivareal/",
   "api": "none",
   "apiName": "No public listing API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.vivareal.com.br/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.vivareal.com.br/termos/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 3,
    "topStars": 18,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "vivino",
   "name": "Vivino",
   "domain": "vivino.com",
   "category": "Food and drink",
   "href": "/sites/vivino/",
   "api": "none",
   "apiName": "No public Vivino API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://vivino.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 35,
    "crawlDelay": null,
    "namedAgents": 93,
    "aiCrawlersNamed": [
     "ClaudeBot",
     "CCBot",
     "Bytespider",
     "Applebot-Extended",
     "Diffbot",
     "omgilibot",
     "AI2Bot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.vivino.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "vrbo",
   "name": "Vrbo",
   "domain": "vrbo.com",
   "category": "Travel",
   "href": "/sites/vrbo/",
   "api": "partner",
   "apiName": "Expedia Group partner APIs",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://vrbo.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 52,
    "crawlDelay": null,
    "namedAgents": 14,
    "aiCrawlersNamed": [
     "ChatGPT-User",
     "OAI-SearchBot",
     "Claude-User",
     "Claude-SearchBot",
     "PerplexityBot",
     "Perplexity-User",
     "meta-externalagent"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 429,
    "url": "https://www.vrbo.com/legal/terms-and-conditions",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "walk-score",
   "name": "Walk Score",
   "domain": "walkscore.com",
   "category": "Local and place data",
   "href": "/sites/walk-score/",
   "api": "public",
   "apiName": "Walk Score API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://walkscore.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 19,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.walkscore.com/terms-of-use.shtml",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "wallapop",
   "name": "Wallapop",
   "domain": "wallapop.com",
   "category": "Ecommerce",
   "href": "/sites/wallapop/",
   "api": "limited",
   "apiName": "Wallapop Connect API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.wallapop.com/robots.txt",
    "checked": "2026-11-13",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://about.wallapop.com/condiciones-de-uso",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 11,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "walmart",
   "name": "Walmart",
   "domain": "walmart.com",
   "category": "Ecommerce",
   "href": "/sites/walmart/",
   "api": "partner",
   "apiName": "Walmart Marketplace and Affiliate APIs",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://walmart.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 57,
    "crawlDelay": null,
    "namedAgents": 3,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://corporate.walmart.com/policies",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 3,
    "topStars": 49,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "wayback-machine",
   "name": "Wayback Machine",
   "domain": "web.archive.org",
   "category": "Research and reference",
   "href": "/sites/wayback-machine/",
   "api": "public",
   "apiName": "Wayback Machine APIs (Availability, CDX Server, Memento)",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "no-answer",
    "status": 0,
    "url": "https://www.web.archive.org/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://archive.org/about/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 7,
    "topStars": 5968,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "we-work-remotely",
   "name": "We Work Remotely",
   "domain": "weworkremotely.com",
   "category": "Jobs",
   "href": "/sites/we-work-remotely/",
   "api": "public",
   "apiName": "We Work Remotely public RSS feeds and Jobs API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://weworkremotely.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 7,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://weworkremotely.com/terms-and-conditions",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "welcome-to-the-jungle",
   "name": "Welcome to the Jungle",
   "domain": "welcometothejungle.com",
   "category": "Jobs",
   "href": "/sites/welcome-to-the-jungle/",
   "api": "limited",
   "apiName": "Welcome to the Jungle Solutions API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://welcometothejungle.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 5,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.welcometothejungle.com/en/pages/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "wellfound",
   "name": "Wellfound",
   "domain": "wellfound.com",
   "category": "Jobs",
   "href": "/sites/wellfound/",
   "api": "none",
   "apiName": "No public Wellfound API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://wellfound.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 31,
    "crawlDelay": null,
    "namedAgents": 0,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "not-read",
    "status": null,
    "url": "https://wellfound.com/terms",
    "checked": "2026-11-13",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "wikipedia",
   "name": "Wikipedia",
   "domain": "wikipedia.org",
   "category": "Research and reference",
   "href": "/sites/wikipedia/",
   "api": "public",
   "apiName": "MediaWiki Action API and Wikimedia Enterprise",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://wikipedia.org/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 428,
    "crawlDelay": null,
    "namedAgents": 33,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://foundation.wikimedia.org/wiki/Policy:Terms_of_Use",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 2,
    "topStars": 347,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "wildberries",
   "name": "Wildberries",
   "domain": "wildberries.ru",
   "category": "Ecommerce",
   "href": "/sites/wildberries/",
   "api": "limited",
   "apiName": "WB API (Wildberries Seller API)",
   "apiChecked": "2026-11-12",
   "robots": {
    "answer": "refused",
    "status": 498,
    "url": "https://www.wildberries.ru/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://legal.wildberries.ru/consumers-offer/country/ru/lang/ru/",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 35,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "willhaben",
   "name": "Willhaben",
   "domain": "willhaben.at",
   "category": "Real estate and marketplace",
   "href": "/sites/willhaben/",
   "api": "none",
   "apiName": "No public listing API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://willhaben.at/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 97,
    "crawlDelay": null,
    "namedAgents": 4,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.willhaben.at/iad/impressum",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "workday",
   "name": "Workday career sites",
   "domain": "myworkdayjobs.com",
   "category": "Jobs",
   "href": "/sites/workday/",
   "api": "limited",
   "apiName": "Workday Web Services (customer tenant APIs)",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "no-answer",
    "status": 0,
    "url": "https://www.myworkdayjobs.com/robots.txt",
    "checked": "2026-11-12",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.workday.com/en-us/legal/site-terms.html",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "x",
   "name": "X (Twitter)",
   "domain": "x.com",
   "category": "Social network",
   "href": "/sites/x/",
   "api": "public",
   "apiName": "X API v2",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://x.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "all",
    "disallowRules": 2,
    "crawlDelay": 1,
    "namedAgents": 11,
    "aiCrawlersNamed": [
     "Google-Extended",
     "meta-externalagent",
     "meta-externalfetcher",
     "FacebookBot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://x.com/en/tos",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 1,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "y-combinator",
   "name": "Y Combinator",
   "domain": "ycombinator.com",
   "category": "Startups and products",
   "href": "/sites/y-combinator/",
   "api": "none",
   "apiName": "No documented YC directory API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://ycombinator.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 4,
    "crawlDelay": null,
    "namedAgents": 3,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.ycombinator.com/legal",
    "checked": "2026-11-12",
    "clauseQuoted": true
   },
   "courtCases": 0,
   "openSource": {
    "repos": 2,
    "topStars": 37,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "yahoo-finance",
   "name": "Yahoo Finance",
   "domain": "finance.yahoo.com",
   "category": "Finance and markets",
   "href": "/sites/yahoo-finance/",
   "api": "none",
   "apiName": "No official Yahoo Finance data API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://finance.yahoo.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 32,
    "crawlDelay": null,
    "namedAgents": 45,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "ClaudeBot",
     "Claude-Web",
     "anthropic-ai",
     "Google-Extended",
     "PerplexityBot",
     "CCBot",
     "Bytespider",
     "Amazonbot",
     "FacebookBot",
     "cohere-ai",
     "Diffbot",
     "omgilibot",
     "YouBot",
     "img2dataset"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://legal.yahoo.com/us/en/yahoo/terms/otos/index.html",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 2,
    "topStars": 500,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "yahoo-news",
   "name": "Yahoo News",
   "domain": "news.yahoo.com",
   "category": "News and publishing",
   "href": "/sites/yahoo-news/",
   "api": "none",
   "apiName": "No official Yahoo News API",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://news.yahoo.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 21,
    "crawlDelay": null,
    "namedAgents": 45,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-SearchBot",
     "Claude-Web",
     "anthropic-ai",
     "Google-Extended",
     "PerplexityBot",
     "CCBot",
     "Bytespider",
     "FacebookBot",
     "cohere-ai",
     "Diffbot",
     "omgilibot",
     "YouBot",
     "img2dataset"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://legal.yahoo.com/us/en/yahoo/terms/otos/index.html",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "yahoo-auctions-japan",
   "name": "Yahoo! Auctions Japan",
   "domain": "auctions.yahoo.co.jp",
   "category": "Ecommerce",
   "href": "/sites/yahoo-auctions-japan/",
   "api": "retired",
   "apiName": "Yahoo! Auctions Web API (Yahoo! Developer Network)",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "no-answer",
    "status": 0,
    "url": "https://www.auctions.yahoo.co.jp/robots.txt",
    "checked": "2026-11-13",
    "rule": null,
    "disallowRules": null,
    "crawlDelay": null,
    "namedAgents": null,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.lycorp.co.jp/ja/company/terms/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "yandex",
   "name": "Yandex",
   "domain": "yandex.com",
   "category": "Search results",
   "href": "/sites/yandex/",
   "api": "limited",
   "apiName": "Yandex Search API and Maps API",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://yandex.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 221,
    "crawlDelay": null,
    "namedAgents": 3,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://yandex.com/legal/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 1,
    "topStars": 1023,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "yelp",
   "name": "Yelp",
   "domain": "yelp.com",
   "category": "Reviews",
   "href": "/sites/yelp/",
   "api": "public",
   "apiName": "Yelp Places API (formerly Yelp Fusion)",
   "apiChecked": "2026-11-11",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://yelp.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "all",
    "disallowRules": 1,
    "crawlDelay": null,
    "namedAgents": 30,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-User",
     "Claude-SearchBot",
     "Claude-Web",
     "anthropic-ai",
     "Google-Extended",
     "PerplexityBot",
     "Perplexity-User",
     "CCBot",
     "meta-externalagent",
     "meta-externalfetcher"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://terms.yelp.com/tos/en_us/20200101_en_us/",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "youtube",
   "name": "YouTube",
   "domain": "youtube.com",
   "category": "Video",
   "href": "/sites/youtube/",
   "api": "public",
   "apiName": "YouTube Data API v3",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://youtube.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 21,
    "crawlDelay": null,
    "namedAgents": 1,
    "aiCrawlersNamed": []
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.youtube.com/t/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 4,
    "topStars": 141544,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "zillow",
   "name": "Zillow",
   "domain": "zillow.com",
   "category": "Real estate",
   "href": "/sites/zillow/",
   "api": "partner",
   "apiName": "Zillow data access (partner / Bridge)",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://zillow.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "some",
    "disallowRules": 215,
    "crawlDelay": null,
    "namedAgents": 21,
    "aiCrawlersNamed": [
     "ClaudeBot",
     "Claude-User",
     "Claude-SearchBot",
     "Amazonbot",
     "Applebot-Extended",
     "meta-externalagent",
     "YouBot",
     "AI2Bot"
    ]
   },
   "terms": {
    "answer": "answered",
    "status": 200,
    "url": "https://www.zillow.com/corp/Terms.htm",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 4,
    "topStars": 170,
    "checked": "2026-11-12"
   }
  },
  {
   "slug": "ziprecruiter",
   "name": "ZipRecruiter",
   "domain": "ziprecruiter.com",
   "category": "Jobs",
   "href": "/sites/ziprecruiter/",
   "api": "partner",
   "apiName": "ZipRecruiter partner APIs",
   "apiChecked": "2026-11-09",
   "robots": {
    "answer": "answered",
    "status": 200,
    "url": "https://ziprecruiter.com/robots.txt",
    "checked": "2026-11-13",
    "rule": "all",
    "disallowRules": 1,
    "crawlDelay": null,
    "namedAgents": 33,
    "aiCrawlersNamed": [
     "GPTBot",
     "ChatGPT-User",
     "OAI-SearchBot",
     "ClaudeBot",
     "Claude-User",
     "Claude-SearchBot",
     "Google-Extended",
     "PerplexityBot",
     "CCBot",
     "Applebot-Extended",
     "meta-externalagent"
    ]
   },
   "terms": {
    "answer": "refused",
    "status": 403,
    "url": "https://www.ziprecruiter.com/terms",
    "checked": "2026-11-12",
    "clauseQuoted": false
   },
   "courtCases": 0,
   "openSource": {
    "repos": 0,
    "topStars": 0,
    "checked": "2026-11-12"
   }
  }
 ],
 "notMeasured": "Whether a page needs a real browser, CAPTCHAs, rate limits, login walls or IP blocking: one plain request is not a scraping test.",
 "licence": "CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/): credit Scraper Finder with a link to https://scraperfinder.com/what-it-takes/",
 "csv": "/data/what-it-takes.csv"
}