{
 "source": "piperic.com/tools/ai-bots",
 "count": 930,
 "bots": [
  {
   "name": "$1",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "$1",
   "sources": "matomo",
   "docs_url": "https://www.so.com/help/help_3_2.html"
  },
  {
   "name": "1001FirmsBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "1001firmsbot",
   "sources": "matomo",
   "docs_url": "https://www.1001firms.com/1001firmsbot.php"
  },
  {
   "name": "2ip",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "2ip",
   "sources": "matomo",
   "docs_url": "https://2ip.io/"
  },
  {
   "name": "Abonti",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "abonti",
   "sources": "matomo",
   "docs_url": "http://abonti.com/"
  },
  {
   "name": "Aboundexbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "aboundexbot",
   "sources": "matomo",
   "docs_url": "http://www.aboundex.com/crawler/"
  },
  {
   "name": "AccessibleWebBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "accessiblewebbot",
   "sources": "matomo",
   "docs_url": "https://accessibleweb.com/bot/"
  },
  {
   "name": "AccompanyBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "accompanybot",
   "sources": "matomo",
   "docs_url": "https://www.accompany.com/"
  },
  {
   "name": "Acoon",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "acoon",
   "sources": "matomo",
   "docs_url": "http://www.acoon.de/robot.asp"
  },
  {
   "name": "AdAuth",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "adauth",
   "sources": "matomo",
   "docs_url": "https://www.adauth.com"
  },
  {
   "name": "Adbeat",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "adbeat",
   "sources": "matomo",
   "docs_url": "https://www.adbeat.com/operation_policy"
  },
  {
   "name": "AddSearchBot",
   "operator": "Addsearch",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "unclear",
   "user_agents": "addsearchbot",
   "sources": "ai.robots.txt; cloudflare; matomo",
   "docs_url": "http://www.addsearch.com/bot"
  },
  {
   "name": "AddThis",
   "operator": "AddThis",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "addthis",
   "sources": "cloudflare",
   "docs_url": "https://docs.oracle.com/en/"
  },
  {
   "name": "Adsbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "adsbot",
   "sources": "matomo",
   "docs_url": "https://seostar.co/robot/"
  },
  {
   "name": "Adscanner",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "adscanner",
   "sources": "matomo",
   "docs_url": "https://www.alleyesonscreens.com/"
  },
  {
   "name": "AdsTxtCrawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "adstxtcrawler",
   "sources": "matomo",
   "docs_url": "https://github.com/InteractiveAdvertisingBureau/adstxtcrawler"
  },
  {
   "name": "adstxtlab.com",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "adstxtlab.com",
   "sources": "matomo",
   "docs_url": "https://adstxtlab.com/validator.php"
  },
  {
   "name": "Aegis",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "aegis",
   "sources": "matomo",
   "docs_url": "https://web.archive.org/web/20180910002802/http://www.aegis.network/"
  },
  {
   "name": "AffiliateSpyBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "affiliatespybot",
   "sources": "matomo",
   "docs_url": "https://www.affiliatespy.io/"
  },
  {
   "name": "Agency Analytics Crawler",
   "operator": "Agency Analytics",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "aa/1.0",
   "sources": "cloudflare",
   "docs_url": "https://agencyanalytics.com/aa-bot"
  },
  {
   "name": "AgentTimes",
   "operator": "The Agent Times",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "unclear",
   "user_agents": "agenttimes",
   "sources": "ai.robots.txt",
   "docs_url": "https://theagenttimes.com/about"
  },
  {
   "name": "AGI Agent",
   "operator": "AGI, Inc",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "agi agent",
   "sources": "cloudflare",
   "docs_url": "https://www.theagi.company/agent"
  },
  {
   "name": "aHrefs Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "ahrefs bot",
   "sources": "matomo",
   "docs_url": "https://ahrefs.com/robot"
  },
  {
   "name": "Ahrefs Site Audit",
   "operator": "Ahrefs",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "ahrefssiteaudit",
   "sources": "cloudflare",
   "docs_url": "https://ahrefs.com/"
  },
  {
   "name": "AhrefsBot",
   "operator": "Ahrefs",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "ahrefsbot",
   "sources": "cloudflare",
   "docs_url": "http://ahrefs.com/robot/"
  },
  {
   "name": "AI Search",
   "operator": "Cloudflare",
   "kind": "ai",
   "purpose": "answer",
   "category": "Ai Search",
   "respects_robots_txt": "yes",
   "user_agents": "cloudflare-ai-search",
   "sources": "cloudflare",
   "docs_url": "https://developers.cloudflare.com/ai-search"
  },
  {
   "name": "AI Search External",
   "operator": "Cloudflare",
   "kind": "ai",
   "purpose": "answer",
   "category": "Ai Search",
   "respects_robots_txt": "unclear",
   "user_agents": "cloudflare-ai-search-external",
   "sources": "cloudflare",
   "docs_url": "https://developers.cloudflare.com/ai-search/"
  },
  {
   "name": "AI2Bot",
   "operator": "Ai2",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "ai2bot",
   "sources": "ai.robots.txt",
   "docs_url": "https://allenai.org/crawler"
  },
  {
   "name": "AI2Bot-DeepResearchEval",
   "operator": "Ai2, a non-profit AI research institute",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "ai2bot-deepresearcheval",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Ai2Bot-Dolma",
   "operator": "Ai2",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "ai2bot-dolma",
   "sources": "ai.robots.txt",
   "docs_url": "https://allenai.org/crawler"
  },
  {
   "name": "aiHitBot",
   "operator": "aiHit",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "aihitbot",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://www.aihitdata.com/about"
  },
  {
   "name": "AirsoftdbBot",
   "operator": "Airsoftdb AB",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "yes",
   "user_agents": "airsoftdbbot",
   "sources": "cloudflare",
   "docs_url": "https://www.airsoftdb.com"
  },
  {
   "name": "AIWebIndex",
   "operator": "Lyrenth that builds an AI-readable index of web content for AI systems",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "unclear",
   "user_agents": "aiwebindex",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Alexa Crawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "alexa crawler",
   "sources": "matomo",
   "docs_url": "https://support.alexa.com/hc/en-us/sections/200100794-Crawlers"
  },
  {
   "name": "Algolia",
   "operator": "Algolia",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "algolia crawler",
   "sources": "cloudflare",
   "docs_url": "https://www.algolia.com/"
  },
  {
   "name": "All Africa Crawler",
   "operator": "AllAfrica Global Media",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "allafrica",
   "sources": "cloudflare",
   "docs_url": "https://allafrica.com/"
  },
  {
   "name": "Alli AI Bot",
   "operator": "Alli AI",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "alliai/1.0",
   "sources": "cloudflare",
   "docs_url": "https://www.alliai.com"
  },
  {
   "name": "Allloadin Favicon Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "allloadin favicon bot",
   "sources": "matomo",
   "docs_url": "https://allloadin.com/"
  },
  {
   "name": "AlltheWeb",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "alltheweb",
   "sources": "matomo",
   "docs_url": "https://web.archive.org/web/20041020050801/http://www.alltheweb.com/help/webmaster/crawler"
  },
  {
   "name": "Alphalens Bot",
   "operator": "Alphalens",
   "kind": "ai",
   "purpose": "answer",
   "category": "Ai Search",
   "respects_robots_txt": "yes",
   "user_agents": "alphalens-bot",
   "sources": "cloudflare",
   "docs_url": "https://alphalensbot.com"
  },
  {
   "name": "AlphaXCrawl",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "alphaxcrawl",
   "sources": "matomo",
   "docs_url": "https://www.fim.uni-passau.de/en/data-science/research/open-search"
  },
  {
   "name": "Amazon AdBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "amazon adbot",
   "sources": "matomo",
   "docs_url": "https://adbot.amazon.com/"
  },
  {
   "name": "Amazon Bedrock AgentCore Browser (AP Northeast)",
   "operator": "Amazon",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "amazon bedrock agentcore browser (ap northeast)",
   "sources": "cloudflare",
   "docs_url": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-tool.html"
  },
  {
   "name": "Amazon Bedrock AgentCore Browser (AP South)",
   "operator": "Amazon",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "amazon bedrock agentcore browser (ap south)",
   "sources": "cloudflare",
   "docs_url": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-tool.html"
  },
  {
   "name": "Amazon Bedrock AgentCore Browser (AP Southeast 1)",
   "operator": "Amazon",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "amazon bedrock agentcore browser (ap southeast 1)",
   "sources": "cloudflare",
   "docs_url": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-tool.html"
  },
  {
   "name": "Amazon Bedrock AgentCore Browser (AP Southeast)",
   "operator": "Amazon",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "amazon bedrock agentcore browser (ap southeast)",
   "sources": "cloudflare",
   "docs_url": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-tool.html"
  },
  {
   "name": "Amazon Bedrock AgentCore Browser (EU Central 1)",
   "operator": "Amazon",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "amazon bedrock agentcore browser (eu central 1)",
   "sources": "cloudflare",
   "docs_url": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-tool.html"
  },
  {
   "name": "Amazon Bedrock AgentCore Browser (EU West 1)",
   "operator": "Amazon",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "amazon bedrock agentcore browser (eu west 1)",
   "sources": "cloudflare",
   "docs_url": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-tool.html"
  },
  {
   "name": "Amazon Bedrock AgentCore Browser (US East 1)",
   "operator": "Amazon",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "amazon bedrock agentcore browser (us east 1)",
   "sources": "cloudflare",
   "docs_url": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-tool.html"
  },
  {
   "name": "Amazon Bedrock AgentCore Browser (US East 2)",
   "operator": "Amazon",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "amazon bedrock agentcore browser (us east 2)",
   "sources": "cloudflare",
   "docs_url": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-tool.html"
  },
  {
   "name": "Amazon Bedrock AgentCore Browser (US West 2)",
   "operator": "Amazon",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "amazon bedrock agentcore browser (us west 2)",
   "sources": "cloudflare",
   "docs_url": "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/browser-tool.html"
  },
  {
   "name": "amazon-kendra",
   "operator": "Amazon",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "yes",
   "user_agents": "amazon kendra; amazon-kendra; amazon-kendra-",
   "sources": "ai.robots.txt; cloudflare; matomo",
   "docs_url": "https://docs.aws.amazon.com/kendra/latest/dg/what-is-kendra.html"
  },
  {
   "name": "amazon-QBusiness",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "amazon-qbusiness",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Amazonbot",
   "operator": "Amazon",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "amazonbot",
   "sources": "ai.robots.txt; cloudflare; matomo",
   "docs_url": "https://developer.amazon.com/support/amazonbot"
  },
  {
   "name": "Amazonbot-Video",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "amazonbot-video",
   "sources": "matomo",
   "docs_url": "https://developer.amazon.com/support/amazonbot"
  },
  {
   "name": "AmazonBuyForMe",
   "operator": "Amazon",
   "kind": "ai",
   "purpose": "agent",
   "category": "AI Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "amazonbuyforme",
   "sources": "ai.robots.txt",
   "docs_url": "https://amazon.com"
  },
  {
   "name": "Amorank Spider",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "amorank spider",
   "sources": "matomo",
   "docs_url": "http://amorank.com/webcrawler.html"
  },
  {
   "name": "Amzn-SearchBot",
   "operator": "Amazon",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "unclear",
   "user_agents": "amzn-searchbot",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://developer.amazon.com/amazonbot"
  },
  {
   "name": "Amzn-User",
   "operator": "Amazon, used for fetching web content to answer user queries through Alexa and other Amazon AI services",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "amzn-user",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Analytics SEO Crawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "analytics seo crawler",
   "sources": "matomo",
   "docs_url": "http://www.analyticsseo.com/crawler"
  },
  {
   "name": "Anchor Browser",
   "operator": "Anchor",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "anchor browser",
   "sources": "cloudflare",
   "docs_url": "https://docs.anchorbrowser.io/"
  },
  {
   "name": "Andibot",
   "operator": "Andi",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "unclear",
   "user_agents": "andibot",
   "sources": "ai.robots.txt",
   "docs_url": "https://andisearch.com/"
  },
  {
   "name": "Anomura",
   "operator": "Direqt",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "yes",
   "user_agents": "anomura",
   "sources": "ai.robots.txt",
   "docs_url": "https://direqt.ai"
  },
  {
   "name": "Ant",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "ant",
   "sources": "matomo",
   "docs_url": "https://www.ant.com/"
  },
  {
   "name": "anthropic-ai",
   "operator": "Anthropic",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "anthropic ai; anthropic-ai",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://www.anthropic.com"
  },
  {
   "name": "Apify Website Content Crawler",
   "operator": "Apify",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "apifywebsitecontentcrawler",
   "sources": "cloudflare",
   "docs_url": "https://apify.com/apify/website-content-crawler"
  },
  {
   "name": "ApifyBot",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "unclear",
   "user_agents": "apifybot",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "ApifyWebsiteContentCrawler",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "unclear",
   "user_agents": "apifywebsitecontentcrawler",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "AppcastBot",
   "operator": "Appcast",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "appcastparsingbot",
   "sources": "cloudflare",
   "docs_url": "https://www.appcast.io"
  },
  {
   "name": "Applebot",
   "operator": "Apple",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "yes",
   "user_agents": "applebot",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "http://www.apple.com/go/applebot"
  },
  {
   "name": "Applebot-Extended",
   "operator": "Apple",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "applebot-extended",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://support.apple.com/en-us/119829#datausage"
  },
  {
   "name": "Aranet-SearchBot",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "unknown",
   "category": "Undocumented AI Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "aranet-searchbot",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "archive.org bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "archive.org bot",
   "sources": "matomo",
   "docs_url": "https://archive.org/details/archive.org_bot"
  },
  {
   "name": "ArchiveBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "archivebot",
   "sources": "matomo",
   "docs_url": "https://wiki.archiveteam.org/index.php?title=ArchiveBot"
  },
  {
   "name": "ArchiveBox",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "archivebox",
   "sources": "matomo",
   "docs_url": "https://archivebox.io/"
  },
  {
   "name": "Arena Bot",
   "operator": "Arena.im",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "arenabot",
   "sources": "cloudflare",
   "docs_url": "https://arena.im"
  },
  {
   "name": "Arquivo Web Crawler",
   "operator": "Arquivo",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Archiver",
   "respects_robots_txt": "yes",
   "user_agents": "arquivo-web-crawler",
   "sources": "cloudflare",
   "docs_url": "https://sobre.arquivo.pt/en/help/crawling-and-archiving-web-content/"
  },
  {
   "name": "Arquivo.pt",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "arquivo.pt",
   "sources": "matomo",
   "docs_url": "https://sobre.arquivo.pt/en/help/crawling-and-archiving-web-content/"
  },
  {
   "name": "ARSNova Filter System",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "arsnova filter system",
   "sources": "matomo",
   "docs_url": "https://particify.de/en/"
  },
  {
   "name": "Artemis Web Reader",
   "operator": "capjamesg",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "artemis web reader/",
   "sources": "cloudflare",
   "docs_url": "https://artemis.jamesg.blog"
  },
  {
   "name": "Artsdata Crawler",
   "operator": "Culture Creates",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "yes",
   "user_agents": "artsdata-crawler",
   "sources": "cloudflare",
   "docs_url": "https://kg.artsdata.ca"
  },
  {
   "name": "Asana",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "asana",
   "sources": "matomo",
   "docs_url": "https://asana.com/"
  },
  {
   "name": "Ask Jeeves",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "ask jeeves",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "AspiegelBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "aspiegelbot",
   "sources": "matomo",
   "docs_url": "https://aspiegel.com/"
  },
  {
   "name": "atlassian-bot",
   "operator": "Atlassian",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "yes",
   "user_agents": "atlassian-bot",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://www.atlassian.com"
  },
  {
   "name": "Attracta",
   "operator": "Attracta",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "attracta",
   "sources": "cloudflare",
   "docs_url": "https://www.attracta.com/"
  },
  {
   "name": "Authory",
   "operator": "Authory",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Archiver",
   "respects_robots_txt": "unclear",
   "user_agents": "authory/",
   "sources": "cloudflare",
   "docs_url": "authory.com"
  },
  {
   "name": "Automattic Analytics",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "automattic analytics",
   "sources": "matomo",
   "docs_url": "https://wordpress.com/crawler/"
  },
  {
   "name": "Awario",
   "operator": "Awario",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "awario",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://awario.com/bots.html"
  },
  {
   "name": "AwarioSmartBot",
   "operator": "Awario",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "awario",
   "sources": "cloudflare",
   "docs_url": "https://awario.com/bots.html"
  },
  {
   "name": "AzureAI-SearchBot",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "unclear",
   "user_agents": "azureai-searchbot",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Backlink-Check.de",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "backlink-check.de",
   "sources": "matomo",
   "docs_url": "http://www.backlink-check.de/bot.html"
  },
  {
   "name": "BacklinkCrawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "backlinkcrawler",
   "sources": "matomo",
   "docs_url": "http://www.backlinktest.com/crawler.html"
  },
  {
   "name": "BacklinksExtendedBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "backlinksextendedbot",
   "sources": "matomo",
   "docs_url": "https://www.semrush.com/bot/"
  },
  {
   "name": "BackupLand",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "backupland",
   "sources": "matomo",
   "docs_url": "https://go.backupland.com/"
  },
  {
   "name": "Baidu ADS Server Proxy",
   "operator": "Baidu",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "baidu-ads",
   "sources": "cloudflare",
   "docs_url": ""
  },
  {
   "name": "Baidu Spider",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "baidu spider",
   "sources": "matomo",
   "docs_url": "http://www.baidu.com/search/spider.htm"
  },
  {
   "name": "Baiduspider",
   "operator": "Baidu",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "baiduspider",
   "sources": "cloudflare",
   "docs_url": "http://help.baidu.com/question?prod_id=99&class=0&id=3001"
  },
  {
   "name": "Barkrowler",
   "operator": "Babbar",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "yes",
   "user_agents": "barkrowler; barkrowler/",
   "sources": "cloudflare; matomo",
   "docs_url": "https://babbar.tech/crawler"
  },
  {
   "name": "Bazqux",
   "operator": "Bazqux",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "bazqux",
   "sources": "cloudflare",
   "docs_url": "https://bazqux.com/fetcher"
  },
  {
   "name": "BazQux Reader",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "bazqux reader",
   "sources": "matomo",
   "docs_url": "https://bazqux.com/fetcher"
  },
  {
   "name": "BDCbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "bdcbot",
   "sources": "matomo",
   "docs_url": "https://bigweb.bigdatacorp.com.br/pages/faq.aspx"
  },
  {
   "name": "BDFetch",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "bdfetch",
   "sources": "matomo",
   "docs_url": "https://web.archive.org/web/20130821043949/http://www.branddimensions.com/"
  },
  {
   "name": "bedrockbot",
   "operator": "Amazon",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "yes",
   "user_agents": "bedrockbot",
   "sources": "ai.robots.txt",
   "docs_url": "https://amazon.com"
  },
  {
   "name": "BestChange Bot",
   "operator": "BestChange",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "bestchange",
   "sources": "cloudflare",
   "docs_url": "https://www.bestchange.com/wiki/rates.html"
  },
  {
   "name": "Bibliotheque Nacional de France Crawler",
   "operator": "Bibliothèque nationale de France",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Academic Research",
   "respects_robots_txt": "unclear",
   "user_agents": "bnf.fr_bot",
   "sources": "cloudflare",
   "docs_url": " https://www.bnf.fr"
  },
  {
   "name": "Big Sur AI",
   "operator": "Big Sur AI",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "bigsur.ai",
   "sources": "cloudflare",
   "docs_url": "https://docs.google.com/document/d/1eZXudiuWBY0LnTyNcXkI7QnTYuoRY2NeOfTjMMJ-QzI/edit?tab=t.0"
  },
  {
   "name": "bigsur.ai",
   "operator": "Big Sur AI that fetches website content to enable AI-powered web agents, sales assistants, and content marketing solutions for busi…",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "bigsur.ai",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Bing Preview",
   "operator": "Microsoft",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "bingpreview",
   "sources": "cloudflare",
   "docs_url": "https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0"
  },
  {
   "name": "BingBot",
   "operator": "Microsoft",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "yes",
   "user_agents": "bingbot; bingbot/",
   "sources": "cloudflare; matomo",
   "docs_url": "https://www.bing.com/bingbot.htm"
  },
  {
   "name": "Birdcrawlerbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "birdcrawlerbot",
   "sources": "matomo",
   "docs_url": "https://crawla.de/de/index.php"
  },
  {
   "name": "BitlyBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "bitlybot",
   "sources": "matomo",
   "docs_url": "https://bitly.com"
  },
  {
   "name": "Blackboard Ally",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "blackboard ally",
   "sources": "matomo",
   "docs_url": "https://ally.ac/"
  },
  {
   "name": "Blekkobot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "blekkobot",
   "sources": "matomo",
   "docs_url": "http://blekko.com/about/blekkobot"
  },
  {
   "name": "BLEXBot",
   "operator": "WebMeUp",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "blexbot",
   "sources": "cloudflare",
   "docs_url": "http://webmeup.com/"
  },
  {
   "name": "BLEXBot Crawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "blexbot crawler",
   "sources": "matomo",
   "docs_url": "http://webmeup-crawler.com"
  },
  {
   "name": "Bloglines",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "bloglines",
   "sources": "matomo",
   "docs_url": "https://web.archive.org/web/20140309033202/http://www.bloglines.com/"
  },
  {
   "name": "Bloglovin",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "bloglovin",
   "sources": "matomo",
   "docs_url": "http://www.bloglovin.com"
  },
  {
   "name": "Blogtrottr",
   "operator": "Blogtrottr",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "blogtrottr",
   "sources": "cloudflare; matomo",
   "docs_url": "https://blogtrottr.com/"
  },
  {
   "name": "Bluesky Link Preview Service",
   "operator": "Bluesky",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "bluesky/",
   "sources": "cloudflare",
   "docs_url": "https://blueskyweb.xyz/ "
  },
  {
   "name": "BoardGamePrices Bot",
   "operator": "KP Software Consult ApS",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "boardgameprices bot",
   "sources": "cloudflare",
   "docs_url": "https://boardgameprices.eu"
  },
  {
   "name": "BoardReader",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "boardreader",
   "sources": "matomo",
   "docs_url": "https://boardreader.com/"
  },
  {
   "name": "BoardReader Blog Indexer",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "boardreader blog indexer",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "BorderxBot",
   "operator": "Borderxlab (Nubestore AI)",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "borderxbot",
   "sources": "cloudflare",
   "docs_url": "https://www.nubestore.ai"
  },
  {
   "name": "Botify",
   "operator": "Botify",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "botify",
   "sources": "cloudflare; matomo",
   "docs_url": "https://www.botify.com/platform/botify-analytics/sitecrawler"
  },
  {
   "name": "Bountii Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "bountii bot",
   "sources": "matomo",
   "docs_url": "http://bountii.com/contact.php"
  },
  {
   "name": "BrandVerity",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "brandverity",
   "sources": "matomo",
   "docs_url": "https://www.brandverity.com/why-is-brandverity-visiting-me"
  },
  {
   "name": "Brandwatch",
   "operator": "Brandwatch",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "magpie-crawler",
   "sources": "cloudflare",
   "docs_url": "https://www.brandwatch.com/legal/magpie-crawler/"
  },
  {
   "name": "Bravebot",
   "operator": "Brave",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Data Providers",
   "respects_robots_txt": "yes",
   "user_agents": "bravebot",
   "sources": "ai.robots.txt; cloudflare; matomo",
   "docs_url": "https://safe.search.brave.com/help/brave-search-crawler"
  },
  {
   "name": "Brightbot",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "unclear",
   "user_agents": "brightbot",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://www.brightbot.app/"
  },
  {
   "name": "Brightbot 1.0",
   "operator": "https://brightdata.com/brightbot",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "brightbot 1.0",
   "sources": "ai.robots.txt",
   "docs_url": "https://brightdata.com/brightbot"
  },
  {
   "name": "BrightEdge",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "brightedge",
   "sources": "matomo",
   "docs_url": "https://www.brightedge.com/"
  },
  {
   "name": "Browserbase",
   "operator": "Browserbase",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "browserbase",
   "sources": "cloudflare",
   "docs_url": "https://browserbase.com"
  },
  {
   "name": "BUbiNG",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "bubing",
   "sources": "matomo",
   "docs_url": "http://law.di.unimi.it/BUbiNG.html"
  },
  {
   "name": "Buck",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "buck",
   "sources": "matomo",
   "docs_url": "https://hypefactors.com/"
  },
  {
   "name": "BuddyBot",
   "operator": "BuddyBotLearning",
   "kind": "ai",
   "purpose": "unknown",
   "category": "AI Learning Companion",
   "respects_robots_txt": "unclear",
   "user_agents": "buddybot",
   "sources": "ai.robots.txt",
   "docs_url": "https://www.buddybotlearning.com"
  },
  {
   "name": "Buffer Link Preview Bot",
   "operator": "Buffer",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "bufferlinkpreviewbot",
   "sources": "cloudflare",
   "docs_url": "https://scraper.buffer.com/about/bots/link-preview-bot"
  },
  {
   "name": "BuiltWith",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "builtwith",
   "sources": "matomo",
   "docs_url": "https://builtwith.com/biup"
  },
  {
   "name": "Butterfly Robot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "butterfly robot",
   "sources": "matomo",
   "docs_url": "http://labs.topsy.com/butterfly"
  },
  {
   "name": "Buttondown RSS-Feed-Parser",
   "operator": "Buttondown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "buttondown",
   "sources": "cloudflare",
   "docs_url": "https://docs.buttondown.com/rss-to-email#buttondown-says-it-cant-reach-my-rss-feed-but-i-can-access-it-just-fine-in-the-browser"
  },
  {
   "name": "Bytespider",
   "operator": "ByteDance",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "no",
   "user_agents": "bytespider",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://bytedance.com/"
  },
  {
   "name": "CaliberBot",
   "operator": "Conductor",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "caliperbot/1.0",
   "sources": "cloudflare",
   "docs_url": "https://www.conductor.com/"
  },
  {
   "name": "Capital One Bot",
   "operator": "Capital One",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "capitalonebot",
   "sources": "cloudflare",
   "docs_url": "https://www.capitalone.com/"
  },
  {
   "name": "CareerBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "careerbot",
   "sources": "matomo",
   "docs_url": "http://www.career-x.de/bot.html"
  },
  {
   "name": "Castopod",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "castopod",
   "sources": "matomo",
   "docs_url": "https://www.castopod.org/"
  },
  {
   "name": "CATExplorador",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "catexplorador",
   "sources": "matomo",
   "docs_url": "https://fundacio.cat/ca/domini/"
  },
  {
   "name": "CCBot",
   "operator": "Common Crawl",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "ccbot",
   "sources": "ai.robots.txt",
   "docs_url": "https://commoncrawl.org"
  },
  {
   "name": "ccBot crawler",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "ccbot crawler",
   "sources": "matomo",
   "docs_url": "http://commoncrawl.org/faq/"
  },
  {
   "name": "Channel3Bot",
   "operator": "Channel3",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "unclear",
   "user_agents": "channel3bot",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://trychannel3.com/channel3bot"
  },
  {
   "name": "Charlotte",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "charlotte",
   "sources": "matomo",
   "docs_url": "https://web.archive.org/web/20070225121634/http://www.searchme.com/support/pages/spider.php"
  },
  {
   "name": "ChatGLM-Spider",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "chatglm-spider",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "ChatGPT Agent",
   "operator": "OpenAI",
   "kind": "ai",
   "purpose": "agent",
   "category": "AI Agents",
   "respects_robots_txt": "yes",
   "user_agents": "chatgpt agent",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://openai.com"
  },
  {
   "name": "ChatGPT-Browser",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "chatgpt-browser",
   "sources": "matomo",
   "docs_url": "https://platform.openai.com/docs/bots"
  },
  {
   "name": "ChatGPT-User",
   "operator": "OpenAI",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "yes",
   "user_agents": "chatgpt-user",
   "sources": "ai.robots.txt; cloudflare; matomo",
   "docs_url": "https://openai.com"
  },
  {
   "name": "Chathive crawler",
   "operator": "Inteso Group BV",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "chathivecrawler",
   "sources": "cloudflare",
   "docs_url": "https://developers.chathive.app/crawler/overview"
  },
  {
   "name": "CheckMark Network",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "checkmark network",
   "sources": "matomo",
   "docs_url": "https://www.checkmarknetwork.com/spider.html/"
  },
  {
   "name": "Choosito",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "choosito",
   "sources": "matomo",
   "docs_url": "https://www.choosito.com/"
  },
  {
   "name": "Chrome-Lighthouse",
   "operator": "Google",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "chrome-lighthouse",
   "sources": "cloudflare",
   "docs_url": "https://developers.google.com/web/tools/lighthouse"
  },
  {
   "name": "Cincraw",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "cincraw",
   "sources": "matomo",
   "docs_url": "http://cincrawdata.net/bot/"
  },
  {
   "name": "CISPA Web Analyzer",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "cispa web analyzer",
   "sources": "matomo",
   "docs_url": "https://notify.cispa.de/"
  },
  {
   "name": "CitibotSiteCrawler",
   "operator": "Citibot",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "citibotsitecrawler",
   "sources": "cloudflare",
   "docs_url": "https://www.citibot.io/"
  },
  {
   "name": "CLASSLA-web",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "classla-web",
   "sources": "matomo",
   "docs_url": "https://www.clarin.si/info/classla-web-crawler/"
  },
  {
   "name": "Claude",
   "operator": "Anthropic",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "claudebot",
   "sources": "cloudflare",
   "docs_url": "https://www.anthropic.com/claude"
  },
  {
   "name": "Claude-Code",
   "operator": "Anthropic",
   "kind": "ai",
   "purpose": "agent",
   "category": "AI Coding Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "claude-code",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Claude-SearchBot",
   "operator": "Anthropic",
   "kind": "ai",
   "purpose": "answer",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "yes",
   "user_agents": "claude-searchbot",
   "sources": "ai.robots.txt; cloudflare; matomo",
   "docs_url": "https://www.anthropic.com"
  },
  {
   "name": "Claude-User",
   "operator": "Anthropic",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "yes",
   "user_agents": "claude-user",
   "sources": "ai.robots.txt; cloudflare; matomo",
   "docs_url": "https://www.anthropic.com"
  },
  {
   "name": "Claude-Web",
   "operator": "Anthropic",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Undocumented AI Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "claude-web",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
  },
  {
   "name": "ClaudeBot",
   "operator": "Anthropic",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "claudebot",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://www.anthropic.com"
  },
  {
   "name": "ClearscopeBot",
   "operator": "Mushi Labs",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "yes",
   "user_agents": "clearscopebot/",
   "sources": "cloudflare",
   "docs_url": "https://www.clearscope.io/support/articles/clearscopebot"
  },
  {
   "name": "Cledara SaaS Management Agent",
   "operator": "Cledara",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "cledarabot",
   "sources": "cloudflare",
   "docs_url": "https://www.cledara.com"
  },
  {
   "name": "Clickagy",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "clickagy",
   "sources": "matomo",
   "docs_url": "https://www.clickagy.com/"
  },
  {
   "name": "Cliqzbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "cliqzbot",
   "sources": "matomo",
   "docs_url": "http://cliqz.com/company/cliqzbot"
  },
  {
   "name": "CloudFlare AMP Fetcher",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "cloudflare amp fetcher",
   "sources": "matomo",
   "docs_url": "https://amp.cloudflare.com/doc/fetcher.html"
  },
  {
   "name": "Cloudflare Browser Run",
   "operator": "Cloudflare",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "cloudflare browser run",
   "sources": "cloudflare",
   "docs_url": "https://cloudflare.com"
  },
  {
   "name": "Cloudflare Crawler",
   "operator": "Cloudflare",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "cloudflarebrowserrenderingcrawler/1.0",
   "sources": "cloudflare",
   "docs_url": "https://cloudflare.com"
  },
  {
   "name": "Cloudflare Custom Error Page Crawler",
   "operator": "Cloudflare",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "collapsify/",
   "sources": "cloudflare",
   "docs_url": ""
  },
  {
   "name": "Cloudflare Custom Hostname Verification",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "cloudflare custom hostname verification",
   "sources": "cloudflare",
   "docs_url": ""
  },
  {
   "name": "Cloudflare Digicert DCV",
   "operator": "Cloudflare",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "digicert dcv",
   "sources": "cloudflare",
   "docs_url": "https://cloudflare.com"
  },
  {
   "name": "Cloudflare Healthchecks",
   "operator": "Cloudflare",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "cloudflare-healthchecks",
   "sources": "cloudflare",
   "docs_url": ""
  },
  {
   "name": "Cloudflare Prefetch",
   "operator": "Cloudflare",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "cloudflare-prefetch",
   "sources": "cloudflare",
   "docs_url": "https://www.cloudflare.com/"
  },
  {
   "name": "Cloudflare Purge",
   "operator": "Cloudflare",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "cloudflare-purge/",
   "sources": "cloudflare",
   "docs_url": ""
  },
  {
   "name": "Cloudflare SpeedTest",
   "operator": "Cloudflare",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "cloudflare speedtest",
   "sources": "cloudflare",
   "docs_url": "https://www.cloudflare.com/"
  },
  {
   "name": "Cloudflare SSLDetector",
   "operator": "Cloudflare",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "cloudflare-ssldetector",
   "sources": "cloudflare",
   "docs_url": ""
  },
  {
   "name": "Cloudflare Stream Webhook",
   "operator": "Cloudflare",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "cloudflare stream webhook",
   "sources": "cloudflare",
   "docs_url": ""
  },
  {
   "name": "Cloudflare Web Scanner",
   "operator": "Cloudflare",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "yes",
   "user_agents": "cloudflarewebscanner",
   "sources": "cloudflare",
   "docs_url": "https://developers.cloudflare.com/ai-crawl-control"
  },
  {
   "name": "Cloudflare-AutoRAG",
   "operator": "Cloudflare",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "yes",
   "user_agents": "cloudflare-autorag",
   "sources": "ai.robots.txt",
   "docs_url": "https://developers.cloudflare.com/autorag"
  },
  {
   "name": "Cloudflare-Traffic-Manager",
   "operator": "Cloudflare",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "cloudflare-traffic-manager",
   "sources": "cloudflare",
   "docs_url": ""
  },
  {
   "name": "CloudServerMarketSpider",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "cloudservermarketspider",
   "sources": "matomo",
   "docs_url": "https://web.archive.org/web/20151228225429/https://cloudservermarket.com/spider.html"
  },
  {
   "name": "CloudVertexBot",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "cloudvertexbot",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "CMU-Cylab",
   "operator": "Carnegie Mellon University",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Academic Research",
   "respects_robots_txt": "unclear",
   "user_agents": "cmu-cylab",
   "sources": "cloudflare",
   "docs_url": "https://epp-jcalandr-01.andrew.cmu.edu/about.html"
  },
  {
   "name": "Cocolyzebot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "cocolyzebot",
   "sources": "matomo",
   "docs_url": "https://cocolyze.com/en/cocolyzebot"
  },
  {
   "name": "Code",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "agent",
   "category": "AI Coding Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "code",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "cognitiveSEO Crawler",
   "operator": "cognitive SEO Internet Marketing Tools.",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "james bot",
   "sources": "cloudflare",
   "docs_url": "cognitiveseo.com"
  },
  {
   "name": "cohere-ai",
   "operator": "Cohere",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "cohere ai; cohere-ai",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://cohere.com"
  },
  {
   "name": "Cohere-Command",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Scraper",
   "respects_robots_txt": "unclear",
   "user_agents": "cohere-command",
   "sources": "matomo",
   "docs_url": "https://cohere.com/"
  },
  {
   "name": "cohere-training-data-crawler",
   "operator": "Cohere to download training data for its LLMs (Large Language Models) that power its enterprise AI products",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "cohere-training-data-crawler",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://cohere.com/"
  },
  {
   "name": "colly",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "colly",
   "sources": "matomo",
   "docs_url": "https://github.com/gocolly/colly/"
  },
  {
   "name": "CommaFeed",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "commafeed",
   "sources": "matomo",
   "docs_url": "http://www.commafeed.com"
  },
  {
   "name": "Comscore",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "comscore",
   "sources": "matomo",
   "docs_url": "https://www.comscore.com/Web-Crawler"
  },
  {
   "name": "ConvermaxBot",
   "operator": "Convermax Corp.",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "convermax/",
   "sources": "cloudflare",
   "docs_url": "https://docs.convermax.com/indexer/"
  },
  {
   "name": "Cookiebot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "cookiebot",
   "sources": "matomo",
   "docs_url": "https://support.cookiebot.com/hc/en-us/articles/360014264140-Scanner-User-Agent"
  },
  {
   "name": "CoostiBot",
   "operator": "Coosti ApS",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "yes",
   "user_agents": "coostibot",
   "sources": "cloudflare",
   "docs_url": "https://coosti.dk"
  },
  {
   "name": "CopyvioDetector",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "copyviodetector",
   "sources": "matomo",
   "docs_url": "https://copyvios.toolforge.org/"
  },
  {
   "name": "Cotoyogi",
   "operator": "ROIS",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "cotoyogi; cotoyogi/",
   "sources": "ai.robots.txt; cloudflare; matomo",
   "docs_url": "https://ds.rois.ac.jp/en_center8/en_crawler/"
  },
  {
   "name": "Coveo Bot",
   "operator": "Coveo",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "coveobot",
   "sources": "cloudflare",
   "docs_url": "https://platform.cloud.coveo.com/"
  },
  {
   "name": "CragCrawler",
   "operator": "CragSoftware, a Brazil-based software company specializing in data engineering and AI web scraping services",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "unclear",
   "user_agents": "cragcrawler",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Crawl4AI",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "unknown",
   "category": "Undocumented AI Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "crawl4ai",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Crawldad",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "crawldad",
   "sources": "matomo",
   "docs_url": "https://gist.github.com/jayhardee9/2f2a2c4dba26564ee040ae32e0dd0972"
  },
  {
   "name": "Crawlson",
   "operator": "Crawlson",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "yes",
   "user_agents": "crawlson; crawlson/",
   "sources": "cloudflare; matomo",
   "docs_url": "https://www.crawlson.com"
  },
  {
   "name": "Crawlspace",
   "operator": "Crawlspace",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "yes",
   "user_agents": "crawlspace",
   "sources": "ai.robots.txt",
   "docs_url": "https://crawlspace.dev"
  },
  {
   "name": "Crawly Project",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "crawly project",
   "sources": "matomo",
   "docs_url": "https://web.archive.org/web/20240326141952/https://crawlyproject.digitaldragon.dev/"
  },
  {
   "name": "creobot",
   "operator": "creoline GmbH",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "creobot",
   "sources": "cloudflare",
   "docs_url": "https://help.creoline.com/de/doc/content-delivery-network-igg0Tq661V"
  },
  {
   "name": "CriteoBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "criteobot",
   "sources": "matomo",
   "docs_url": "https://www.criteo.com/criteo-crawler/"
  },
  {
   "name": "CrowdTangle",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "crowdtangle",
   "sources": "matomo",
   "docs_url": "https://help.crowdtangle.com/en/articles/3009319-crowdtangle-bot"
  },
  {
   "name": "CrystalSemanticsBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "crystalsemanticsbot",
   "sources": "matomo",
   "docs_url": "https://web.archive.org/web/20121230203310/http://www.crystalsemantics.com/user-agent/"
  },
  {
   "name": "CSS Certificate Spider",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "css certificate spider",
   "sources": "matomo",
   "docs_url": "http://www.css-security.com/certificatespider/"
  },
  {
   "name": "Cursor",
   "operator": "Cursor",
   "kind": "ai",
   "purpose": "agent",
   "category": "AI Coding Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "cursor",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "CuteStat",
   "operator": "CuteStat",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "yes",
   "user_agents": "cutestat",
   "sources": "cloudflare",
   "docs_url": "https://www.cutestat.com/bot"
  },
  {
   "name": "Cxense",
   "operator": "Cxense",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "cxensebot/",
   "sources": "cloudflare",
   "docs_url": "https://www.cxense.com"
  },
  {
   "name": "CyberFind Crawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "cyberfind crawler",
   "sources": "matomo",
   "docs_url": "https://www.cyberfind.net/bot.html"
  },
  {
   "name": "Cốc Cốc",
   "operator": "Coccoc",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "coccocbot",
   "sources": "cloudflare",
   "docs_url": "http://help.coccoc.com/searchengine"
  },
  {
   "name": "Cốc Cốc Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "cốc cốc bot",
   "sources": "matomo",
   "docs_url": "https://help.coccoc.com/en/search-engine/coccoc-robots"
  },
  {
   "name": "Daric2",
   "operator": "Cloudflare",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "daric2",
   "sources": "cloudflare",
   "docs_url": "https://developers.cloudflare.com/ai-crawl-control/"
  },
  {
   "name": "Daric3",
   "operator": "Cloudflare",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "daric3",
   "sources": "cloudflare",
   "docs_url": "https://developers.cloudflare.com/ai-crawl-control/"
  },
  {
   "name": "Daric4",
   "operator": "Cloudflare",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "daric4",
   "sources": "cloudflare",
   "docs_url": "https://developers.cloudflare.com/ai-crawl-control/"
  },
  {
   "name": "DaspeedBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "daspeedbot",
   "sources": "matomo",
   "docs_url": "https://daspeed.io/"
  },
  {
   "name": "DataForSEO",
   "operator": "DataForSEO",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "rsiteauditor",
   "sources": "cloudflare",
   "docs_url": "https://dataforseo.com/"
  },
  {
   "name": "DataForSEO Bot",
   "operator": "DataForSEO",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "dataforseobot",
   "sources": "cloudflare",
   "docs_url": "https://dataforseo.com/"
  },
  {
   "name": "DataForSeoBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "dataforseobot",
   "sources": "matomo",
   "docs_url": "https://dataforseo.com/dataforseo-bot"
  },
  {
   "name": "datagnionbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "datagnionbot",
   "sources": "matomo",
   "docs_url": "https://www.datagnion.com/bot.html"
  },
  {
   "name": "Datanyze",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "datanyze",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Dataprovider",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "dataprovider",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Dataprovider.com",
   "operator": "Dataprovider.com",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "dataprovider",
   "sources": "cloudflare",
   "docs_url": "https://www.dataprovider.com/spider/"
  },
  {
   "name": "Datenbank Crawler",
   "operator": "Datenbank",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "datenbank crawler",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Daum",
   "operator": "Daum",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "daum; daum/4.1",
   "sources": "cloudflare; matomo",
   "docs_url": "https://cs.daum.net/faq/15/4118.html?faqId=28966"
  },
  {
   "name": "Dazoobot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "dazoobot",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Deep SEARCH 9",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "deep search 9",
   "sources": "matomo",
   "docs_url": "https://www.copyright.com/blog/ccc-expands-corporate-solutions-offering-with-new-technology/"
  },
  {
   "name": "Deepfield Genome",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "deepfield genome",
   "sources": "matomo",
   "docs_url": "https://www.nokia.com/networks/ip-networks/deepfield/genome/"
  },
  {
   "name": "deepnoc",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "deepnoc",
   "sources": "matomo",
   "docs_url": "https://deepnoc.com/bot"
  },
  {
   "name": "DeepSeekBot",
   "operator": "DeepSeek",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "no",
   "user_agents": "deepseekbot",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://www.deepseek.com/bot"
  },
  {
   "name": "DepSpid",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "depspid",
   "sources": "matomo",
   "docs_url": "https://web.archive.org/web/20080321224033/http://about.depspid.net/"
  },
  {
   "name": "Devin",
   "operator": "Devin AI",
   "kind": "ai",
   "purpose": "agent",
   "category": "AI Coding Agents",
   "respects_robots_txt": "yes",
   "user_agents": "devin",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://docs.devin.ai/get-started/devin-intro"
  },
  {
   "name": "Diffbot",
   "operator": "Diffbot",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "unclear",
   "user_agents": "diffbot",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://www.diffbot.com/"
  },
  {
   "name": "DigiCert DCV",
   "operator": "DigiCert",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "digicert dcv bot",
   "sources": "cloudflare",
   "docs_url": "https://www.digicert.com/"
  },
  {
   "name": "Direqt Anomura",
   "operator": "Direqt",
   "kind": "ai",
   "purpose": "answer",
   "category": "Ai Search",
   "respects_robots_txt": "unclear",
   "user_agents": "anomura",
   "sources": "cloudflare",
   "docs_url": "https://dashboarddocs.direqt.ai/direqt-bots/direqt-crawlers-and-user-agents"
  },
  {
   "name": "Discobot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "discobot",
   "sources": "matomo",
   "docs_url": "http://discoveryengine.com/discobot.html"
  },
  {
   "name": "discordBot",
   "operator": "Discord, Inc.",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "discordbot/",
   "sources": "cloudflare",
   "docs_url": "https://discord.com"
  },
  {
   "name": "Disqus",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "disqus",
   "sources": "matomo",
   "docs_url": "https://help.disqus.com/en/articles/1765357-ads-txt-implementation-guide"
  },
  {
   "name": "dlvr.it Feed Fetcher",
   "operator": "dlvr.it",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "yes",
   "user_agents": "dlvr.it/1.0",
   "sources": "cloudflare",
   "docs_url": "https://dlvr.it"
  },
  {
   "name": "Domain Codex",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "domain codex",
   "sources": "matomo",
   "docs_url": "https://www.domaincodex.com/"
  },
  {
   "name": "Domain Re-Animator Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "domain re-animator bot",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Domain Research Project",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "domain research project",
   "sources": "matomo",
   "docs_url": "https://trentwil.es/domains.html"
  },
  {
   "name": "DomainAppender",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "domainappender",
   "sources": "matomo",
   "docs_url": "https://www.profound.net/product/domain_append/"
  },
  {
   "name": "DomainCrawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "domaincrawler",
   "sources": "matomo",
   "docs_url": "https://domaincrawler.com/about-us/"
  },
  {
   "name": "Domains Project",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "domains project",
   "sources": "matomo",
   "docs_url": "https://domainsproject.org"
  },
  {
   "name": "DomainStatsBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "domainstatsbot",
   "sources": "matomo",
   "docs_url": "https://domainstats.com/pages/our-bot"
  },
  {
   "name": "DomCop Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "domcop bot",
   "sources": "matomo",
   "docs_url": "https://www.domcop.com/bot"
  },
  {
   "name": "DotBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "dotbot",
   "sources": "matomo",
   "docs_url": "http://www.opensiteexplorer.org/dotbot"
  },
  {
   "name": "Doximity-Diffbot",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Scraper",
   "respects_robots_txt": "unclear",
   "user_agents": "doximity-diffbot",
   "sources": "matomo",
   "docs_url": "https://docs.diffbot.com/docs/getting-started-with-crawl"
  },
  {
   "name": "Dubbotbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "dubbotbot",
   "sources": "matomo",
   "docs_url": "https://help.dubbot.com/en/articles/6746594-example-custom-user-agent"
  },
  {
   "name": "DuckAssistBot",
   "operator": "DuckDuckGo",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "duckassistbot",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/"
  },
  {
   "name": "DuckDuckBot",
   "operator": "DuckDuckGo",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "duckduckbot",
   "sources": "cloudflare; matomo",
   "docs_url": "http://duckduckgo.com/duckduckbot.html"
  },
  {
   "name": "DuplexWeb-Google",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "duplexweb-google",
   "sources": "matomo",
   "docs_url": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers"
  },
  {
   "name": "Easou Spider",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "easou spider",
   "sources": "matomo",
   "docs_url": "http://www.easou.com/search/spider.html"
  },
  {
   "name": "EasyScan",
   "operator": "codire GmbH",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "easyscan",
   "sources": "cloudflare",
   "docs_url": "https://www.it-recht-kanzlei.de/website-scanner-fuer-mandanten.php"
  },
  {
   "name": "eCairn-Grabber",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "ecairn-grabber",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Echobot Bot",
   "operator": "Echobox",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "echobot bot; w4mwnpbxf3mfabxokjrw",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://www.echobox.com/"
  },
  {
   "name": "EchoboxBot",
   "operator": "Echobox",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "unclear",
   "user_agents": "echoboxbot",
   "sources": "ai.robots.txt",
   "docs_url": "https://echobox.com"
  },
  {
   "name": "EchoFeed",
   "operator": "Robb Knight",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "echofeed",
   "sources": "cloudflare",
   "docs_url": "https://echofeed.app"
  },
  {
   "name": "EFF Do Not Track Policy Checker",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "eff do not track policy checker",
   "sources": "matomo",
   "docs_url": "https://www.eff.org/dnt-policy"
  },
  {
   "name": "EFF Do Not Track Verifier",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "eff do not track verifier",
   "sources": "matomo",
   "docs_url": "https://www.eff.org/issues/do-not-track"
  },
  {
   "name": "Element451Bot",
   "operator": "Spark451, Inc. (Element451)",
   "kind": "ai",
   "purpose": "answer",
   "category": "Ai Search",
   "respects_robots_txt": "yes",
   "user_agents": "element451bot",
   "sources": "cloudflare",
   "docs_url": "https://element451.com"
  },
  {
   "name": "EMail Exractor",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "email exractor",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Embedly",
   "operator": "Embedly",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "embedly",
   "sources": "cloudflare; matomo",
   "docs_url": "https://embed.ly"
  },
  {
   "name": "eMoney Advisor",
   "operator": "eMoney Advisor",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "emoneybot/",
   "sources": "cloudflare",
   "docs_url": "https://emoneyadvisor.com/"
  },
  {
   "name": "Entfer",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "entfer",
   "sources": "matomo",
   "docs_url": "https://entfer.com/"
  },
  {
   "name": "Epivoz Crawler",
   "operator": "Techmeme",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "epivozcrawler/",
   "sources": "cloudflare",
   "docs_url": "https://www.techmeme.com/"
  },
  {
   "name": "evc-batch",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "evc-batch",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Everyfeed",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "everyfeed",
   "sources": "matomo",
   "docs_url": "https://web.archive.org/web/20050930235914/http://www.everyfeed.com/"
  },
  {
   "name": "ExaBot",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "unclear",
   "user_agents": "exabot",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "http://www.exabot.com/go/robot"
  },
  {
   "name": "ExactSeek Crawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "exactseek crawler",
   "sources": "matomo",
   "docs_url": "http://www.exactseek.com"
  },
  {
   "name": "Example3",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "example3",
   "sources": "matomo",
   "docs_url": "https://www.example3.com/"
  },
  {
   "name": "Eyeotabot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "eyeotabot",
   "sources": "matomo",
   "docs_url": "https://www.eyeota.com/"
  },
  {
   "name": "eZ Publish Link Validator",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "ez publish link validator",
   "sources": "matomo",
   "docs_url": "https://ez.no/"
  },
  {
   "name": "EzoicBot",
   "operator": "Ezoic Inc",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "http://www.ezoic.com/bot.html",
   "sources": "cloudflare",
   "docs_url": "https://www.ezoic.com/bot/"
  },
  {
   "name": "Ezooms",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "ezooms",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "FacebookBot",
   "operator": "Meta",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "facebookbot",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://developers.facebook.com/docs/sharing/bot"
  },
  {
   "name": "facebookexternalhit",
   "operator": "Meta",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "no",
   "user_agents": "facebookexternalhit",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers"
  },
  {
   "name": "Factset_spyderbot",
   "operator": "Factset",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "factset_spyderbot",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://www.factset.com/ai"
  },
  {
   "name": "Fastmail Bot",
   "operator": "Fastmail",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "fastmailua",
   "sources": "cloudflare",
   "docs_url": "https://www.fastmail.com/policies/bots/"
  },
  {
   "name": "Faveeo",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "faveeo",
   "sources": "matomo",
   "docs_url": "http://www.faveeo.com"
  },
  {
   "name": "FedReporter Bot for FFIEC",
   "operator": "Fed Reporter, Inc.",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "fedreporterdatabot",
   "sources": "cloudflare",
   "docs_url": "https://fedreporter.net/FedReporterBotDocumentation/Readme.txt"
  },
  {
   "name": "Feed Wrangler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "feed wrangler",
   "sources": "matomo",
   "docs_url": "https://feedwrangler.net/"
  },
  {
   "name": "Feedbin",
   "operator": "Feedbin",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "feedbin",
   "sources": "cloudflare; matomo",
   "docs_url": "https://feedbin.com/help"
  },
  {
   "name": "FeedBlitz",
   "operator": "FeedBlitz LLC",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "feedblitz",
   "sources": "cloudflare",
   "docs_url": "https://www.feedblitz.com"
  },
  {
   "name": "FeedBurner",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "feedburner",
   "sources": "matomo",
   "docs_url": "http://www.feedburner.com"
  },
  {
   "name": "Feeder",
   "operator": "Really Simple AB",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "yes",
   "user_agents": "feeder.co",
   "sources": "cloudflare",
   "docs_url": "https://feeder.co"
  },
  {
   "name": "Feedly",
   "operator": "Feedly",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "feedly; feedly/1",
   "sources": "cloudflare; matomo",
   "docs_url": "https://www.feedly.com/fetcher.html"
  },
  {
   "name": "FeedOtter",
   "operator": "FeedOtter",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "feedotter",
   "sources": "cloudflare",
   "docs_url": "http://www.feedotter.com/privacy"
  },
  {
   "name": "Feedspot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "feedspot",
   "sources": "matomo",
   "docs_url": "http://www.feedspot.com"
  },
  {
   "name": "FeedWind Crawler",
   "operator": "Mikle",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "feedwind/",
   "sources": "cloudflare",
   "docs_url": "https://feed.mikle.com"
  },
  {
   "name": "Femtosearch",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "femtosearch",
   "sources": "matomo",
   "docs_url": "http://femtosearch.com/"
  },
  {
   "name": "Fever",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "fever",
   "sources": "matomo",
   "docs_url": "http://feedafever.com/"
  },
  {
   "name": "FindFiles.net",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "findfiles.net",
   "sources": "matomo",
   "docs_url": "https://findfiles.net/bot"
  },
  {
   "name": "Findxbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "findxbot",
   "sources": "matomo",
   "docs_url": "http://www.findxbot.com"
  },
  {
   "name": "FirecrawlAgent",
   "operator": "Firecrawl that extracts web content and converts it into structured data for use in LLM and AI applications",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "yes",
   "user_agents": "firecrawlagent",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://www.firecrawl.dev/"
  },
  {
   "name": "FirmlyAI Bot",
   "operator": "Firmly",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "firmlyai bot",
   "sources": "cloudflare",
   "docs_url": "https://www.firmly.ai/"
  },
  {
   "name": "FishBot",
   "operator": "FishBot",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "fishbot",
   "sources": "cloudflare",
   "docs_url": "https://fish.audio"
  },
  {
   "name": "FIT Feeds Fetcher",
   "operator": "follow.it",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "follow.it",
   "sources": "cloudflare",
   "docs_url": "https://follow.it"
  },
  {
   "name": "Flipboard",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "flipboard",
   "sources": "matomo",
   "docs_url": "http://flipboard.com/browserproxy"
  },
  {
   "name": "FlipboardProxy",
   "operator": "Flipboard",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "flipboardproxy",
   "sources": "cloudflare",
   "docs_url": "https://about.flipboard.com/browserproxy/"
  },
  {
   "name": "FlipboardRSS",
   "operator": "Flipboard",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "flipboardrss",
   "sources": "cloudflare",
   "docs_url": "https://about.flipboard.com/rss-guidelines/"
  },
  {
   "name": "FlyingPress",
   "operator": "FlyingWeb LLC",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "flyingpress",
   "sources": "cloudflare",
   "docs_url": "https://flyingpress.com"
  },
  {
   "name": "FN Legislative and Regulatory Bot",
   "operator": "FiscalNote Holdings, Inc.",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "fiscalnote",
   "sources": "cloudflare",
   "docs_url": "https://fiscalnote.com"
  },
  {
   "name": "FontRadar",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "fontradar",
   "sources": "matomo",
   "docs_url": "https://www.fontradar.com/"
  },
  {
   "name": "fragFINN",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "fragfinn",
   "sources": "matomo",
   "docs_url": "https://www.fragfinn.de/"
  },
  {
   "name": "Freespoke",
   "operator": "Freespoke",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "crawler.freespoke.com",
   "sources": "cloudflare",
   "docs_url": "https://docs.freespoke.com/search/bot/"
  },
  {
   "name": "FreshRSS",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "freshrss",
   "sources": "matomo",
   "docs_url": "https://freshrss.org/"
  },
  {
   "name": "FriendlyCrawler",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "friendlycrawler",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "FullFact Ingest",
   "operator": "FullFact",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "fullfact",
   "sources": "cloudflare",
   "docs_url": "https://fullfact.ai"
  },
  {
   "name": "Functionize",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "functionize",
   "sources": "matomo",
   "docs_url": "https://www.functionize.com/"
  },
  {
   "name": "Funnelback",
   "operator": "Squiz - FunnelBack",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "funnelback",
   "sources": "cloudflare",
   "docs_url": "https://docs.squiz.net/funnelback/docs/latest/"
  },
  {
   "name": "Gaisbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "gaisbot",
   "sources": "matomo",
   "docs_url": "https://web.archive.org/web/20090604121511/https://gais.cs.ccu.edu.tw/robot.php"
  },
  {
   "name": "GDNP",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "gdnp",
   "sources": "matomo",
   "docs_url": "https://gdnplus.com/"
  },
  {
   "name": "GeedoBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "geedobot",
   "sources": "matomo",
   "docs_url": "https://geedo.com/bot/"
  },
  {
   "name": "GeedoShopProductFinder",
   "operator": "Geedo SIA",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "yes",
   "user_agents": "geedoshopproductfinder",
   "sources": "cloudflare; matomo",
   "docs_url": "https://geedo.com/product-finder/"
  },
  {
   "name": "GeistHaus-PageFetcher",
   "operator": "GeistHaus, a company developing AI systems for therapy and psychological assessment",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "geisthaus-pagefetcher",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Gemini-AI",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Search Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "gemini-ai",
   "sources": "matomo",
   "docs_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers"
  },
  {
   "name": "Gemini-Deep-Research",
   "operator": "Google",
   "kind": "ai",
   "purpose": "agent",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "gemini-deep-research",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://gemini.google/overview/deep-research/"
  },
  {
   "name": "Generic Bot",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "generic bot",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "GhostExplore",
   "operator": "Ghost",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "ghostexplore",
   "sources": "cloudflare",
   "docs_url": "https://ghost.org"
  },
  {
   "name": "Gigablast",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "gigablast",
   "sources": "matomo",
   "docs_url": "https://github.com/gigablast/open-source-search-engine"
  },
  {
   "name": "Gigabot",
   "operator": "Gigablast",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "gigabot",
   "sources": "cloudflare",
   "docs_url": "https://www.gigablast.com/?c=main"
  },
  {
   "name": "Github Camo",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "github camo",
   "sources": "matomo",
   "docs_url": "https://github.com/atmos/camo"
  },
  {
   "name": "Gluten Free Crawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "gluten free crawler",
   "sources": "matomo",
   "docs_url": "http://glutenfreepleasure.com/"
  },
  {
   "name": "Gmail Image Proxy",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "gmail image proxy",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Golfe",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "golfe",
   "sources": "matomo",
   "docs_url": "http://www.goo-olfe.ae/bot.html"
  },
  {
   "name": "Goo",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "goo",
   "sources": "matomo",
   "docs_url": "http://search.goo.ne.jp/option/use/sub4/sub4-1"
  },
  {
   "name": "Google Area 120 Privacy Policy Fetcher",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "google area 120 privacy policy fetcher",
   "sources": "matomo",
   "docs_url": "https://area120.google.com/"
  },
  {
   "name": "Google Calendar Importer",
   "operator": "Google",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "google-calendar-importer",
   "sources": "cloudflare",
   "docs_url": "https://support.google.com/calendar/answer/37100?hl=en&co=GENIE.Platform%3DDesktop"
  },
  {
   "name": "Google Cloud Scheduler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "google cloud scheduler",
   "sources": "matomo",
   "docs_url": "https://cloud.google.com/scheduler"
  },
  {
   "name": "Google Digital Asset Links",
   "operator": "Google",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "googleassociationservice",
   "sources": "cloudflare",
   "docs_url": ""
  },
  {
   "name": "Google Favicon",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "google favicon",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Google Feed Fetcher",
   "operator": "Google",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "google.com/feedfetcher",
   "sources": "cloudflare",
   "docs_url": "https://support.google.com/webmasters/answer/1061943?hl=en"
  },
  {
   "name": "Google Image Proxy",
   "operator": "Google",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "googleimageproxy",
   "sources": "cloudflare",
   "docs_url": "https://support.google.com/webmasters/answer/1061943?hl=en"
  },
  {
   "name": "Google Images",
   "operator": "Google",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "googlebot-image",
   "sources": "cloudflare",
   "docs_url": "https://support.google.com/webmasters/answer/1061943?hl=en"
  },
  {
   "name": "Google Page Renderer",
   "operator": "Google",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "google-pagerenderer",
   "sources": "cloudflare",
   "docs_url": "https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot"
  },
  {
   "name": "Google Publisher Center",
   "operator": "Google",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "googleproducer",
   "sources": "cloudflare",
   "docs_url": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers#googleproducer"
  },
  {
   "name": "Google Scholar",
   "operator": "Google",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "googlebot-ia/",
   "sources": "cloudflare",
   "docs_url": "https://scholar.google.com/intl/en/scholar/libraries.html"
  },
  {
   "name": "Google Search Console",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "google search console",
   "sources": "matomo",
   "docs_url": "https://search.google.com/search-console/about"
  },
  {
   "name": "Google StoreBot",
   "operator": "Google",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "google storebot; storebot-google",
   "sources": "cloudflare; matomo",
   "docs_url": "https://developers.google.com/search/docs/advanced/crawling/overview-google-crawlers"
  },
  {
   "name": "Google Videos",
   "operator": "Google",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "googlebot-video",
   "sources": "cloudflare",
   "docs_url": "https://support.google.com/webmasters/answer/1061943?hl=en"
  },
  {
   "name": "Google Web Snippet",
   "operator": "Google",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "google-pagerenderer",
   "sources": "cloudflare",
   "docs_url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"
  },
  {
   "name": "Google-AdWords-Express",
   "operator": "Google",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "google-adwords-express",
   "sources": "cloudflare",
   "docs_url": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers"
  },
  {
   "name": "Google-Adwords-Instant",
   "operator": "Google",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "google-adwords-instant",
   "sources": "cloudflare",
   "docs_url": "https://www.google.com/adsbot.html"
  },
  {
   "name": "Google-Agent",
   "operator": "Google",
   "kind": "ai",
   "purpose": "agent",
   "category": "AI Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "google-agent",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers#google-agent"
  },
  {
   "name": "Google-CloudVertexBot",
   "operator": "Google",
   "kind": "ai",
   "purpose": "agent",
   "category": "AI Agents",
   "respects_robots_txt": "yes",
   "user_agents": "cloudvertexbot; google-cloudvertexbot",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers#google-cloudvertexbot"
  },
  {
   "name": "Google-Extended",
   "operator": "Google",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "google-extended",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers"
  },
  {
   "name": "Google-Firebase",
   "operator": "Google",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "google-firebase",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Google-Flights-Search-Spackle",
   "operator": "Google",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "google-flights-search-spackle",
   "sources": "cloudflare",
   "docs_url": "https://www.google.com/travel/flights/saves"
  },
  {
   "name": "Google-Gemini-CLI",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "agent",
   "category": "AI Coding Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "google-gemini-cli",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Google-ilp",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "google-ilp",
   "sources": "matomo",
   "docs_url": "https://www.google.com/"
  },
  {
   "name": "Google-NotebookLM",
   "operator": "Google",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "google-notebooklm",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Google-Read-Aloud",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "google-read-aloud",
   "sources": "matomo",
   "docs_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/read-aloud-user-agent"
  },
  {
   "name": "Google-Safety",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "google-safety",
   "sources": "matomo",
   "docs_url": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers"
  },
  {
   "name": "google-xrawler",
   "operator": "Google",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "google-xrawler",
   "sources": "cloudflare",
   "docs_url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"
  },
  {
   "name": "GoogleAgent-Mariner",
   "operator": "Google",
   "kind": "ai",
   "purpose": "agent",
   "category": "AI Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "googleagent-mariner",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "GoogleAgent-URLContext",
   "operator": "Google that retrieves web content on behalf of Gemini API users",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "googleagent-urlcontext",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "GoogleBot",
   "operator": "Google",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "googlebot; googlebot/",
   "sources": "cloudflare; matomo",
   "docs_url": "https://support.google.com/webmasters/answer/1061943?hl=en"
  },
  {
   "name": "Googlebot News",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "googlebot news",
   "sources": "matomo",
   "docs_url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"
  },
  {
   "name": "GoogleOther",
   "operator": "Google",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "yes",
   "user_agents": "googleother",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers"
  },
  {
   "name": "GoogleOther-Image",
   "operator": "Google",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "yes",
   "user_agents": "googleother-image",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "GoogleOther-Video",
   "operator": "Google",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "yes",
   "user_agents": "googleother-video",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Gowikibot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "gowikibot",
   "sources": "matomo",
   "docs_url": "http:/www.gowikibot.com"
  },
  {
   "name": "Gozle",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "gozle",
   "sources": "matomo",
   "docs_url": "https://gozle.com.tm/en/blog/post/1"
  },
  {
   "name": "GPTBot",
   "operator": "OpenAI",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "gptbot",
   "sources": "ai.robots.txt; cloudflare; matomo",
   "docs_url": "https://openai.com"
  },
  {
   "name": "Grapeshot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "grapeshot",
   "sources": "matomo",
   "docs_url": "https://www.grapeshot.com/crawler"
  },
  {
   "name": "Gregarius",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "gregarius",
   "sources": "matomo",
   "docs_url": "https://web.archive.org/web/20100614011837/http://devlog.gregarius.net/docs/ua/"
  },
  {
   "name": "GTmetrix",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "gtmetrix",
   "sources": "matomo",
   "docs_url": "https://gtmetrix.com/"
  },
  {
   "name": "HaloBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "halobot",
   "sources": "matomo",
   "docs_url": "https://www.haloscan.com/"
  },
  {
   "name": "Hatena",
   "operator": "Hatena",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "yes",
   "user_agents": "hatena",
   "sources": "cloudflare",
   "docs_url": "https://developer.hatena.ne.jp/ja/documents/other/apis/useragents/"
  },
  {
   "name": "Hatena Bookmark",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "hatena bookmark",
   "sources": "matomo",
   "docs_url": "https://www.hatena.ne.jp/faq/"
  },
  {
   "name": "Hatena Favicon",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "hatena favicon",
   "sources": "matomo",
   "docs_url": "https://www.hatena.ne.jp/faq/"
  },
  {
   "name": "Headline",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "headline",
   "sources": "matomo",
   "docs_url": "https://headline.com/legal/crawler"
  },
  {
   "name": "Heexybot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "heexybot",
   "sources": "matomo",
   "docs_url": "https://heexy.org/bot"
  },
  {
   "name": "HelloWork",
   "operator": "HelloWork Group",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "helloworkjobpostingbot",
   "sources": "cloudflare",
   "docs_url": "https://www.hellowork-group.com/en/"
  },
  {
   "name": "HenkBot",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "unclear",
   "user_agents": "henkbot",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Henry Shopping Agent",
   "operator": "Henry Labs",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "henry shopping agent",
   "sources": "cloudflare",
   "docs_url": "https://docs.henrylabs.ai/"
  },
  {
   "name": "Heritrix",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "heritrix",
   "sources": "matomo",
   "docs_url": "https://webarchive.jira.com/wiki/display/Heritrix/Heritrix"
  },
  {
   "name": "Heureka Feed",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "heureka feed",
   "sources": "matomo",
   "docs_url": "https://sluzby.heureka.cz/napoveda/heurekabot/"
  },
  {
   "name": "HEY Email Privacy Proxy",
   "operator": "37signals",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "hey.com/imageproxy",
   "sources": "cloudflare",
   "docs_url": "https://www.hey.com/spy-trackers/"
  },
  {
   "name": "HIFIBot",
   "operator": "HIFI",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "hifibot",
   "sources": "cloudflare",
   "docs_url": "https://hi.fi/"
  },
  {
   "name": "htmlyse",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "htmlyse",
   "sources": "matomo",
   "docs_url": "https://www.htmlyse.com/"
  },
  {
   "name": "httpx",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "httpx",
   "sources": "matomo",
   "docs_url": "https://github.com/projectdiscovery/httpx"
  },
  {
   "name": "HuaweiWebCatBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "huaweiwebcatbot",
   "sources": "matomo",
   "docs_url": "https://isecurity.huawei.com"
  },
  {
   "name": "HubPages",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "hubpages",
   "sources": "matomo",
   "docs_url": "https://hubpages.com/help/crawlingpolicy"
  },
  {
   "name": "HubSpot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "hubspot",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "HubSpot Feed Fetcher",
   "operator": "HubSpot",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "hubspot-feedfetcher",
   "sources": "cloudflare",
   "docs_url": "https://www.hubspot.com/"
  },
  {
   "name": "HubSpot Page Fetcher",
   "operator": "HubSpot",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "hubspot page fetcher",
   "sources": "cloudflare",
   "docs_url": "https://www.hubspot.com/"
  },
  {
   "name": "HubSpotContentSearchBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "hubspotcontentsearchbot",
   "sources": "matomo",
   "docs_url": "https://developers.hubspot.com/docs/cms/start-building/building-blocks/modules/content-search"
  },
  {
   "name": "Huckabuy Bot",
   "operator": "Huckabuy",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "huckabot/",
   "sources": "cloudflare",
   "docs_url": "https://huckabuy.com/"
  },
  {
   "name": "Hype Machine",
   "operator": "Hype Machine",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "hype machine",
   "sources": "cloudflare",
   "docs_url": "https://hypem.com/latest"
  },
  {
   "name": "HypeStat",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "hypestat",
   "sources": "matomo",
   "docs_url": "https://hypestat.com/crawler"
  },
  {
   "name": "i-search-crawler",
   "operator": "Scala Communications, Inc.",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "yes",
   "user_agents": "i-search-crawler",
   "sources": "cloudflare",
   "docs_url": "https://scala-com.jp/"
  },
  {
   "name": "iAskBot",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "unknown",
   "category": "Undocumented AI Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "iaskbot",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "iaskspider",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "unknown",
   "category": "Undocumented AI Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "iaskspider",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "iaskspider/2.0",
   "operator": "iAsk",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "no",
   "user_agents": "iaskspider/2.0",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "IbouBot",
   "operator": "Ibou",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "yes",
   "user_agents": "iboubot",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://ibou.io/iboubot.html"
  },
  {
   "name": "ICC-Crawler",
   "operator": "NICT",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "icc-crawler; icc-crawler/",
   "sources": "ai.robots.txt; cloudflare; matomo",
   "docs_url": "https://nict.go.jp"
  },
  {
   "name": "Idealo",
   "operator": "Idealo internet GmbH",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "yes",
   "user_agents": "idealo-bot",
   "sources": "cloudflare",
   "docs_url": "https://partner.idealo.com/de/bot"
  },
  {
   "name": "IDG",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "idg",
   "sources": "matomo",
   "docs_url": "https://www.spaziodati.eu/"
  },
  {
   "name": "Iframely",
   "operator": "Iframely",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "https://iframely.com/docs/about; iframely",
   "sources": "cloudflare; matomo",
   "docs_url": "https://iframely.com"
  },
  {
   "name": "IFTTT RSS Feed Service",
   "operator": "IFTTT RSS Feed Servuce",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "ifttt",
   "sources": "cloudflare",
   "docs_url": "https://ifttt.com/feed/details"
  },
  {
   "name": "IIS Site Analysis",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "iis site analysis",
   "sources": "matomo",
   "docs_url": "http://www.iis.net/iisbot.html"
  },
  {
   "name": "ImageSift",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Scraper",
   "respects_robots_txt": "unclear",
   "user_agents": "imagesift",
   "sources": "matomo",
   "docs_url": "https://imagesift.com/"
  },
  {
   "name": "ImagesiftBot",
   "operator": "ImageSift",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "yes",
   "user_agents": "imagesiftbot",
   "sources": "ai.robots.txt",
   "docs_url": "https://imagesift.com"
  },
  {
   "name": "imageSpider",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "imagespider",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "img2dataset",
   "operator": "img2dataset",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "img2dataset",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://github.com/rom1504/img2dataset"
  },
  {
   "name": "IndeedJobBot",
   "operator": "Indeed",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "yes",
   "user_agents": "indeedjobbot",
   "sources": "cloudflare",
   "docs_url": "https://www.indeed.com/about/indeed-crawlers"
  },
  {
   "name": "Inetdex Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "inetdex bot",
   "sources": "matomo",
   "docs_url": "https://www.inetdex.com/"
  },
  {
   "name": "Infegy",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "infegy",
   "sources": "matomo",
   "docs_url": "https://infegy.com/"
  },
  {
   "name": "InfoRoom Reader",
   "operator": "GreenerSoft",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "inforoomreader",
   "sources": "cloudflare",
   "docs_url": "https://inforoom.fr"
  },
  {
   "name": "InfoTigerBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "infotigerbot",
   "sources": "matomo",
   "docs_url": "https://infotiger.com/bot"
  },
  {
   "name": "Innguma Fetcher",
   "operator": "Innguma",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "innguma",
   "sources": "cloudflare",
   "docs_url": "https://factory.innguma.com/fetcher/"
  },
  {
   "name": "Innologica",
   "operator": "Innologica",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "inoreader.com",
   "sources": "cloudflare",
   "docs_url": "https://www.inoreader.com"
  },
  {
   "name": "Inspici",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "inspici",
   "sources": "matomo",
   "docs_url": "https://www.inspici.com/"
  },
  {
   "name": "Instapaper",
   "operator": "Instant Paper",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "instapaper; instapaper/4.0",
   "sources": "cloudflare",
   "docs_url": "https://www.instapaper.com/publishers"
  },
  {
   "name": "InsytfulBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "insytfulbot",
   "sources": "matomo",
   "docs_url": "https://www.insytful.com/"
  },
  {
   "name": "Intelligence X",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "intelligence x",
   "sources": "matomo",
   "docs_url": "https://intelx.io/"
  },
  {
   "name": "Internet Archive",
   "operator": "Internet Archive",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Archiver",
   "respects_robots_txt": "yes",
   "user_agents": "archive.org_bot",
   "sources": "cloudflare",
   "docs_url": "https://www.archive.org/details/archive.org_bot"
  },
  {
   "name": "Internet Archive - Archive-It",
   "operator": "Archive-It",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Archiver",
   "respects_robots_txt": "unclear",
   "user_agents": "archive-it",
   "sources": "cloudflare",
   "docs_url": "https://archive-it.org/files/site-owners.html"
  },
  {
   "name": "InternetMeasurement",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "internetmeasurement",
   "sources": "matomo",
   "docs_url": "https://internet-measurement.com/"
  },
  {
   "name": "IONOS Crawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "ionos crawler",
   "sources": "matomo",
   "docs_url": "https://www.ionos.de/terms-gtc/faq-crawler-en/"
  },
  {
   "name": "IP-Guide Crawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "ip-guide crawler",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "IPS Agent",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "ips agent",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "IsItWP",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "isitwp",
   "sources": "matomo",
   "docs_url": "https://www.isitwp.com/"
  },
  {
   "name": "ISSCyberRiskCrawler",
   "operator": "ISS-Corporate",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "no",
   "user_agents": "isscyberriskcrawler",
   "sources": "ai.robots.txt",
   "docs_url": "https://iss-cyber.com"
  },
  {
   "name": "iTMS",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "itms",
   "sources": "matomo",
   "docs_url": "https://support.apple.com/en-us/119829"
  },
  {
   "name": "JobboerseBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "jobboersebot",
   "sources": "matomo",
   "docs_url": "https://www.xing.com/jobs"
  },
  {
   "name": "JobicyBot",
   "operator": "Jobicy",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "yes",
   "user_agents": "jobicybot",
   "sources": "cloudflare",
   "docs_url": "https://jobicy.com/"
  },
  {
   "name": "jobswithgptcom-bot",
   "operator": "JobsWithGPT",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "jobswithgptcom",
   "sources": "cloudflare",
   "docs_url": "https://jobswithgpt.com/bot.html"
  },
  {
   "name": "JungleKeyThumbnail",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "junglekeythumbnail",
   "sources": "matomo",
   "docs_url": "https://junglekey.com/"
  },
  {
   "name": "KadoBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "kadobot",
   "sources": "matomo",
   "docs_url": "https://www.kadolijst.nl/bot"
  },
  {
   "name": "Kagi Bot",
   "operator": "Kagi",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "kagibot/",
   "sources": "cloudflare",
   "docs_url": "https://kagi.com"
  },
  {
   "name": "kagi-fetcher",
   "operator": "Kagi that fetches web content to answer user queries through Kagi AI, their suite of AI-powered tools including Assistant, Res…",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "kagi-fetcher",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Kagibot",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "kagibot",
   "sources": "matomo",
   "docs_url": "https://kagi.com/bot"
  },
  {
   "name": "KakaoTalk Scrap",
   "operator": "Kakao",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "kakaotalk-scrap",
   "sources": "cloudflare",
   "docs_url": "https://developers.kakao.com/docs/latest/en/message-template/common\\#scrap"
  },
  {
   "name": "Kangaroo Bot",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "kangaroo bot",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "kb.dk_bot",
   "operator": "Netarkivet",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Academic Research",
   "respects_robots_txt": "unclear",
   "user_agents": "netarkivindsamling",
   "sources": "cloudflare",
   "docs_url": "https://www.kb.dk/en/find-materials/collections/netarkivet"
  },
  {
   "name": "KeldanNewsCrawler",
   "operator": "Keldan ehf.",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "keldannewscrawler",
   "sources": "cloudflare",
   "docs_url": "https://keldan.is"
  },
  {
   "name": "Kernel",
   "operator": "Kernel",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "kernel",
   "sources": "cloudflare",
   "docs_url": "https://www.kernel.sh/docs"
  },
  {
   "name": "Keys.so",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "keys.so",
   "sources": "matomo",
   "docs_url": "https://www.keys.so/"
  },
  {
   "name": "Kimi-User",
   "operator": "Moonshot AI that fetches web content on behalf of users interacting with Kimi",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "kimi-user",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "KimiBot",
   "operator": "Moonshot AI",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "kimibot",
   "sources": "cloudflare",
   "docs_url": "https://www.kimi.com/policies/kimi-crawlers"
  },
  {
   "name": "KlarnaBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "klarnabot",
   "sources": "matomo",
   "docs_url": "https://docs.klarna.com/klarna-bot/"
  },
  {
   "name": "KlaviyoAIBot",
   "operator": "Klaviyo",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "yes",
   "user_agents": "klaviyoaibot",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://www.klaviyo.com"
  },
  {
   "name": "KomodiaBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "komodiabot",
   "sources": "matomo",
   "docs_url": "http://www.komodia.com/newwiki/index.php/URL_server_crawler"
  },
  {
   "name": "Konturbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "konturbot",
   "sources": "matomo",
   "docs_url": "https://kontur.ru/"
  },
  {
   "name": "Kouio",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "kouio",
   "sources": "matomo",
   "docs_url": "http://kouio.com/"
  },
  {
   "name": "Kozmonavt",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "kozmonavt",
   "sources": "matomo",
   "docs_url": "https://kozmonavt.ml"
  },
  {
   "name": "KStandBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "kstandbot",
   "sources": "matomo",
   "docs_url": "https://url-classification.io/wiki/index.php?title=URL_server_crawler"
  },
  {
   "name": "KunatoCrawler",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "unknown",
   "category": "Undocumented AI Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "kunatocrawler",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "LAC IA Harvester",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "lac ia harvester",
   "sources": "matomo",
   "docs_url": "https://library-archives.canada.ca/eng/services/government-canada/web-social-media-preservation-program/Pages/web-archive.aspx"
  },
  {
   "name": "laion-huggingface-processor",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "laion-huggingface-processor",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "LAIONDownloader",
   "operator": "Large-scale Artificial Intelligence Open Network",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "no",
   "user_agents": "laiondownloader",
   "sources": "ai.robots.txt",
   "docs_url": "https://laion.ai/"
  },
  {
   "name": "Larbin web crawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "larbin web crawler",
   "sources": "matomo",
   "docs_url": "http://larbin.sourceforge.net"
  },
  {
   "name": "LCC",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "lcc",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://corpora.uni-leipzig.de/crawler_faq.html"
  },
  {
   "name": "leak.info",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "leak.info",
   "sources": "matomo",
   "docs_url": "http://www.leak.info/"
  },
  {
   "name": "LetSearch",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "letsearch",
   "sources": "matomo",
   "docs_url": "https://letsearch.ru/bots"
  },
  {
   "name": "Level9SearchBot",
   "operator": "Level 9",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "level9searchbot/",
   "sources": "cloudflare",
   "docs_url": "https://level9.com/"
  },
  {
   "name": "Library Of Congress Web Archiving",
   "operator": "United States Library of Congress",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Academic Research",
   "respects_robots_txt": "unclear",
   "user_agents": "https://www.loc.gov/programs/web-archiving/for-site-owners/",
   "sources": "cloudflare",
   "docs_url": "https://www.loc.gov/programs/web-archiving/for-site-owners"
  },
  {
   "name": "LightspeedSystemsCrawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "lightspeedsystemscrawler",
   "sources": "matomo",
   "docs_url": "https://www.lightspeedsystems.com/"
  },
  {
   "name": "LINER Bot",
   "operator": "Liner Bot",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "linerbot",
   "sources": "cloudflare",
   "docs_url": "https://docs.getliner.com/docs/linerbot"
  },
  {
   "name": "LinerBot",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "linerbot",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Linespider",
   "operator": "LINE Corporation",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "linespider",
   "sources": "cloudflare; matomo",
   "docs_url": "https://linecorp.com/en/"
  },
  {
   "name": "Linguee Bot",
   "operator": "Linguee",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "no",
   "user_agents": "linguee bot",
   "sources": "ai.robots.txt",
   "docs_url": "https://www.linguee.com"
  },
  {
   "name": "LinkBloom",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "linkbloom",
   "sources": "matomo",
   "docs_url": "https://hamidsoltani.com/linkbloom"
  },
  {
   "name": "LinkCheck",
   "operator": "Siteimprove",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "linkcheck by siteimprove",
   "sources": "cloudflare",
   "docs_url": "https://support.siteimprove.com/hc/en-gb/articles/115000402352-Siteimprove-s-Crawler-Frequently-Asked-Questions"
  },
  {
   "name": "LinkChecker Bot",
   "operator": "LinkChecker",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "yes",
   "user_agents": "linkchecker.pro",
   "sources": "cloudflare",
   "docs_url": "https://linkchecker.pro/robot/"
  },
  {
   "name": "LinkCheckerBot",
   "operator": "LinkChecker.Pro",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "yes",
   "user_agents": "linkcheckerbot",
   "sources": "cloudflare",
   "docs_url": "https://linkchecker.pro/robot"
  },
  {
   "name": "Linkdex Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "linkdex bot",
   "sources": "matomo",
   "docs_url": "http://www.linkdex.com/bots"
  },
  {
   "name": "LinkedInBot",
   "operator": "LinkedIn",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "linkedinbot",
   "sources": "cloudflare",
   "docs_url": "https://www.linkedin.com/"
  },
  {
   "name": "LinkpadBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "linkpadbot",
   "sources": "matomo",
   "docs_url": "https://www.linkpad.org/"
  },
  {
   "name": "LinksIndexerBot",
   "operator": "Links Indexer",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "linksindexerbot",
   "sources": "cloudflare",
   "docs_url": "https://linksindexer.com/bot"
  },
  {
   "name": "LinkupBot",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "unclear",
   "user_agents": "linkupbot",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "LinkWalker",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "linkwalker",
   "sources": "matomo",
   "docs_url": "https://www.phishlabs.com/"
  },
  {
   "name": "LiveJournal",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "livejournal",
   "sources": "matomo",
   "docs_url": "https://www.livejournal.com/"
  },
  {
   "name": "Loomly Bot",
   "operator": "Loomly",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "loomlybot",
   "sources": "cloudflare",
   "docs_url": "https://www.loomly.com"
  },
  {
   "name": "Lumar",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "lumar",
   "sources": "matomo",
   "docs_url": "https://www.lumar.io/spdr/"
  },
  {
   "name": "LumeWebScan",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "lumewebscan",
   "sources": "matomo",
   "docs_url": "https://lumeweaver.com/"
  },
  {
   "name": "LumtelBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "lumtelbot",
   "sources": "matomo",
   "docs_url": "https://lumtel.com"
  },
  {
   "name": "MaCoCu",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "macocu",
   "sources": "matomo",
   "docs_url": "https://www.clarin.si/info/macocu-massive-collection-and-curation-of-monolingual-and-bilingual-data/"
  },
  {
   "name": "MADBbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "madbbot",
   "sources": "matomo",
   "docs_url": "https://madb.zapto.org/bot.html"
  },
  {
   "name": "MagiBot",
   "operator": "Peak Labs",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "magibot",
   "sources": "cloudflare",
   "docs_url": "https://www.peak-labs.com/docs/en/magi/about-magibot"
  },
  {
   "name": "MagnetmeBot",
   "operator": "Magnet.me",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "magnet.me-web/",
   "sources": "cloudflare",
   "docs_url": "https://magnet.me/bot.html"
  },
  {
   "name": "Mail.Ru Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "mail.ru bot",
   "sources": "matomo",
   "docs_url": "http://help.mail.ru/webmaster/indexing/robots/types_robots"
  },
  {
   "name": "MailRUBot",
   "operator": "Mail Russia",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "yes",
   "user_agents": "mail.ru_bot",
   "sources": "cloudflare",
   "docs_url": "https://go.mail.ru/help/robots"
  },
  {
   "name": "Make.com",
   "operator": "Make.com",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "make.com",
   "sources": "cloudflare",
   "docs_url": "https://www.make.com"
  },
  {
   "name": "MakeMerryBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "makemerrybot",
   "sources": "matomo",
   "docs_url": "https://makemerry.app/bots"
  },
  {
   "name": "Manus Bot",
   "operator": "Manus",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "manus bot",
   "sources": "cloudflare",
   "docs_url": "https://manus.im/help/Manus-user"
  },
  {
   "name": "Manus-User",
   "operator": "Butterfly Effect, a company based in China",
   "kind": "ai",
   "purpose": "agent",
   "category": "AI Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "manus-user",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Marginalia",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "marginalia",
   "sources": "matomo",
   "docs_url": "https://www.marginalia.nu/marginalia-search/for-webmasters/"
  },
  {
   "name": "Marginalia Search",
   "operator": "Marginalia",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "search.marginalia.nu",
   "sources": "cloudflare",
   "docs_url": "https://search.marginalia.nu/"
  },
  {
   "name": "MariaDB/MySQL Knowledge Base",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "mariadb/mysql knowledge base",
   "sources": "matomo",
   "docs_url": "https://github.com/williamdes/mariadb-mysql-kbs"
  },
  {
   "name": "marketgoo",
   "operator": "marketgoo",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "yes",
   "user_agents": "marketgoo",
   "sources": "cloudflare; matomo",
   "docs_url": "https://www.marketgoo.com/"
  },
  {
   "name": "Mars Finder",
   "operator": "Mars Flag",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "yes",
   "user_agents": "(dbot)",
   "sources": "cloudflare",
   "docs_url": "https://www.marsflag.com/ja/marsfinder/"
  },
  {
   "name": "masscan",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "masscan",
   "sources": "matomo",
   "docs_url": "https://github.com/robertdavidgraham/masscan"
  },
  {
   "name": "masscan-ng",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "masscan-ng",
   "sources": "matomo",
   "docs_url": "https://github.com/bi-zone/masscan-ng"
  },
  {
   "name": "Meanpath Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "meanpath bot",
   "sources": "matomo",
   "docs_url": "http://www.meanpath.com/meanpathbot.html"
  },
  {
   "name": "MediaMonitoringBot",
   "operator": "MediaMonitoringBot",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "yes",
   "user_agents": "mediamonitoringbot/",
   "sources": "cloudflare",
   "docs_url": "https://mediamonitoringbot.com/"
  },
  {
   "name": "Mediatoolkit Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "mediatoolkit bot",
   "sources": "matomo",
   "docs_url": "https://mediatoolkit.com"
  },
  {
   "name": "Mediatoolkitbot",
   "operator": "Determ",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "yes",
   "user_agents": "mediatoolkitbot",
   "sources": "cloudflare",
   "docs_url": "https://www.determ.com/"
  },
  {
   "name": "Mediumbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "mediumbot",
   "sources": "matomo",
   "docs_url": "https://medium.com/"
  },
  {
   "name": "MegaIndex",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "megaindex",
   "sources": "matomo",
   "docs_url": "https://megaindex.com/crawler"
  },
  {
   "name": "MelonMesa Bot",
   "operator": "MelonMesa",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "centcombot",
   "sources": "cloudflare",
   "docs_url": "https://centcom.melonmesa.com/"
  },
  {
   "name": "MeltwaterNews",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "meltwaternews",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Meta-ExternalAds",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "Social Media Agent",
   "respects_robots_txt": "unclear",
   "user_agents": "meta-externalads",
   "sources": "matomo",
   "docs_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers"
  },
  {
   "name": "meta-externalagent",
   "operator": "Meta",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "meta-externalagent",
   "sources": "ai.robots.txt; cloudflare; matomo",
   "docs_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers"
  },
  {
   "name": "meta-externalfetcher",
   "operator": "Meta",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "meta-externalfetcher; meta-externalfetcher/",
   "sources": "ai.robots.txt; cloudflare; matomo",
   "docs_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/"
  },
  {
   "name": "meta-webindexer",
   "operator": "Meta",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "meta-webindexer; meta-webindexer/1.1",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/"
  },
  {
   "name": "MetaInspector",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "metainspector",
   "sources": "matomo",
   "docs_url": "https://github.com/jaimeiniesta/metainspector"
  },
  {
   "name": "MetaJobBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "metajobbot",
   "sources": "matomo",
   "docs_url": "http://www.metajob.at/the/crawler"
  },
  {
   "name": "MicroAdBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "microadbot",
   "sources": "matomo",
   "docs_url": "https://www.microad.co.jp/"
  },
  {
   "name": "Microsoft Preview",
   "operator": "Microsoft",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "microsoftpreview/",
   "sources": "cloudflare",
   "docs_url": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0"
  },
  {
   "name": "MidasNetworkBot",
   "operator": "Midas Network d.o.o.",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "yes",
   "user_agents": "midasnetworkbot",
   "sources": "cloudflare",
   "docs_url": "https://midas-network.com"
  },
  {
   "name": "MirrorWebCrawler",
   "operator": "MirrorWeb Ltd",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "mirrorweb",
   "sources": "cloudflare",
   "docs_url": "https://www.mirrorweb.com/"
  },
  {
   "name": "MistralAI-User",
   "operator": "Mistral",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "mistralai-user",
   "sources": "ai.robots.txt; cloudflare; matomo",
   "docs_url": "https://docs.mistral.ai/robots/#mistralai-user"
  },
  {
   "name": "MistralAI-User/1.0",
   "operator": "Mistral AI",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "yes",
   "user_agents": "mistralai-user/1.0",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Mixnode",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "mixnode",
   "sources": "matomo",
   "docs_url": "https://www.mixnode.com/"
  },
  {
   "name": "MixRank Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "mixrank bot",
   "sources": "matomo",
   "docs_url": "http://mixrank.com"
  },
  {
   "name": "MJ12 Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "mj12 bot",
   "sources": "matomo",
   "docs_url": "http://majestic12.co.uk/bot.php"
  },
  {
   "name": "MJ12Bot",
   "operator": "Majestic",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "mj12bot",
   "sources": "cloudflare",
   "docs_url": "https://majestic.com/"
  },
  {
   "name": "Mnogosearch",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "mnogosearch",
   "sources": "matomo",
   "docs_url": "http://www.mnogosearch.org/"
  },
  {
   "name": "Mojeek",
   "operator": "Mojeek",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "mojeekbot",
   "sources": "cloudflare",
   "docs_url": "https://www.mojeek.com/bot.html"
  },
  {
   "name": "MojeekBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "mojeekbot",
   "sources": "matomo",
   "docs_url": "http://www.mojeek.com/bot.html"
  },
  {
   "name": "Monitor Backlinks",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "monitor backlinks",
   "sources": "matomo",
   "docs_url": "https://www.seoptimer.com/monitor-backlinks/"
  },
  {
   "name": "MonitoRSS",
   "operator": "MonitoRSS",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "yes",
   "user_agents": "monitorss/",
   "sources": "cloudflare",
   "docs_url": "https://monitorss.xyz"
  },
  {
   "name": "Monsidobot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "monsidobot",
   "sources": "matomo",
   "docs_url": "https://monsido.com/bot-html"
  },
  {
   "name": "MoodleBot Linkchecker",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "moodlebot linkchecker",
   "sources": "matomo",
   "docs_url": "https://docs.moodle.org/en/Usage"
  },
  {
   "name": "Morningscore Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "morningscore bot",
   "sources": "matomo",
   "docs_url": "https://morningscore.io/"
  },
  {
   "name": "MotoMinerBot",
   "operator": "Motominer",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "motominerbot",
   "sources": "cloudflare",
   "docs_url": "https://motominer.com/Bot"
  },
  {
   "name": "Moz dotbot",
   "operator": "Moz",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "dotbot",
   "sources": "cloudflare",
   "docs_url": "https://moz.com/link-explorer"
  },
  {
   "name": "Moz rogerbot",
   "operator": "Moz",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "yes",
   "user_agents": "rogerbot-crawler",
   "sources": "cloudflare",
   "docs_url": "https://moz.com/help/moz-procedures/crawlers/rogerbot"
  },
  {
   "name": "Mozilla-Tabstack",
   "operator": "Mozilla",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "yes",
   "user_agents": "mozilla-tabstack",
   "sources": "ai.robots.txt",
   "docs_url": "https://docs.tabstack.ai/trust/controlling-access"
  },
  {
   "name": "MRGbot",
   "operator": "MRG Web Services srl",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "yes",
   "user_agents": "mrgbot/",
   "sources": "cloudflare",
   "docs_url": "https://www.mrg.ro"
  },
  {
   "name": "MSN",
   "operator": "Microsoft",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "msnbot/",
   "sources": "cloudflare",
   "docs_url": "https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0"
  },
  {
   "name": "MTRobot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "mtrobot",
   "sources": "matomo",
   "docs_url": "https://metrics-tools.de/robot.html"
  },
  {
   "name": "Muck Rack Scraper",
   "operator": "Muck Rack",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "muckrack.com",
   "sources": "cloudflare",
   "docs_url": "https://muckrack.com/"
  },
  {
   "name": "MuckRack",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "muckrack",
   "sources": "matomo",
   "docs_url": "https://muckrack.com/"
  },
  {
   "name": "MuscatFerret",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "muscatferret",
   "sources": "matomo",
   "docs_url": "http://www.webtop.com/"
  },
  {
   "name": "MyCentralAIScraperBot",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI data scraper",
   "respects_robots_txt": "unclear",
   "user_agents": "mycentralaiscraperbot",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "NagetBot",
   "operator": "Naget Inc (founded by Chris Samarinas, headquarter in Amherst, Massachusetts)",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI data scraper",
   "respects_robots_txt": "unclear",
   "user_agents": "nagetbot",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "najdu.s.holubem.eu",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "najdu.s.holubem.eu",
   "sources": "matomo",
   "docs_url": "https://najdu.s.holubem.eu/"
  },
  {
   "name": "NalezenCzBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "nalezenczbot",
   "sources": "matomo",
   "docs_url": "http://www.nalezen.cz/about-crawler"
  },
  {
   "name": "NameProtectBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "nameprotectbot",
   "sources": "matomo",
   "docs_url": "https://www.cscglobal.com/cscglobal/home/"
  },
  {
   "name": "Nava Labs ASP (Dev)",
   "operator": "Nava Labs",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "nava/",
   "sources": "cloudflare",
   "docs_url": "https://dev.labs-asp.navateam.com/bot-disclosure"
  },
  {
   "name": "Navu",
   "operator": "HivePoint, Inc.",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "navubot",
   "sources": "cloudflare",
   "docs_url": "https://navu.co"
  },
  {
   "name": "nbertaupete95",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "nbertaupete95",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Neevabot",
   "operator": "Neeva",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "neevabot",
   "sources": "cloudflare; matomo",
   "docs_url": "https://neeva.com/neevabot"
  },
  {
   "name": "Netcraft Survey Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "netcraft survey bot",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "netEstate",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "netestate",
   "sources": "matomo",
   "docs_url": "http://www.website-datenbank.de/Impressum"
  },
  {
   "name": "netEstate Imprint Crawler",
   "operator": "netEstate",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "netestate imprint crawler; netestate ne crawler",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "http://www.website-datenbank.de/"
  },
  {
   "name": "Netpeak Checker",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "netpeak checker",
   "sources": "matomo",
   "docs_url": "https://netpeaksoftware.com/checker"
  },
  {
   "name": "NetTrack",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "nettrack",
   "sources": "matomo",
   "docs_url": "https://web.archive.org/web/20160607151934/https://nettrack.info/"
  },
  {
   "name": "Netvibes",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "netvibes",
   "sources": "matomo",
   "docs_url": "http://www.netvibes.com/"
  },
  {
   "name": "NETZZAPPEN",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "netzzappen",
   "sources": "matomo",
   "docs_url": "https://www.netzzappen.com/"
  },
  {
   "name": "New York Times Newsgathering",
   "operator": "The New York Times",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "scraping@nytimes\\.com",
   "sources": "cloudflare",
   "docs_url": "https://int.nyt.com/assets/scraping.json"
  },
  {
   "name": "newsai",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI data scraper",
   "respects_robots_txt": "unclear",
   "user_agents": "newsai",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "NewsBank",
   "operator": "NewsBank",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Archiver",
   "respects_robots_txt": "unclear",
   "user_agents": "newsbank",
   "sources": "cloudflare",
   "docs_url": "https://www.newsbank.com/"
  },
  {
   "name": "NewsBlur",
   "operator": "NewsBlur",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "newsblur",
   "sources": "cloudflare; matomo",
   "docs_url": "https://newsblur.com/"
  },
  {
   "name": "NewsGator",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "newsgator",
   "sources": "matomo",
   "docs_url": "http://www.newsgator.com"
  },
  {
   "name": "Newslitbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "newslitbot",
   "sources": "matomo",
   "docs_url": "https://www.newslit.co/"
  },
  {
   "name": "NewsNow",
   "operator": "NewsNowUK",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "newsnow",
   "sources": "cloudflare",
   "docs_url": "https://www.newsnow.co.uk/h/"
  },
  {
   "name": "NiceCrawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "nicecrawler",
   "sources": "matomo",
   "docs_url": "https://www.nicecrawler.com/"
  },
  {
   "name": "NitroBot",
   "operator": "NitroPack Ltd",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "nitro-",
   "sources": "cloudflare",
   "docs_url": "https://nitropack.io/"
  },
  {
   "name": "NLCrawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "nlcrawler",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Nooshub",
   "operator": "Nooshub",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "nooshub",
   "sources": "cloudflare",
   "docs_url": "https://www.nooshub.com"
  },
  {
   "name": "Nostra",
   "operator": "Nostra",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "nostra",
   "sources": "cloudflare",
   "docs_url": "https://www.nostra.ai/edge-optimization"
  },
  {
   "name": "Notabot",
   "operator": "Helpfeel",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "notabot",
   "sources": "cloudflare",
   "docs_url": "https://corp.helpfeel.com/ja/home"
  },
  {
   "name": "NotebookLM",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "notebooklm",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "NovaAct",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "agent",
   "category": "AI Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "novaact",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Novellum AI Crawl",
   "operator": "Novellum",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "novellum",
   "sources": "cloudflare",
   "docs_url": "https://crawl.corp.novellum.ai/docs"
  },
  {
   "name": "Nutch-based Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "nutch-based bot",
   "sources": "matomo",
   "docs_url": "https://nutch.apache.org"
  },
  {
   "name": "Nuzzel",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "nuzzel",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "OAI-SearchBot",
   "operator": "OpenAI",
   "kind": "ai",
   "purpose": "answer",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "yes",
   "user_agents": "oai-searchbot",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://openai.com"
  },
  {
   "name": "oBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "obot",
   "sources": "matomo",
   "docs_url": "https://www.xforce-security.com/crawler/"
  },
  {
   "name": "Observer",
   "operator": "Obsrvr",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "yes",
   "user_agents": "obsrvr.net",
   "sources": "cloudflare",
   "docs_url": "https://obsrvr.net/about"
  },
  {
   "name": "Odnoklassniki Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "odnoklassniki bot",
   "sources": "matomo",
   "docs_url": "https://odnoklassniki.ru"
  },
  {
   "name": "OI-Crawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "oi-crawler",
   "sources": "matomo",
   "docs_url": "https://openintel.nl/webcrawl/"
  },
  {
   "name": "omgili",
   "operator": "Webz.io",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "omgili",
   "sources": "ai.robots.txt",
   "docs_url": "https://webz.io/"
  },
  {
   "name": "omgilibot",
   "operator": "Webz.io",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "yes",
   "user_agents": "omgilibot",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://webz.io/"
  },
  {
   "name": "OMIM Litrack",
   "operator": "Johns Hopkins University",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Academic Research",
   "respects_robots_txt": "unclear",
   "user_agents": "omim-litrack",
   "sources": "cloudflare",
   "docs_url": "https://omim.org"
  },
  {
   "name": "Onalytica",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "onalytica",
   "sources": "matomo",
   "docs_url": "https://www.airslate.com/bot/explore/onalytica-bot"
  },
  {
   "name": "OnCrawl",
   "operator": "Oncrawl",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "www.oncrawl.com",
   "sources": "cloudflare",
   "docs_url": "https://www.oncrawl.com/"
  },
  {
   "name": "Online Webceo Bot",
   "operator": "Online Webceo",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "online-webceo-bot",
   "sources": "cloudflare",
   "docs_url": "https://www.webceo.com/webceo-bots.htm"
  },
  {
   "name": "OpenAI",
   "operator": "OpenAI",
   "kind": "ai",
   "purpose": "unknown",
   "category": "Undocumented AI Agents",
   "respects_robots_txt": "yes",
   "user_agents": "openai",
   "sources": "ai.robots.txt",
   "docs_url": "https://openai.com"
  },
  {
   "name": "opencode",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "agent",
   "category": "AI Coding Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "opencode",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Opengraph Bot",
   "operator": "Opengraph",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "opengraph.io/",
   "sources": "cloudflare",
   "docs_url": "https://www.opengraph.io/"
  },
  {
   "name": "OpenGraph.io",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "opengraph.io",
   "sources": "matomo",
   "docs_url": "https://www.opengraph.io"
  },
  {
   "name": "Openindex Spider",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "openindex spider",
   "sources": "matomo",
   "docs_url": "https://www.openindex.io/about-our-spider/"
  },
  {
   "name": "OpenLinkProfiler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "openlinkprofiler",
   "sources": "matomo",
   "docs_url": "http://openlinkprofiler.org/bot"
  },
  {
   "name": "OpenRobotsTXT",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "openrobotstxt",
   "sources": "matomo",
   "docs_url": "https://openrobotstxt.org/ua"
  },
  {
   "name": "OpenRSS",
   "operator": "OpenRSS",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "openrss",
   "sources": "cloudflare",
   "docs_url": "https://openrss.org"
  },
  {
   "name": "OpenWebSpider",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "openwebspider",
   "sources": "matomo",
   "docs_url": "http://www.openwebspider.org"
  },
  {
   "name": "Operator",
   "operator": "OpenAI",
   "kind": "ai",
   "purpose": "agent",
   "category": "AI Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "operator",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Orange Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "orange bot",
   "sources": "matomo",
   "docs_url": "http://lemoteur.orange.fr"
  },
  {
   "name": "Orlo Link Preview",
   "operator": "Orlo",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "orlo-linkpreview/",
   "sources": "cloudflare",
   "docs_url": "https://orlo.tech"
  },
  {
   "name": "Oseox",
   "operator": "Oseox",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "yes",
   "user_agents": "oseox",
   "sources": "cloudflare",
   "docs_url": "https://tool.oseox.com/"
  },
  {
   "name": "OSZKbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "oszkbot",
   "sources": "matomo",
   "docs_url": "http://mekosztaly.oszk.hu/mia/"
  },
  {
   "name": "OurFamilyWizard",
   "operator": "OurFamilyWizard",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "cozi-icalendar-feedreader",
   "sources": "cloudflare",
   "docs_url": "https://ourfamilywizard.com"
  },
  {
   "name": "Outbrain",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "outbrain",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Overcast",
   "operator": "Overcast Radio",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "overcast/1.0 podcast sync",
   "sources": "cloudflare",
   "docs_url": "https://overcast.fm/"
  },
  {
   "name": "OWLer",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "owler",
   "sources": "matomo",
   "docs_url": "https://openwebsearch.eu/owler/"
  },
  {
   "name": "Ozon Web Grabber",
   "operator": "Ozon",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "odklbot/",
   "sources": "cloudflare",
   "docs_url": "https://ok.ru"
  },
  {
   "name": "Pageburst",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "pageburst",
   "sources": "matomo",
   "docs_url": "https://pageburstls.elsevier.com/"
  },
  {
   "name": "Pagefreezer Website Archiving",
   "operator": "Pagefreezer",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Archiver",
   "respects_robots_txt": "unclear",
   "user_agents": "pagefreezer",
   "sources": "cloudflare",
   "docs_url": "https://www.pagefreezer.com/"
  },
  {
   "name": "PagePeeker",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "pagepeeker",
   "sources": "matomo",
   "docs_url": "https://pagepeeker.com/robots/"
  },
  {
   "name": "PageThing",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "pagething",
   "sources": "matomo",
   "docs_url": "https://www.pagething.com/"
  },
  {
   "name": "Pandalytics",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "pandalytics",
   "sources": "matomo",
   "docs_url": "https://www.domainsbot.com/business-intelligence/"
  },
  {
   "name": "PanguBot",
   "operator": "the Chinese company Huawei",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "pangubot",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Panscient",
   "operator": "Panscient",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "panscient",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://panscient.com"
  },
  {
   "name": "panscient.com",
   "operator": "Panscient",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "panscient.com",
   "sources": "ai.robots.txt",
   "docs_url": "https://panscient.com"
  },
  {
   "name": "PaperLiBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "paperlibot",
   "sources": "matomo",
   "docs_url": "http://support.paper.li/entries/20023257-what-is-paper-li"
  },
  {
   "name": "Paqlebot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "paqlebot",
   "sources": "matomo",
   "docs_url": "https://www.paqle.dk/about/paqlebot"
  },
  {
   "name": "parse.ly",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "parse.ly",
   "sources": "matomo",
   "docs_url": "https://www.parse.ly/help/integration/crawler"
  },
  {
   "name": "ParticleNewsBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "particlenewsbot",
   "sources": "cloudflare",
   "docs_url": "https://particle.news/bot"
  },
  {
   "name": "Payhawk Invoice Fetching Agent",
   "operator": "Payhawk",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "payhawk invoice fetching agent",
   "sources": "cloudflare",
   "docs_url": "https://payhawk.com/help/overview-of-the-financial-controller-agent"
  },
  {
   "name": "payroll-bot",
   "operator": "ADP",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "adpresearchbot/",
   "sources": "cloudflare",
   "docs_url": "https://payroll-bot.adp.com/docs"
  },
  {
   "name": "Peer39",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "peer39",
   "sources": "matomo",
   "docs_url": "https://www.peer39.com/crawler-notice"
  },
  {
   "name": "Perplexity-User",
   "operator": "Perplexity",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "no",
   "user_agents": "perplexity-user",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://www.perplexity.ai/"
  },
  {
   "name": "PerplexityBot",
   "operator": "Perplexity",
   "kind": "ai",
   "purpose": "answer",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "yes",
   "user_agents": "perplexitybot",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://www.perplexity.ai/"
  },
  {
   "name": "Petal Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "petal bot",
   "sources": "matomo",
   "docs_url": "https://aspiegel.com/petalbot"
  },
  {
   "name": "PetalBot",
   "operator": "Huawei",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "yes",
   "user_agents": "petalbot",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://huawei.com/"
  },
  {
   "name": "PhindBot",
   "operator": "phind",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "phindbot",
   "sources": "ai.robots.txt",
   "docs_url": "https://www.phind.com/"
  },
  {
   "name": "Picsearch bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "picsearch bot",
   "sources": "matomo",
   "docs_url": "http://www.picsearch.com/bot.html"
  },
  {
   "name": "Pigafetta",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "pigafetta",
   "sources": "matomo",
   "docs_url": "https://visual-seo.com/Pigafetta-Bot"
  },
  {
   "name": "Pinterest",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "pinterest",
   "sources": "matomo",
   "docs_url": "https://help.pinterest.com/en/business/article/pinterest-crawler"
  },
  {
   "name": "PinterestBot",
   "operator": "Pinterest",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "pinterest.com/",
   "sources": "cloudflare",
   "docs_url": "https://help.pinterest.com/en/business/article/pinterest-crawler"
  },
  {
   "name": "PiplBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "piplbot",
   "sources": "matomo",
   "docs_url": "http://www.pipl.com/bot"
  },
  {
   "name": "Platebreaker",
   "operator": "Platebreaker",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "platebreaker",
   "sources": "cloudflare",
   "docs_url": "https://www.platebreaker.com"
  },
  {
   "name": "PlayStore-Google",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "playstore-google",
   "sources": "matomo",
   "docs_url": "https://play.google.com/"
  },
  {
   "name": "Plukkie",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "plukkie",
   "sources": "matomo",
   "docs_url": "http://www.botje.com/plukkie.htm"
  },
  {
   "name": "Pocket Casts Feed Parser",
   "operator": "Pocket Casts",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "pocket casts feed parser",
   "sources": "cloudflare",
   "docs_url": "https://support.pocketcasts.com/knowledge-base/pocket-casts-feed-parser/"
  },
  {
   "name": "Podroll Analyzer",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "podroll analyzer",
   "sources": "matomo",
   "docs_url": "https://podroll.fm"
  },
  {
   "name": "Poggio-Citations",
   "operator": "Poggio, a company that provides AI sales enablement tools for creating tailored narratives, business cases, and account plan…",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "poggio-citations",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Pompos",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "pompos",
   "sources": "matomo",
   "docs_url": "https://web.archive.org/web/20090124084536/http://dir.com/pompos.html"
  },
  {
   "name": "Poseidon Research Crawler",
   "operator": "Poseidon Research",
   "kind": "ai",
   "purpose": "search",
   "category": "AI research crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "poseidon research crawler",
   "sources": "ai.robots.txt",
   "docs_url": "https://www.poseidonresearch.com"
  },
  {
   "name": "Potions",
   "operator": "Potions",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "potions/",
   "sources": "cloudflare",
   "docs_url": "https://get-potions.com"
  },
  {
   "name": "prerender",
   "operator": "Prerender, LLC",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "prerender",
   "sources": "cloudflare; matomo",
   "docs_url": "https://docs.prerender.io/article/33-overview-of-prerender-crawlers"
  },
  {
   "name": "Presearch",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "presearch",
   "sources": "matomo",
   "docs_url": "https://presearch.com/"
  },
  {
   "name": "PressEngine Bot",
   "operator": "PressEngine",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "yes",
   "user_agents": "pressenginebot",
   "sources": "cloudflare",
   "docs_url": "https://www.pressengine.net"
  },
  {
   "name": "Pricey",
   "operator": "Wilico",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "pricey",
   "sources": "cloudflare",
   "docs_url": "https://www.pricey.jp"
  },
  {
   "name": "PritTorrent",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "prittorrent",
   "sources": "matomo",
   "docs_url": "https://github.com/astro/prittorrent"
  },
  {
   "name": "Project Patchwatch",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "project patchwatch",
   "sources": "matomo",
   "docs_url": "http://66.240.192.82/"
  },
  {
   "name": "Project Resonance",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "project resonance",
   "sources": "matomo",
   "docs_url": "https://project-resonance.com/"
  },
  {
   "name": "Protopage",
   "operator": "Protopage Ltd",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "protopage",
   "sources": "cloudflare",
   "docs_url": "https://www.protopage.com/"
  },
  {
   "name": "QAtechBot",
   "operator": "QA.tech",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "qatechbot",
   "sources": "cloudflare",
   "docs_url": "https://qa.tech"
  },
  {
   "name": "QualifiedBot",
   "operator": "Qualified",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "qualifiedbot",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://www.qualified.com"
  },
  {
   "name": "Quantcast",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "quantcast",
   "sources": "matomo",
   "docs_url": "https://www.quantcast.com/bot/"
  },
  {
   "name": "Quartr Crawler",
   "operator": "Quartr",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "quartr crawler",
   "sources": "cloudflare",
   "docs_url": "https://quartr.com"
  },
  {
   "name": "Querit-SearchBot",
   "operator": "Querit that indexes web content for their search API service, which is designed to provide real-time search results for larg…",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "unclear",
   "user_agents": "querit-searchbot",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "QueritBot",
   "operator": "Querit, a company providing a search API for large language model integration",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "unclear",
   "user_agents": "queritbot",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "QuerySeekerSpider",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "queryseekerspider",
   "sources": "matomo",
   "docs_url": "http://queryseeker.com/bot.html"
  },
  {
   "name": "QuillBot",
   "operator": "Quillbot",
   "kind": "ai",
   "purpose": "unknown",
   "category": "Undocumented AI Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "quillbot",
   "sources": "ai.robots.txt",
   "docs_url": "https://quillbot.com"
  },
  {
   "name": "quillbot.com",
   "operator": "Quillbot",
   "kind": "ai",
   "purpose": "unknown",
   "category": "Undocumented AI Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "quillbot.com",
   "sources": "ai.robots.txt",
   "docs_url": "https://quillbot.com"
  },
  {
   "name": "Quora Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "quora bot",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Quora Link Preview",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "quora link preview",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Qwantbot",
   "operator": "Qwant",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "qwantbot",
   "sources": "cloudflare; matomo",
   "docs_url": "https://help.qwant.com/bot/"
  },
  {
   "name": "Rainmeter",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "rainmeter",
   "sources": "matomo",
   "docs_url": "https://www.rainmeter.net"
  },
  {
   "name": "Rakuten Image extraction bot",
   "operator": "Rakuten",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "rakutenusabot-image",
   "sources": "cloudflare",
   "docs_url": "https://product-image.ebates.com/item-gsp/rakutenusabot.html"
  },
  {
   "name": "RamblerMail Image Proxy",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "ramblermail image proxy",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "RavenCrawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "ravencrawler",
   "sources": "matomo",
   "docs_url": "https://raventools.com/site-auditor/"
  },
  {
   "name": "RDTvlokipBot",
   "operator": "Theo CHARLET",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "yes",
   "user_agents": "rdtvlokipbot",
   "sources": "cloudflare",
   "docs_url": "https://search.rdtvlokip.fr"
  },
  {
   "name": "RedekenBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "redekenbot",
   "sources": "matomo",
   "docs_url": "https://www.redeken.com/en/help/bot.html"
  },
  {
   "name": "ReqBin",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "reqbin",
   "sources": "matomo",
   "docs_url": "https://reqbin.com/curl"
  },
  {
   "name": "Research JLU",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "research jlu",
   "sources": "matomo",
   "docs_url": "https://www.uni-giessen.de/en/research"
  },
  {
   "name": "Research Scan",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "research scan",
   "sources": "matomo",
   "docs_url": "http://researchscan.comsys.rwth-aachen.de/"
  },
  {
   "name": "Retool",
   "operator": "Retool",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "retool/",
   "sources": "cloudflare",
   "docs_url": "https://docs.tryretool.com/docs/apis"
  },
  {
   "name": "Revvim",
   "operator": "Revvim",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "revvimgort",
   "sources": "cloudflare",
   "docs_url": "revvim.com"
  },
  {
   "name": "Rogerbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "rogerbot",
   "sources": "matomo",
   "docs_url": "http://moz.com/help/pro/what-is-rogerbot-"
  },
  {
   "name": "ROI Hunter",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "roi hunter",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "RSiteAuditor",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "rsiteauditor",
   "sources": "matomo",
   "docs_url": "https://help.agencyanalytics.com/en/articles/3469586-allow-our-site-auditor-via-robots-txt-and-firewall"
  },
  {
   "name": "RSS API",
   "operator": "RSS API (by Tibush GmbH)",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "rssapi/",
   "sources": "cloudflare",
   "docs_url": "https://rssapi.net/"
  },
  {
   "name": "rss2tg_bot",
   "operator": "Yellow Rubber Duck Consulting",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "rss2tg bot",
   "sources": "cloudflare",
   "docs_url": "https://rss2tg.duck.consulting/"
  },
  {
   "name": "RyeBot",
   "operator": "Rye",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "ryebot",
   "sources": "cloudflare",
   "docs_url": "https://docs.rye.com/api-v2-experimental/ryebot"
  },
  {
   "name": "Ryowl",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "ryowl",
   "sources": "matomo",
   "docs_url": "https://ryowl.org"
  },
  {
   "name": "SabsimBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sabsimbot",
   "sources": "matomo",
   "docs_url": "https://sabsim.com"
  },
  {
   "name": "SafeDNSBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "safednsbot",
   "sources": "matomo",
   "docs_url": "https://www.safedns.com/searchbot"
  },
  {
   "name": "Sandoba//Crawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sandoba//crawler",
   "sources": "matomo",
   "docs_url": "https://www.sandoba.com/en/crawler/"
  },
  {
   "name": "SBIder",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sbider",
   "sources": "matomo",
   "docs_url": "https://www.sitesell.com/sbider.html"
  },
  {
   "name": "SBIntuitionsBot",
   "operator": "SB Intuitions",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "yes",
   "user_agents": "sbintuitionsbot",
   "sources": "ai.robots.txt",
   "docs_url": "https://www.sbintuitions.co.jp/en/"
  },
  {
   "name": "Scamadviser External Hit",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "scamadviser external hit",
   "sources": "matomo",
   "docs_url": "https://www.scamadviser.com/"
  },
  {
   "name": "Scour RSS Bot",
   "operator": "Scour",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "scour.ing",
   "sources": "cloudflare",
   "docs_url": "https://scour.ing/bot"
  },
  {
   "name": "Scraping Robot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "scraping robot",
   "sources": "matomo",
   "docs_url": "https://scrapingrobot.com/"
  },
  {
   "name": "Scrapy",
   "operator": "Zyte",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "scrapy",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://www.zyte.com"
  },
  {
   "name": "Screaming Frog SEO Spider",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "screaming frog seo spider",
   "sources": "matomo",
   "docs_url": "http://www.screamingfrog.co.uk/seo-spider"
  },
  {
   "name": "ScreenerBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "screenerbot",
   "sources": "matomo",
   "docs_url": "http://www.screenerbot.com"
  },
  {
   "name": "Screpy",
   "operator": "screpy.com",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "screpy",
   "sources": "cloudflare",
   "docs_url": "https://screpy.com/faq/"
  },
  {
   "name": "SearchAtlas Bot",
   "operator": "Search Atlas",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "searchatlas bot",
   "sources": "cloudflare",
   "docs_url": "https://help.searchatlas.com/en/"
  },
  {
   "name": "Seekport",
   "operator": "SISTRIX",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "seekport; seekportbot",
   "sources": "cloudflare; matomo",
   "docs_url": "https://www.seekport.com/"
  },
  {
   "name": "Selectika AI",
   "operator": "Selectika",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "selectikascraper",
   "sources": "cloudflare",
   "docs_url": "https://selectika.com"
  },
  {
   "name": "Sellers.Guide",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sellers.guide",
   "sources": "matomo",
   "docs_url": "https://sellers.guide/"
  },
  {
   "name": "semaltbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "semaltbot",
   "sources": "matomo",
   "docs_url": "https://semalt.net/"
  },
  {
   "name": "Semantic Scholar Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "semantic scholar bot",
   "sources": "matomo",
   "docs_url": "https://www.semanticscholar.org/crawler"
  },
  {
   "name": "Semrushbot",
   "operator": "Semrush",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "semrushbot; semrushbot-sa",
   "sources": "cloudflare; matomo",
   "docs_url": "https://www.semrush.com/bot/"
  },
  {
   "name": "SemrushBot-OCOB",
   "operator": "Semrush",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "yes",
   "user_agents": "semrushbot-ocob",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://www.semrush.com/"
  },
  {
   "name": "SemrushBot-SWA",
   "operator": "Semrush",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "yes",
   "user_agents": "semrushbot-swa",
   "sources": "ai.robots.txt",
   "docs_url": "https://www.semrush.com/"
  },
  {
   "name": "SemrushBotBacklinkAudit",
   "operator": "Semrush",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "semrushbot-ba",
   "sources": "cloudflare",
   "docs_url": "https://www.semrush.com/backlink_audit/"
  },
  {
   "name": "SemrushBotBacklinks",
   "operator": "Semrush",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "semrushbot/",
   "sources": "cloudflare",
   "docs_url": "https://www.semrush.com/bot/"
  },
  {
   "name": "SemrushBotLinkBuilding",
   "operator": "Semrush",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "semrushbot;",
   "sources": "cloudflare",
   "docs_url": "https://www.semrush.com/link_building/"
  },
  {
   "name": "SemrushBotSI",
   "operator": "Semrush",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "semrushbot-si",
   "sources": "cloudflare",
   "docs_url": "https://www.semrush.com/on-page-seo-checker/ ; https://www.semrush.com/seo-content-template/ ; https://www.semrush.com/topic-research/"
  },
  {
   "name": "SemrushBotSwa",
   "operator": "Semrush",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "semrushbot-swa/",
   "sources": "cloudflare",
   "docs_url": "https://www.semrush.com/swa/"
  },
  {
   "name": "Senuto",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "senuto",
   "sources": "matomo",
   "docs_url": "https://www.senuto.com/"
  },
  {
   "name": "SEO Audit Check Bot",
   "operator": "SEO Audit Check",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "seo-audit-check-bot",
   "sources": "cloudflare",
   "docs_url": "https://www.webceo.com/webceo-bots.htm"
  },
  {
   "name": "seo4ajax",
   "operator": "Capsule Code",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "seo4ajax.com",
   "sources": "cloudflare",
   "docs_url": "https://www.seo4ajax.com"
  },
  {
   "name": "Seobility",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "seobility",
   "sources": "matomo",
   "docs_url": "https://www.seobility.net/en/faq/?category=crawling#!aboutourbot"
  },
  {
   "name": "SeobilityBot",
   "operator": "Seobility",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "seobilitybot",
   "sources": "cloudflare",
   "docs_url": "https://www.seobility.net/sites/bot.html"
  },
  {
   "name": "SEOENGBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "seoengbot",
   "sources": "matomo",
   "docs_url": "http://www.seoengine.com/seoengbot.htm"
  },
  {
   "name": "SEOkicks",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "seokicks",
   "sources": "matomo",
   "docs_url": "https://www.seokicks.de/robot.html"
  },
  {
   "name": "SeolytBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "seolytbot",
   "sources": "matomo",
   "docs_url": "https://seolyt.com/"
  },
  {
   "name": "Seoscanners.net",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "seoscanners.net",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "SERankingBacklinksBot",
   "operator": "SE Ranking",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "serankingbacklinksbot",
   "sources": "cloudflare; matomo",
   "docs_url": "https://seranking.com/backlinks-crawler"
  },
  {
   "name": "Serendeputy Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "serendeputy bot",
   "sources": "matomo",
   "docs_url": "https://serendeputy.com/about/serendeputy-bot"
  },
  {
   "name": "SerpstatBot",
   "operator": "Serpstat",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "serpstatbot",
   "sources": "cloudflare; matomo",
   "docs_url": "https://serpstatbot.com/"
  },
  {
   "name": "ServerHunterSpider",
   "operator": "Server Hunter",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "serverhunterspider",
   "sources": "cloudflare",
   "docs_url": "https://www.serverhunter.com/"
  },
  {
   "name": "Seznam",
   "operator": "Seznam",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "seznambot",
   "sources": "cloudflare",
   "docs_url": "https://napoveda.seznam.cz/en/seznambot-crawler/"
  },
  {
   "name": "Seznam Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "seznam bot",
   "sources": "matomo",
   "docs_url": "http://www.mapy.cz/cz/seznambot.html"
  },
  {
   "name": "Seznam Email Proxy",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "seznam email proxy",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Seznam Zbozi.cz",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "seznam zbozi.cz",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "sfFeedReader",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "sffeedreader",
   "sources": "matomo",
   "docs_url": "https://github.com/diem-project/sfFeed2Plugin"
  },
  {
   "name": "Shap-User",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "shap-user",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "ShapBot",
   "operator": "Parallel",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "yes",
   "user_agents": "shapbot; shapbot/",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://parallel.ai"
  },
  {
   "name": "ShopAlike",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "shopalike",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Shopify Partner",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "shopify partner",
   "sources": "matomo",
   "docs_url": "https://www.shopify.com/partners"
  },
  {
   "name": "Shortwave Image Fetcher",
   "operator": "Shortwave Communications Inc.",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "shortwave image fetcher",
   "sources": "cloudflare",
   "docs_url": "https://www.shortwave.com/"
  },
  {
   "name": "Sidetrade indexer bot",
   "operator": "Sidetrade",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "sidetrade indexer bot",
   "sources": "ai.robots.txt",
   "docs_url": "https://www.sidetrade.com"
  },
  {
   "name": "SilverReader",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "silverreader",
   "sources": "matomo",
   "docs_url": "http://silverreader.com"
  },
  {
   "name": "Simbiat Software",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "simbiat software",
   "sources": "matomo",
   "docs_url": "https://www.simbiat.eu"
  },
  {
   "name": "Sirdata",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sirdata",
   "sources": "matomo",
   "docs_url": "https://semantic-api.docs.sirdata.net/contextual-api/contextual-api/introduction"
  },
  {
   "name": "SISTRIX Crawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sistrix crawler",
   "sources": "matomo",
   "docs_url": "http://crawler.sistrix.net"
  },
  {
   "name": "SISTRIX Optimizer",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sistrix optimizer",
   "sources": "matomo",
   "docs_url": "https://optimizer.sistrix.com"
  },
  {
   "name": "SiteAuditBot",
   "operator": "Semrush",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "siteauditbot",
   "sources": "cloudflare; matomo",
   "docs_url": "https://www.semrush.com/"
  },
  {
   "name": "Sitebulb",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sitebulb",
   "sources": "matomo",
   "docs_url": "https://sitebulb.com/"
  },
  {
   "name": "SiteCheckerBotCrawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sitecheckerbotcrawler",
   "sources": "matomo",
   "docs_url": "https://sitechecker.pro/"
  },
  {
   "name": "SiteGuru",
   "operator": "SiteGuru",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "yes",
   "user_agents": "sitegurucrawler",
   "sources": "cloudflare",
   "docs_url": "https://www.siteguru.co/crawler"
  },
  {
   "name": "Siteimprove",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "siteimprove",
   "sources": "matomo",
   "docs_url": "https://siteimprove.com/"
  },
  {
   "name": "Siteimprove Crawl",
   "operator": "Siteimprove",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "sitecheck-sitecrawl",
   "sources": "cloudflare",
   "docs_url": "https://support.siteimprove.com/hc/en-gb/articles/206345523-What-IP-addresses-and-user-agents-are-used-by-Siteimprove-"
  },
  {
   "name": "SitemapParser-VIPnytt",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sitemapparser-vipnytt",
   "sources": "matomo",
   "docs_url": "https://github.com/VIPnytt/SitemapParser/"
  },
  {
   "name": "SiteOne Crawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "siteone crawler",
   "sources": "matomo",
   "docs_url": "https://crawler.siteone.io/bot/"
  },
  {
   "name": "SiteScore",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sitescore",
   "sources": "matomo",
   "docs_url": "https://sitescore.ai/"
  },
  {
   "name": "SiteSearch360",
   "operator": "SEMKNOX GmbH",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sitesearch360",
   "sources": "cloudflare",
   "docs_url": "https://www.sitesearch360.com/"
  },
  {
   "name": "SiteSucker",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sitesucker",
   "sources": "matomo",
   "docs_url": "http://ricks-apps.com/osx/sitesucker/"
  },
  {
   "name": "Skroutz ImageBot",
   "operator": "Skroutz S.A.",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "skroutz imagebot",
   "sources": "cloudflare",
   "docs_url": "https://www.skroutz.gr/"
  },
  {
   "name": "SkroutzBot",
   "operator": "Skroutz S.A.",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "skroutzbot",
   "sources": "cloudflare",
   "docs_url": "https://www.skroutz.gr/"
  },
  {
   "name": "Skype",
   "operator": "Skype",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "skypeuripreview",
   "sources": "cloudflare",
   "docs_url": "https://www.skype.com/en/"
  },
  {
   "name": "Slack Image Proxy",
   "operator": "Slack",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "slack-imgproxy",
   "sources": "cloudflare",
   "docs_url": "https://api.slack.com/robots"
  },
  {
   "name": "Slackbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "slackbot",
   "sources": "matomo",
   "docs_url": "https://api.slack.com/robots"
  },
  {
   "name": "SMTBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "smtbot",
   "sources": "matomo",
   "docs_url": "https://www.similartech.com/smtbot"
  },
  {
   "name": "SMTnet PM Bot",
   "operator": "SMTnet",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "yes",
   "user_agents": "smtnetpmbot/",
   "sources": "cloudflare",
   "docs_url": "https://smtnet.com/"
  },
  {
   "name": "Snapchat Ads",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "snapchat ads",
   "sources": "matomo",
   "docs_url": "https://businesshelp.snapchat.com/s/article/adsbot-crawler?language=en_US"
  },
  {
   "name": "Snapchat Proxy",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "snapchat proxy",
   "sources": "matomo",
   "docs_url": "https://www.snapchat.com"
  },
  {
   "name": "Sogou Spider",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "sogou spider",
   "sources": "matomo",
   "docs_url": "http://www.sogou.com/docs/help/webmasters.htm"
  },
  {
   "name": "Soso Spider",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "soso spider",
   "sources": "matomo",
   "docs_url": "http://help.soso.com/webspider.htm"
  },
  {
   "name": "sourcedash",
   "operator": "Source Dashboard",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "yes",
   "user_agents": "sourcedash",
   "sources": "cloudflare",
   "docs_url": "https://www.sourcedashboard.co.uk"
  },
  {
   "name": "Sparkler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sparkler",
   "sources": "matomo",
   "docs_url": "https://github.com/USCDataScience/sparkler"
  },
  {
   "name": "Spawning AI",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "spawning ai",
   "sources": "matomo",
   "docs_url": "https://spawning.ai/"
  },
  {
   "name": "Spider",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "spider",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "SpiderLing",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "spiderling",
   "sources": "matomo",
   "docs_url": "https://nlp.fi.muni.cz/projects/biwec/"
  },
  {
   "name": "Spinn3r",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "spinn3r",
   "sources": "matomo",
   "docs_url": "http://spinn3r.com/robot"
  },
  {
   "name": "SplitSignalBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "splitsignalbot",
   "sources": "matomo",
   "docs_url": "https://www.semrush.com/bot/"
  },
  {
   "name": "Sprinklr",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sprinklr",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Sputnik Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sputnik bot",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Sputnik Favicon Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sputnik favicon bot",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Sputnik Image Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sputnik image bot",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Spyglasses",
   "operator": "Spyglasses",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "spyglassesbot",
   "sources": "cloudflare",
   "docs_url": "https://www.spyglasses.io"
  },
  {
   "name": "sqalix",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sqalix",
   "sources": "matomo",
   "docs_url": "https://www.sqalix.com/"
  },
  {
   "name": "StackScopeBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "stackscopebot",
   "sources": "matomo",
   "docs_url": "https://stackscope.dev/bot"
  },
  {
   "name": "start.me",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "start.me",
   "sources": "matomo",
   "docs_url": "https://about.start.me/"
  },
  {
   "name": "Startpagina Linkchecker",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "startpagina linkchecker",
   "sources": "matomo",
   "docs_url": "https://www.startpagina.nl/linkchecker"
  },
  {
   "name": "Statabot",
   "operator": "StataCorp LLC",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "statabot",
   "sources": "cloudflare",
   "docs_url": "https://www.stata.com/support/statabot"
  },
  {
   "name": "Statista",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "statista",
   "sources": "matomo",
   "docs_url": "https://www.statista.com/"
  },
  {
   "name": "StatistikAustria",
   "operator": "Statistik Bot",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "statistikaustria",
   "sources": "cloudflare",
   "docs_url": "https://www.statistik.at/ueber-uns/innovationen-und-experimentelle-statistik/einsatz-von-kassenscannerdaten-und-webscraping-in-der-preisstatistik"
  },
  {
   "name": "StatOnline.ru",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "statonline.ru",
   "sources": "matomo",
   "docs_url": "https://statonline.ru/"
  },
  {
   "name": "StatsDroneBot",
   "operator": "StatsDrone",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "statsdronebot",
   "sources": "cloudflare",
   "docs_url": "https://statsdrone.com/statsdrone-bot-documentation/"
  },
  {
   "name": "StatusNestBacklinkSpider",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "statusnestbacklinkspider",
   "sources": "matomo",
   "docs_url": "https://statusnest.com/bot"
  },
  {
   "name": "Steam Chat",
   "operator": "Valve Software",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "steamchat",
   "sources": "cloudflare",
   "docs_url": "https://store.steampowered.com"
  },
  {
   "name": "Steve Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "steve bot",
   "sources": "matomo",
   "docs_url": "https://www.shareaholic.com/steve"
  },
  {
   "name": "Stract",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "stract",
   "sources": "matomo",
   "docs_url": "https://stract.com/webmasters"
  },
  {
   "name": "Strivve Automation",
   "operator": "Strivve",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "strivve automation",
   "sources": "cloudflare",
   "docs_url": "https://strivve.com/what-and-how-strivve-works"
  },
  {
   "name": "Sublinq",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sublinq",
   "sources": "matomo",
   "docs_url": "https://web.archive.org/web/20220626191617/https://sublinq.com/"
  },
  {
   "name": "Substack Content Fetch",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "substack content fetch",
   "sources": "matomo",
   "docs_url": "https://substack.com/"
  },
  {
   "name": "SuggestBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "suggestbot",
   "sources": "matomo",
   "docs_url": "https://github.com/nettrom/suggestbot"
  },
  {
   "name": "Superfeedr Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "superfeedr bot",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "SurdotlyBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "surdotlybot",
   "sources": "matomo",
   "docs_url": "http://sur.ly/bot.html"
  },
  {
   "name": "Survey Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "survey bot",
   "sources": "matomo",
   "docs_url": "http://www.domaintools.com/webmasters/surveybot.php"
  },
  {
   "name": "svEye",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "sveye",
   "sources": "matomo",
   "docs_url": "https://sveye.semantic-visions.com/"
  },
  {
   "name": "Swiftbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "swiftbot",
   "sources": "matomo",
   "docs_url": "https://swiftype.com/swiftbot"
  },
  {
   "name": "Swisscows Favicons",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "swisscows favicons",
   "sources": "matomo",
   "docs_url": "https://swisscows.com/"
  },
  {
   "name": "Synapse",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "synapse",
   "sources": "matomo",
   "docs_url": "https://github.com/matrix-org/synapse"
  },
  {
   "name": "t3versions",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "t3versions",
   "sources": "matomo",
   "docs_url": "https://www.t3versions.com/bot"
  },
  {
   "name": "Taboolabot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "taboolabot",
   "sources": "matomo",
   "docs_url": "https://help.taboola.com/hc/en-us/articles/115002347594-The-Taboola-Crawler"
  },
  {
   "name": "TactiScout",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "tactiscout",
   "sources": "matomo",
   "docs_url": "https://find-it.world/TempCrawl/Crawltheque.php"
  },
  {
   "name": "Tag Inspector",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "tag inspector",
   "sources": "matomo",
   "docs_url": "https://taginspector.com/"
  },
  {
   "name": "Tarmot Gezgin",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "tarmot gezgin",
   "sources": "matomo",
   "docs_url": "http://www.tarmot.com/gezgin/"
  },
  {
   "name": "TavilyBot",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "unclear",
   "user_agents": "tavilybot",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "tchelebi",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "tchelebi",
   "sources": "matomo",
   "docs_url": "https://tchelebi.io/"
  },
  {
   "name": "Telegram Bot",
   "operator": "Telegram",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "telegrambot",
   "sources": "cloudflare",
   "docs_url": ""
  },
  {
   "name": "Terra Cotta",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "unclear",
   "user_agents": "terra cotta",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "TerraCotta",
   "operator": "Ceramic AI",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "yes",
   "user_agents": "terracotta",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://ceramic.ai/"
  },
  {
   "name": "TestCrawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "testcrawler",
   "sources": "matomo",
   "docs_url": "https://www.comcepta.com/"
  },
  {
   "name": "The British Library Legal Deposit Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "the british library legal deposit bot",
   "sources": "matomo",
   "docs_url": "https://www.bl.uk/"
  },
  {
   "name": "The Trade Desk Content",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "the trade desk content",
   "sources": "matomo",
   "docs_url": "https://www.thetradedesk.com/us/ttd-content"
  },
  {
   "name": "Thinkbot",
   "operator": "Thinkbot",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "no",
   "user_agents": "thinkbot",
   "sources": "ai.robots.txt",
   "docs_url": "https://www.thinkbot.agency"
  },
  {
   "name": "TigerBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "tigerbot",
   "sources": "matomo",
   "docs_url": "https://tiger.ch/"
  },
  {
   "name": "TikTokSpider",
   "operator": "ByteDance",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "tiktokspider",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://www.tiktok.com/"
  },
  {
   "name": "Timpibot",
   "operator": "Timpi",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "timpibot",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://timpi.io"
  },
  {
   "name": "TinEye",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "tineye",
   "sources": "matomo",
   "docs_url": "https://tineye.com/"
  },
  {
   "name": "TinEye Crawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "tineye crawler",
   "sources": "matomo",
   "docs_url": "http://www.tineye.com/crawler.html"
  },
  {
   "name": "Tiny Tiny RSS",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "tiny tiny rss",
   "sources": "matomo",
   "docs_url": "http://tt-rss.org"
  },
  {
   "name": "TongyiBot",
   "operator": "Alibaba that fetches web content for the Tongyi Qianwen assistant and related Qwen-generated answers",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "tongyibot",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Toutiao",
   "operator": "Bytedance",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "zhanzhang.toutiao.com",
   "sources": "cloudflare",
   "docs_url": "https://zhanzhang.toutiao.com/docs/intro/26899"
  },
  {
   "name": "TraceMyFile",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "tracemyfile",
   "sources": "matomo",
   "docs_url": "https://www.tracemyfile.com/"
  },
  {
   "name": "Trae",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "agent",
   "category": "AI Coding Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "trae",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Trellis-Services",
   "operator": "Mediavine",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "trellis-services",
   "sources": "cloudflare",
   "docs_url": "https://www.mediavine.com"
  },
  {
   "name": "Trendiction Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "trendiction bot",
   "sources": "matomo",
   "docs_url": "http://www.trendiction.de/bot"
  },
  {
   "name": "Trendsmap",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "trendsmap",
   "sources": "matomo",
   "docs_url": "https://www.trendsmap.com/"
  },
  {
   "name": "TrustBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "trustbot",
   "sources": "matomo",
   "docs_url": "https://truster.info/en/"
  },
  {
   "name": "TrustedSite",
   "operator": "TrustedSite (formerly McAfee SECURE)",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Academic Research",
   "respects_robots_txt": "unclear",
   "user_agents": "trustedsite",
   "sources": "cloudflare",
   "docs_url": "https://www.trustedsite.com"
  },
  {
   "name": "Tumblr",
   "operator": "Automattic",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "tumblr/",
   "sources": "cloudflare",
   "docs_url": "https://automattic.com/"
  },
  {
   "name": "TurnitinBot",
   "operator": "Turnitin",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Academic Research",
   "respects_robots_txt": "unclear",
   "user_agents": "turnitin; turnitinbot",
   "sources": "cloudflare; matomo",
   "docs_url": "https://turnitin.com/robot/crawlerinfo.html"
  },
  {
   "name": "TweetedTimes Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "tweetedtimes bot",
   "sources": "matomo",
   "docs_url": "https://tweetedtimes.com/"
  },
  {
   "name": "Tweetmeme Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "tweetmeme bot",
   "sources": "matomo",
   "docs_url": "http://tweetmeme.com/"
  },
  {
   "name": "TwinAgent",
   "operator": "Twin, a platform that creates automated workers to perform tasks by integrating with APIs and controlling web applications through browser automa…",
   "kind": "ai",
   "purpose": "agent",
   "category": "AI Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "twinagent",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://twin.so"
  },
  {
   "name": "Twingly Recon",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "twingly recon",
   "sources": "matomo",
   "docs_url": "https://app.twingly.com/public-docs/crawler"
  },
  {
   "name": "twitterbot",
   "operator": "Twitter",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "twitterbot/",
   "sources": "cloudflare",
   "docs_url": "https://twitter.com"
  },
  {
   "name": "Twurly",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "twurly",
   "sources": "matomo",
   "docs_url": "https://twurly.org/"
  },
  {
   "name": "UCSB Network Measurement",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "ucsb network measurement",
   "sources": "matomo",
   "docs_url": "https://www.it.ucsb.edu/"
  },
  {
   "name": "UkrNet Mail Proxy",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "ukrnet mail proxy",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "uMBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "umbot",
   "sources": "matomo",
   "docs_url": "https://www.ubermetrics-technologies.com/"
  },
  {
   "name": "UniversalFeedParser",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "universalfeedparser",
   "sources": "matomo",
   "docs_url": "https://github.com/kurtmckee/feedparser"
  },
  {
   "name": "upday",
   "operator": "upday GmbH & Co. KG",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "yes",
   "user_agents": "upday/",
   "sources": "cloudflare",
   "docs_url": "https://www.upday.com/"
  },
  {
   "name": "URLAppendBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "urlappendbot",
   "sources": "matomo",
   "docs_url": "http://www.profound.net/urlappendbot.html"
  },
  {
   "name": "URLSuMaBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "urlsumabot",
   "sources": "matomo",
   "docs_url": "https://www.urlsuma.de/"
  },
  {
   "name": "UseAI",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "useai",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Vagabondo",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "vagabondo",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "ValidBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "validbot",
   "sources": "matomo",
   "docs_url": "https://www.validbot.com/"
  },
  {
   "name": "Velen Public Web Crawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "velen public web crawler",
   "sources": "matomo",
   "docs_url": "https://hunter.io/robot"
  },
  {
   "name": "VelenPublicWebCrawler",
   "operator": "Velen Crawler",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "velenpublicwebcrawler",
   "sources": "ai.robots.txt",
   "docs_url": "https://velen.io"
  },
  {
   "name": "VeryHip",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "veryhip",
   "sources": "matomo",
   "docs_url": "https://veryhip.com/"
  },
  {
   "name": "videootv Bot",
   "operator": "Digital Green",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "yes",
   "user_agents": "videootvbot",
   "sources": "cloudflare",
   "docs_url": "https://videoo.tv/"
  },
  {
   "name": "VirusTotal Cloud",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "virustotal cloud",
   "sources": "matomo",
   "docs_url": "https://www.virustotal.com/"
  },
  {
   "name": "Visual Site Mapper Crawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "visual site mapper crawler",
   "sources": "matomo",
   "docs_url": "http://www.visualsitemapper.com/crawler"
  },
  {
   "name": "Visually.io Shopify Editor",
   "operator": "Visually.io",
   "kind": "ai",
   "purpose": "assistant",
   "category": "Ai Assistant",
   "respects_robots_txt": "unclear",
   "user_agents": "visually.io shopify editor",
   "sources": "cloudflare",
   "docs_url": "https://help.visually.io/visually.io-editor-cloudflare-signed-agent"
  },
  {
   "name": "VK Robot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "vk robot",
   "sources": "matomo",
   "docs_url": "https://dev.vk.com/en/"
  },
  {
   "name": "VK Share Button",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "vk share button",
   "sources": "matomo",
   "docs_url": "https://dev.vk.com/en/widgets/share"
  },
  {
   "name": "VORTEX",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "vortex",
   "sources": "matomo",
   "docs_url": "https://marty.anstey.ca/robots/vortex"
  },
  {
   "name": "vuhuvBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "vuhuvbot",
   "sources": "matomo",
   "docs_url": "https://vuhuv.com/bot.html"
  },
  {
   "name": "W3 Validator Services",
   "operator": "World Wide Web Consortium (W3C)",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "w3c-checklink",
   "sources": "cloudflare",
   "docs_url": "https://validator.w3.org/services"
  },
  {
   "name": "WARDBot",
   "operator": "WEBSPARK",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "wardbot",
   "sources": "ai.robots.txt; cloudflare",
   "docs_url": "https://ward.ai/robot"
  },
  {
   "name": "WebbCrawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "webbcrawler",
   "sources": "matomo",
   "docs_url": "http://badcheese.com/crawler.html"
  },
  {
   "name": "WebCEO",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "webceo",
   "sources": "matomo",
   "docs_url": "https://www.webceo.com/"
  },
  {
   "name": "WebDataStats",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "webdatastats",
   "sources": "matomo",
   "docs_url": "https://webdatastats.com/policy.html"
  },
  {
   "name": "Webliobot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "webliobot",
   "sources": "matomo",
   "docs_url": "https://help.weblio.jp/ejje/jpdictionary/j_info/j_i_bcontents/h0021"
  },
  {
   "name": "Weborama",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "weborama",
   "sources": "matomo",
   "docs_url": "https://weborama.com/"
  },
  {
   "name": "WebPros",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "webpros",
   "sources": "matomo",
   "docs_url": "https://webpros.com/"
  },
  {
   "name": "Website-info",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "website-info",
   "sources": "matomo",
   "docs_url": "https://website-info.net/robot"
  },
  {
   "name": "WebSpiderMount",
   "operator": "AspenTechLabs Inc",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Aggregator",
   "respects_robots_txt": "unclear",
   "user_agents": "webspidermount",
   "sources": "cloudflare",
   "docs_url": "https://webspidermount.com"
  },
  {
   "name": "webtru",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "webtru",
   "sources": "matomo",
   "docs_url": "https://webtru.io/"
  },
  {
   "name": "Webwiki",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "webwiki",
   "sources": "matomo",
   "docs_url": "https://www.webwiki.com/"
  },
  {
   "name": "webzio",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Search Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "webzio",
   "sources": "matomo",
   "docs_url": "https://docs.webz.io/reference/how-it-works-video"
  },
  {
   "name": "Webzio-Extended",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "unclear",
   "user_agents": "webzio-extended",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://docs.webz.io/reference/how-it-works-video"
  },
  {
   "name": "WellKnownBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "wellknownbot",
   "sources": "matomo",
   "docs_url": "https://well-known.dev"
  },
  {
   "name": "WeSEE:Search",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "wesee:search",
   "sources": "matomo",
   "docs_url": "http://www.wesee.com/bot"
  },
  {
   "name": "WeViKaBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "wevikabot",
   "sources": "matomo",
   "docs_url": "http://www.wevika.de"
  },
  {
   "name": "WhatCMS",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "whatcms",
   "sources": "matomo",
   "docs_url": "https://whatcms.org/"
  },
  {
   "name": "WhereGoes",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "wheregoes",
   "sources": "matomo",
   "docs_url": "https://wheregoes.com/"
  },
  {
   "name": "Who.is Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "who.is bot",
   "sources": "matomo",
   "docs_url": "https://who.is/"
  },
  {
   "name": "Wibybot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "wibybot",
   "sources": "matomo",
   "docs_url": "https://www.wiby.me/"
  },
  {
   "name": "WikiDo",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "wikido",
   "sources": "matomo",
   "docs_url": "https://www.wikido.com/"
  },
  {
   "name": "Willow Internet Crawler",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "willow internet crawler",
   "sources": "matomo",
   "docs_url": "https://twotrees.com/company/indexing-web-crawlers/"
  },
  {
   "name": "WindowsForum-AI",
   "operator": "WindowsForum.com",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "windowsforum-ai",
   "sources": "cloudflare",
   "docs_url": "https://windowsforum.com"
  },
  {
   "name": "WireReaderBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "wirereaderbot",
   "sources": "matomo",
   "docs_url": "https://wirereader.app/"
  },
  {
   "name": "WMF Citoid",
   "operator": "Wikimedia Foundation",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "citoid/wmf",
   "sources": "cloudflare",
   "docs_url": "https://wikitech.wikimedia.org/wiki/Citoid"
  },
  {
   "name": "WMF Zotero Translation Server",
   "operator": "Wikimedia Foundation",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "zoterotranslationserver/wmf",
   "sources": "cloudflare",
   "docs_url": "https://wikitech.wikimedia.org/wiki/Zotero"
  },
  {
   "name": "WooRank",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "woorank",
   "sources": "matomo",
   "docs_url": "https://www.woorank.com/"
  },
  {
   "name": "WordCountBot",
   "operator": "Weglot",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "wordcountbot",
   "sources": "cloudflare",
   "docs_url": "https://weglot.com"
  },
  {
   "name": "Workona",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "workona",
   "sources": "matomo",
   "docs_url": "https://workona.com/"
  },
  {
   "name": "Wotbox",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "wotbox",
   "sources": "matomo",
   "docs_url": "http://www.wotbox.com/bot/"
  },
  {
   "name": "wp.com feedbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "wp.com feedbot",
   "sources": "matomo",
   "docs_url": "https://wordpress.com/"
  },
  {
   "name": "wpbot",
   "operator": "QuantumCloud",
   "kind": "ai",
   "purpose": "scraper",
   "category": "AI Data Providers",
   "respects_robots_txt": "unclear",
   "user_agents": "wpbot",
   "sources": "ai.robots.txt",
   "docs_url": "https://www.quantumcloud.com"
  },
  {
   "name": "WPMU DEV",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "wpmu dev",
   "sources": "matomo",
   "docs_url": "https://wpmudev.com/docs/wpmu-dev-plugins/broken-link-checker/#broken-link-checker-user-agent"
  },
  {
   "name": "WRTNBot",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "unknown",
   "category": "Undocumented AI Agents",
   "respects_robots_txt": "unclear",
   "user_agents": "wrtnbot",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "XoviBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "xovibot",
   "sources": "matomo",
   "docs_url": "https://www.xovibot.net"
  },
  {
   "name": "xtate",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "xtate",
   "sources": "matomo",
   "docs_url": "https://github.com/babycoff/xtate"
  },
  {
   "name": "XY Archive Compliance Bot",
   "operator": "XY Archive Compliance",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Archiver",
   "respects_robots_txt": "unclear",
   "user_agents": "xy-archive-compliance",
   "sources": "cloudflare",
   "docs_url": "https://xy-archive.helpscoutdocs.com/article/61-does-xy-archive-have-a-dedicated-ip-address"
  },
  {
   "name": "YaCy",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "yacy",
   "sources": "matomo",
   "docs_url": "http://yacy.net/bot.html"
  },
  {
   "name": "Yahoo Gemini",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "yahoo gemini",
   "sources": "matomo",
   "docs_url": "https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html"
  },
  {
   "name": "Yahoo Japan SEO Crawler",
   "operator": "Yahoo",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "j-brw; j-brw/",
   "sources": "cloudflare",
   "docs_url": "https://www.yahoo-help.jp"
  },
  {
   "name": "Yahoo Link Preview",
   "operator": "Yahoo",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "yahoo link preview; yahoo! link preview",
   "sources": "cloudflare; matomo",
   "docs_url": "https://help.yahoo.com/kb/mail/yahoo-link-preview-SLN23615.html"
  },
  {
   "name": "Yahoo Mail",
   "operator": "Yahoo",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "yahoomailproxy",
   "sources": "cloudflare",
   "docs_url": "https://help.yahoo.com/kb/yahoo-mail-proxy-SLN28749.html"
  },
  {
   "name": "Yahoo Slurp",
   "operator": "Yahoo",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "yes",
   "user_agents": "yahoo! slurp",
   "sources": "cloudflare; matomo",
   "docs_url": "http://help.yahoo.com/help/us/ysearch/slurp"
  },
  {
   "name": "Yahoo! Cache System",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "yahoo! cache system",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Yahoo! JAPAN",
   "operator": "Yahoo! JAPAN",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "yes",
   "user_agents": "j-dlc/; yahoo! japan",
   "sources": "cloudflare; matomo",
   "docs_url": "https://www.yahoo.co.jp/"
  },
  {
   "name": "Yahoo! Japan $1",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "yahoo! japan $1",
   "sources": "matomo",
   "docs_url": "https://support.yahoo-net.jp/PccSearch/s/article/H000007955"
  },
  {
   "name": "YahooCacheSystem",
   "operator": "Yahoo",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "yahoocachesystem",
   "sources": "cloudflare",
   "docs_url": "https://www.yahoo.com/"
  },
  {
   "name": "YahooMailProxy",
   "operator": "Yahoo",
   "kind": "crawler",
   "purpose": "assistant",
   "category": "Page Preview",
   "respects_robots_txt": "unclear",
   "user_agents": "yahoomailproxy",
   "sources": "cloudflare",
   "docs_url": "https://help.yahoo.com/kb/yahoo-mail-proxy-SLN28749.html"
  },
  {
   "name": "YaK",
   "operator": "Meltwater",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "unclear",
   "user_agents": "yak",
   "sources": "ai.robots.txt; matomo",
   "docs_url": "https://www.meltwater.com/en/suite/consumer-intelligence"
  },
  {
   "name": "YandexAdditional",
   "operator": "Yandex",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "yandexadditional",
   "sources": "ai.robots.txt",
   "docs_url": "https://yandex.ru"
  },
  {
   "name": "YandexAdditionalBot",
   "operator": "Yandex",
   "kind": "ai",
   "purpose": "training",
   "category": "AI Data Scrapers",
   "respects_robots_txt": "yes",
   "user_agents": "yandexadditionalbot",
   "sources": "ai.robots.txt",
   "docs_url": "https://yandex.ru"
  },
  {
   "name": "YandexBot",
   "operator": "Yandex",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "http://yandex.com/bots",
   "sources": "cloudflare",
   "docs_url": "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html"
  },
  {
   "name": "YandexBot-MirrorDetector",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "yandexbot-mirrordetector",
   "sources": "matomo",
   "docs_url": "https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html"
  },
  {
   "name": "Yeti",
   "operator": "Naver",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "yeti/",
   "sources": "cloudflare",
   "docs_url": "https://help.naver.com/support/contents/contents.help?serviceNo=19634&categoryNo=19668&_website_robots"
  },
  {
   "name": "Yeti/Naverbot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "yeti/naverbot",
   "sources": "matomo",
   "docs_url": "http://help.naver.com/robots/"
  },
  {
   "name": "YGS Group Falconer Scraper",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "training",
   "category": "Ai Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "ygs-scraper-bot",
   "sources": "cloudflare",
   "docs_url": "https://ygscontentlicensing.com/falconer/"
  },
  {
   "name": "YioopBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "yioopbot",
   "sources": "matomo",
   "docs_url": "https://www.yioop.com/bot"
  },
  {
   "name": "YiyanBot",
   "operator": "Baidu that fetches web content for the yiyan",
   "kind": "ai",
   "purpose": "assistant",
   "category": "AI Assistants",
   "respects_robots_txt": "unclear",
   "user_agents": "yiyanbot",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "yoozBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "yoozbot",
   "sources": "matomo",
   "docs_url": "https://www.yooz.ir/"
  },
  {
   "name": "YouBot",
   "operator": "You.com",
   "kind": "ai",
   "purpose": "answer",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "yes",
   "user_agents": "youbot",
   "sources": "ai.robots.txt; cloudflare; matomo",
   "docs_url": "https://about.you.com/youchat/"
  },
  {
   "name": "Youdao Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "youdao bot",
   "sources": "matomo",
   "docs_url": "http://www.youdao.com/help/webmaster/spider"
  },
  {
   "name": "Yourls",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "yourls",
   "sources": "matomo",
   "docs_url": "http://yourls.org"
  },
  {
   "name": "Yunyun Bot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search bot",
   "respects_robots_txt": "unclear",
   "user_agents": "yunyun bot",
   "sources": "matomo",
   "docs_url": "http://www.yunyun.com/SiteInfo.php?r=about"
  },
  {
   "name": "Zaldamo",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "zaldamo",
   "sources": "matomo",
   "docs_url": "https://www.zaldamo.com/search.html"
  },
  {
   "name": "ZanistaBot",
   "operator": "Unknown",
   "kind": "ai",
   "purpose": "search",
   "category": "AI Search Crawlers",
   "respects_robots_txt": "unclear",
   "user_agents": "zanistabot",
   "sources": "ai.robots.txt",
   "docs_url": ""
  },
  {
   "name": "Zao",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "zao",
   "sources": "matomo",
   "docs_url": ""
  },
  {
   "name": "Ze List",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Feed Fetcher",
   "respects_robots_txt": "unclear",
   "user_agents": "ze list",
   "sources": "matomo",
   "docs_url": "https://www.zelist.ro/"
  },
  {
   "name": "Zeno",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "zeno",
   "sources": "matomo",
   "docs_url": "https://github.com/internetarchive/Zeno"
  },
  {
   "name": "Zookabot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "zookabot",
   "sources": "matomo",
   "docs_url": "http://zookabot.com"
  },
  {
   "name": "Zoombot",
   "operator": "SEOZoom s.r.l.",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Search Engine Optimization",
   "respects_robots_txt": "unclear",
   "user_agents": "zoombot",
   "sources": "cloudflare; matomo",
   "docs_url": "https://www.seozoom.it/"
  },
  {
   "name": "ZoomInfo",
   "operator": "ZoomInfo",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "zoominfobot",
   "sources": "cloudflare",
   "docs_url": "https://zoominfo.com"
  },
  {
   "name": "ZoominfoBot",
   "operator": "Unknown",
   "kind": "crawler",
   "purpose": "scraper",
   "category": "Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "zoominfobot",
   "sources": "matomo",
   "docs_url": "https://www.zoominfo.com"
  },
  {
   "name": "ZumBot",
   "operator": "Zum Internet Corp",
   "kind": "crawler",
   "purpose": "search",
   "category": "Search Engine Crawler",
   "respects_robots_txt": "unclear",
   "user_agents": "zumbot",
   "sources": "cloudflare; matomo",
   "docs_url": "http://www.zuminternet.com/en"
  }
 ]
}