{
  "version": "2026-09-01",
  "source": "https://rankforge.cc/learn/ai-crawler-user-agents",
  "license": "CC BY 4.0",
  "notes": "Answer engines fetch pages so an assistant can cite them; blocking one removes you from that engine's citations. Training crawlers collect content for model training and send no traffic back, so blocking them is an editorial choice with no measurable traffic cost. Ambiguous agents carry non-AI consequences if blocked. robots.txt is a request, not enforcement.",
  "agents": [
    {
      "agent": "oai-searchbot",
      "kind": "answer",
      "description": "ChatGPT search index",
      "token_only": false
    },
    {
      "agent": "chatgpt-user",
      "kind": "answer",
      "description": "ChatGPT (user-initiated fetch)",
      "token_only": false
    },
    {
      "agent": "perplexitybot",
      "kind": "answer",
      "description": "Perplexity search index",
      "token_only": false
    },
    {
      "agent": "perplexity-user",
      "kind": "answer",
      "description": "Perplexity (user-initiated fetch)",
      "token_only": false
    },
    {
      "agent": "claude-user",
      "kind": "answer",
      "description": "Claude (user-initiated fetch)",
      "token_only": false
    },
    {
      "agent": "claude-searchbot",
      "kind": "answer",
      "description": "Claude search index",
      "token_only": false
    },
    {
      "agent": "duckassistbot",
      "kind": "answer",
      "description": "DuckDuckGo AI assist",
      "token_only": false
    },
    {
      "agent": "youbot",
      "kind": "answer",
      "description": "You.com",
      "token_only": false
    },
    {
      "agent": "mistralai-user",
      "kind": "answer",
      "description": "Le Chat (user-initiated fetch)",
      "token_only": false
    },
    {
      "agent": "amazonbot",
      "kind": "answer",
      "description": "Alexa / Amazon answers",
      "token_only": false
    },
    {
      "agent": "applebot",
      "kind": "answer",
      "description": "Apple Intelligence & Siri",
      "token_only": false
    },
    {
      "agent": "gptbot",
      "kind": "training",
      "description": "OpenAI training",
      "token_only": false
    },
    {
      "agent": "claudebot",
      "kind": "training",
      "description": "Anthropic training",
      "token_only": false
    },
    {
      "agent": "anthropic-ai",
      "kind": "training",
      "description": "Anthropic (legacy agent)",
      "token_only": false
    },
    {
      "agent": "ccbot",
      "kind": "training",
      "description": "Common Crawl",
      "token_only": false
    },
    {
      "agent": "google-extended",
      "kind": "training",
      "description": "Google Gemini training opt-out token",
      "token_only": true
    },
    {
      "agent": "applebot-extended",
      "kind": "training",
      "description": "Apple training opt-out token",
      "token_only": true
    },
    {
      "agent": "meta-externalagent",
      "kind": "training",
      "description": "Meta AI training",
      "token_only": false
    },
    {
      "agent": "bytespider",
      "kind": "training",
      "description": "ByteDance training",
      "token_only": false
    },
    {
      "agent": "cohere-ai",
      "kind": "training",
      "description": "Cohere training",
      "token_only": false
    },
    {
      "agent": "ai2bot",
      "kind": "training",
      "description": "Allen Institute training",
      "token_only": false
    },
    {
      "agent": "omgilibot",
      "kind": "training",
      "description": "Webz.io training feed",
      "token_only": false
    },
    {
      "agent": "diffbot",
      "kind": "training",
      "description": "Diffbot knowledge graph",
      "token_only": false
    },
    {
      "agent": "bingbot",
      "kind": "ambiguous",
      "description": "Bing search; also grounds Microsoft Copilot. No separate token — blocking it removes you from Bing search.",
      "token_only": false
    },
    {
      "agent": "googlebot",
      "kind": "ambiguous",
      "description": "Google Search. Never block as an AI measure; use Google-Extended to control Gemini training instead.",
      "token_only": false
    },
    {
      "agent": "facebookexternalhit",
      "kind": "ambiguous",
      "description": "Generates link previews when a URL is shared. Nothing to do with AI training; blocking it breaks social previews.",
      "token_only": false
    },
    {
      "agent": "google-cloudvertexbot",
      "kind": "ambiguous",
      "description": "Fetches sites for Vertex AI customers building their own agents, rather than for Google's own training.",
      "token_only": false
    }
  ]
}