{
 "id": "robots-txt-ai",
 "name": "robots.txt for AI agents",
 "family": "rules",
 "about": "The file that tells crawlers and agents which pages to leave alone.",
 "page": "https://agentdrives.com/standards/robots-txt-ai",
 "updated": "2026-10-24",
 "numbers": [
  {
   "label": "robots.txt closes the sign-up page",
   "n": 15,
   "total": 151,
   "fact": "https://agentdrives.com/data/facts#who.robots_closed"
  },
  {
   "label": "robots.txt kept our agent out of sign-up",
   "n": 1,
   "total": 151,
   "fact": "https://agentdrives.com/data/facts#who.robots_stopped"
  }
 ],
 "products": [
  {
   "slug": "akiflow",
   "name": "Akiflow",
   "level": 1,
   "page": "https://agentdrives.com/tools/akiflow"
  },
  {
   "slug": "browse",
   "name": "Browse AI",
   "level": 1,
   "page": "https://agentdrives.com/tools/browse"
  },
  {
   "slug": "clickhouse",
   "name": "ClickHouse Cloud",
   "level": 4,
   "page": "https://agentdrives.com/tools/clickhouse"
  },
  {
   "slug": "clickup",
   "name": "ClickUp",
   "level": 1,
   "page": "https://agentdrives.com/tools/clickup"
  },
  {
   "slug": "close",
   "name": "Close",
   "level": 1,
   "page": "https://agentdrives.com/tools/close"
  },
  {
   "slug": "convex",
   "name": "Convex",
   "level": 4,
   "page": "https://agentdrives.com/tools/convex"
  },
  {
   "slug": "figma",
   "name": "Figma",
   "level": 1,
   "page": "https://agentdrives.com/tools/figma"
  },
  {
   "slug": "helpscout",
   "name": "Help Scout",
   "level": 1,
   "page": "https://agentdrives.com/tools/helpscout"
  },
  {
   "slug": "hubspot",
   "name": "HubSpot",
   "level": 1,
   "page": "https://agentdrives.com/tools/hubspot"
  },
  {
   "slug": "mattermost",
   "name": "Mattermost",
   "level": 3,
   "page": "https://agentdrives.com/tools/mattermost"
  },
  {
   "slug": "plain",
   "name": "Plain",
   "level": 1,
   "page": "https://agentdrives.com/tools/plain"
  },
  {
   "slug": "planetscale",
   "name": "PlanetScale",
   "level": 2,
   "page": "https://agentdrives.com/tools/planetscale"
  },
  {
   "slug": "sendgrid",
   "name": "SendGrid",
   "level": 1,
   "page": "https://agentdrives.com/tools/sendgrid"
  },
  {
   "slug": "signwell",
   "name": "SignWell",
   "level": 4,
   "page": "https://agentdrives.com/tools/signwell"
  },
  {
   "slug": "smartsheet",
   "name": "Smartsheet",
   "level": 1,
   "page": "https://agentdrives.com/tools/smartsheet"
  }
 ],
 "licence": "CC BY 4.0"
}