User-agent: Googlebot
Allow: /api/ai/hero/
Allow: /api/ai/inline-image/
Allow: /api/ai/meta-image
Allow: /api/og
Disallow: /api/
Disallow: /site.webmanifest
Disallow: /favicon.ico
# A NAMED GROUP DOES NOT INHERIT `User-agent: *` (2026-09-10). Googlebot reads only the group that
# names it, so the private paths listed in the wildcard group below were never disallowed for the
# one crawler that matters most. Sponsor product guides are public URLs shown to a brand.
Disallow: /preview/
Disallow: /preview
Disallow: /for/
Disallow: /admin/
Disallow: /admin
Disallow: /brand/
Disallow: /brand
Disallow: /lab/
Disallow: /lab

User-agent: Googlebot-News
Allow: /api/ai/hero/
Allow: /api/ai/inline-image/
Allow: /api/ai/meta-image
Allow: /api/og
Disallow: /api/
# Same gap as the Googlebot group above.
Disallow: /preview/
Disallow: /preview
Disallow: /for/
Disallow: /admin/
Disallow: /admin
Disallow: /brand/
Disallow: /brand

User-agent: Twitterbot
Allow: /api/ai/hero/
Allow: /api/ai/inline-image/
Allow: /api/og
# A named group inherits nothing (see above), so the brand dashboard is disallowed here too.
Disallow: /brand/
Disallow: /brand

# AI CRAWLERS, INCLUDING TRAINING BOTS, ARE WELCOME ON PUBLIC PAGES (Kevin, 2026-09-24). Cloudflare
# is being switched to allow them too. One stacked group; like every named group it does NOT
# inherit `User-agent: *`, so the private paths are repeated here.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
Allow: /
Allow: /api/ai/hero/
Allow: /api/ai/inline-image/
Allow: /api/og
Disallow: /admin/
Disallow: /admin
Disallow: /brand/
Disallow: /brand
Disallow: /preview/
Disallow: /preview
Disallow: /for/
Disallow: /api/
Disallow: /lab/
Disallow: /lab

User-agent: *
# Content Signals (contentsignals.org): search, AI answers and AI training all permitted (Kevin, 2026-09-24).
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /api/ai/hero/
Allow: /api/ai/inline-image/
Allow: /api/og
Disallow: /admin/
Disallow: /admin
Disallow: /brand/
Disallow: /brand
Disallow: /preview/
Disallow: /preview
Disallow: /for/
Disallow: /api/
Disallow: /article/
Disallow: /article
Disallow: /apps/
Disallow: /apps
Disallow: /content/
Disallow: /content
Disallow: /lab/
Disallow: /lab
Disallow: /site.webmanifest

Sitemap: https://idevice.com/sitemap.xml
Sitemap: https://idevice.com/news-sitemap.xml

# Agentic Resource Discovery (agenticresourcediscovery.org, section 5.1 "Agentmap Directive"): where the machine-readable list of our MCP server and AI agent is.
Agentmap: https://idevice.com/.well-known/ard.json