# ----------------------------------------------------------------------
# DEFAULT — allow every crawler to read all public content
# ----------------------------------------------------------------------
User-Agent: *
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /profile/
Disallow: /private/
# ----------------------------------------------------------------------
# AI SEARCH / CITATION CRAWLERS
# ----------------------------------------------------------------------
# OpenAI (ChatGPT search results & user-triggered fetches)
User-agent: OAI-SearchBot
Allow: /
User-agent: ChatGPT-User
Allow: /
# Anthropic (Claude search citations & user-triggered fetches)
User-agent: Claude-SearchBot
Allow: /
User-agent: Claude-User
Allow: /
# Perplexity (index & user-triggered fetches)
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity-User
Allow: /
# Microsoft Bing (powers Bing + Copilot)
User-agent: Bingbot
Allow: /
# Google (powers Search + AI Overviews + AI Mode)
User-agent: Googlebot
Allow: /
# Apple (Siri / Spotlight / Apple Intelligence answers)
User-agent: Applebot
Allow: /
# Amazon (Alexa / Rufus answers)
User-agent: Amazonbot
Allow: /
# DuckDuckGo (DuckAssist answers)
User-agent: DuckAssistBot
Allow: /
# Meta AI
User-agent: meta-externalagent
Allow: /
# ----------------------------------------------------------------------
# AI TRAINING / GROUNDING TOKENS
# ----------------------------------------------------------------------
# OpenAI training crawler
User-agent: GPTBot
Allow: /
# Anthropic training crawler
User-agent: ClaudeBot
Allow: /
# Google (Gemini / Vertex training & grounding)
User-agent: Google-Extended
Allow: /
# Apple (Apple Intelligence training)
User-agent: Applebot-Extended
Allow: /
# Common Crawl (feeds many open LLM datasets)
User-agent: CCBot
Allow: /
# ----------------------------------------------------------------------
# SITEMAPS
# ----------------------------------------------------------------------
Sitemap: https://www.11prop.com/sitemap-offplan-pages.xml
Sitemap: https://www.11prop.com/sitemap-pages.xml
Sitemap: https://www.11prop.com/sitemap-blogs.xml
Sitemap: https://www.11prop.com/sitemap-news.xml
Sitemap: https://www.11prop.com/sitemap-properties.xml
Sitemap: https://www.11prop.com/sitemap-developers.xml
Sitemap: https://www.11prop.com/sitemap-communities.xml