# =============================================
# robots.txt - orteam.ru
# Updated: 2026-10-08
# =============================================

# --- Main search crawlers ---
# Googlebot and Bingbot use User-agent: *.
# Keep public site pages open for search indexing.

User-agent: *
Disallow: /assets/theme/data/
Disallow: /assets/components/
Disallow: /connectors/
Disallow: /manager/
Disallow: /core/
Disallow: /speed/
Disallow: /test-calculator/
Disallow: /*?
Allow: /*.css?
# Existing FAQ and carousel controls; keep other query URLs excluded.
Allow: /assets/theme/js/index9.js?
Allow: /assets/theme/js/custom.js?
# --- Yandex duplicate URL cleanup ---
# Keep the same technical exclusions as User-agent: * because named groups do not inherit them.
User-agent: Yandex
Disallow: /assets/theme/data/
Disallow: /assets/components/
Disallow: /connectors/
Disallow: /manager/
Disallow: /core/
Disallow: /speed/
Disallow: /test-calculator/
Disallow: /*?
Allow: /*.css?
# Existing FAQ and carousel controls; keep other query URLs excluded.
Allow: /assets/theme/js/index9.js?
Allow: /assets/theme/js/custom.js?
Clean-param: search_source&search_domain&utm_source&utm_medium&utm_campaign&utm_content&utm_term&yclid&ysclid&gclid&fbclid

# --- AI search/retrieval/user-request agents: allowed for public pages only ---
# These agents can bring cited traffic or retrieve pages for a user's direct request.
# Keep the same technical exclusions as User-agent: * because named groups do not inherit them.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: DuckAssistBot
User-agent: MistralAI-User
Disallow: /assets/theme/data/
Disallow: /assets/components/
Disallow: /connectors/
Disallow: /manager/
Disallow: /core/
Disallow: /speed/
Disallow: /test-calculator/
Disallow: /*?
Allow: /*.css?
Allow: /

# --- AI training/data-use agents: blocked ---
# Training/data crawlers are blocked unless Orteam explicitly opts in later.
# Google-Extended also controls Gemini grounding, not only training; keep blocked until a separate decision.

User-agent: GPTBot
Disallow: /assets/theme/data/
Disallow: /

User-agent: ClaudeBot
Disallow: /assets/theme/data/
Disallow: /

User-agent: anthropic-ai
Disallow: /assets/theme/data/
Disallow: /

User-agent: Google-Extended
Disallow: /assets/theme/data/
Disallow: /

User-agent: CCBot
Disallow: /assets/theme/data/
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /assets/theme/data/
Disallow: /

User-agent: Applebot-Extended
Disallow: /assets/theme/data/
Disallow: /

User-agent: Bytespider
Disallow: /assets/theme/data/
Disallow: /

User-agent: cohere-ai
Disallow: /assets/theme/data/
Disallow: /

User-agent: Amazonbot
Disallow: /assets/theme/data/
Disallow: /

User-agent: DeepSeekBot
Disallow: /assets/theme/data/
Disallow: /

User-agent: TikTokSpider
Disallow: /assets/theme/data/
Disallow: /

User-agent: ChatGLM-Spider
Disallow: /assets/theme/data/
Disallow: /

User-agent: PetalBot
Disallow: /assets/theme/data/
Disallow: /

User-agent: PanguBot
Disallow: /assets/theme/data/
Disallow: /

User-agent: YouBot
Disallow: /assets/theme/data/
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /assets/theme/data/
Disallow: /

User-agent: FacebookBot
Disallow: /assets/theme/data/
Disallow: /

# --- Sitemap ---
Sitemap: https://orteam.ru/sitemap.xml
Sitemap: https://orteam.ru/sitemap-uslugi.xml

# --- AI-readable site description ---
# Keep the actual LLM document at the standard root path:
# https://orteam.ru/llms.txt