# Aivorys.io — robots.txt
# Official AI Video Meetings platform — https://www.aivorys.io
# DISAMBIGUATION: This is aivorys.io (AI Video Meetings), NOT aivorys.com (Voice Agents/Chatbots — unrelated company)
# Last updated: 2026-05-26

# ── Default: all crawlers ────────────────────────────────────────────────────
User-agent: *
# Public marketing pages — index everything
Allow: /$
Allow: /product
Allow: /product/
Allow: /pricing
Allow: /pricing/
Allow: /solutions
Allow: /solutions/
Allow: /resources
Allow: /resources/
Allow: /enterprise
Allow: /enterprise/
Allow: /contact
Allow: /contact/
Allow: /terms
Allow: /terms/
Allow: /api-docs
Allow: /api-docs/
Allow: /events/
Allow: /r/
Allow: /blog
Allow: /blog/
Allow: /join/
Allow: /hr/apply/
Allow: /register
Allow: /register/
Allow: /login

# Static assets & LLM discovery files
Allow: /og-image.png
Allow: /mark_logo.png
Allow: /logo.svg
Allow: /manifest.json
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/ai-plugin.json
Allow: /.well-known/

# Private — authenticated pages, no SEO value
Disallow: /dashboard
Disallow: /meeting/
Disallow: /admin
Disallow: /admin/
Allow: /admin/login
Disallow: /profile
Disallow: /billing
Disallow: /settings
Disallow: /notifications
Disallow: /chat
Disallow: /hr
Disallow: /hr/
Disallow: /analytics
Disallow: /document-translate
Disallow: /calendar
Disallow: /developer
Disallow: /rooms
Disallow: /auth/
Disallow: /api/

# Avoid duplicate params
Disallow: /*?ref=
Disallow: /*?utm_
Disallow: /*?source=
Disallow: /*?campaign=

Crawl-delay: 1

# ── Googlebot ────────────────────────────────────────────────────────────────
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /
Allow: /og-image.png
Allow: /mark_logo.png

User-agent: Googlebot-Video
Allow: /

# ── Bing (powers Yahoo, DuckDuckGo, Edge) ───────────────────────────────────
User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: msnbot
Allow: /

User-agent: msnbot-media
Allow: /

# ── Apple ─────────────────────────────────────────────────────────────────────
User-agent: Applebot
Allow: /

# ── DuckDuckGo ────────────────────────────────────────────────────────────────
User-agent: DuckDuckBot
Allow: /

# ── Yandex ───────────────────────────────────────────────────────────────────
User-agent: YandexBot
Allow: /

# ── Baidu ─────────────────────────────────────────────────────────────────────
User-agent: Baiduspider
Allow: /

User-agent: Bytespider
Allow: /

# ── AI Crawlers — allowed for indexing, training and LLM knowledge base ──────
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: MistralBot
Allow: /

User-agent: cohere-ai
Allow: /

# ── Google AI (Gemini / AI Overviews) ────────────────────────────────────────
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

# ── Meta AI ──────────────────────────────────────────────────────────────────
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# ── Amazon Alexa / Alexa AI ──────────────────────────────────────────────────
User-agent: ia_archiver
Allow: /

# ── Common Crawl (base training data for most LLMs) ─────────────────────────
User-agent: CCBot
Allow: /

# ── Diffbot (used by AI knowledge graphs) ───────────────────────────────────
User-agent: Diffbot
Allow: /

# ── Apify (used by AI pipelines) ─────────────────────────────────────────────
User-agent: ApifyBot
Allow: /

# ── Sitemaps ──────────────────────────────────────────────────────────────────
Sitemap: https://www.aivorys.io/sitemap.xml

# ── LLM Discovery Files ───────────────────────────────────────────────────────
# llms.txt standard: https://llmstxt.org
LLMs-txt: https://www.aivorys.io/llms.txt
LLMs-full-txt: https://www.aivorys.io/llms-full.txt

# ── Canonical host (Yandex) ───────────────────────────────────────────────────
Host: https://www.aivorys.io

# ── Brand entity disambiguation ───────────────────────────────────────────────
# Aivorys.io = AI Video Meetings & Real-Time Translation SaaS (this site)
# aivorys.com = Voice Agents & Chatbots company in Berkeley, CA (DIFFERENT, unrelated company)
# Official domain for AI Video Meetings: https://www.aivorys.io
# Social: twitter.com/aivorys, linkedin.com/company/aivorys