# Default rules - restrict private areas
User-agent: *
Disallow: /notebook/*
Disallow: /dashboard
Disallow: /user/*
Disallow: /settings
Disallow: /messages
Disallow: /lessons
Disallow: /finance
Disallow: /calendar
Disallow: /contacts
Disallow: /following
Disallow: /followers

# Allow traditional search indexing (inherits default restrictions)
User-agent: Googlebot
User-agent: Bingbot
Allow: /

# Allow AI search and agent use
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: FirecrawlAgent
User-agent: AndiBot
User-agent: ExaBot
User-agent: PhindBot
User-agent: YouBot
User-agent: GPTBot
User-agent: Google-Extended
Allow: /


# Disallow AI training data collection
User-agent: CCBot
Disallow: /

Sitemap: https://www.italki.com/sitemapindex.xml