# robots.txt - Society of Teen Scientists
# Updated: 2026-01-12

# Note: each named bot below obeys only its own group and ignores "User-agent: *",
# so the symposium-media Disallow lines must be repeated in every group.

# OpenAI (ChatGPT, SearchGPT)
User-agent: GPTBot
Allow: /
Crawl-delay: 1
Disallow: /video/
Disallow: /images/symposium/

User-agent: ChatGPT-User
Allow: /
Disallow: /video/
Disallow: /images/symposium/

# Anthropic (Claude)
User-agent: anthropic-ai
Allow: /
Crawl-delay: 1
Disallow: /video/
Disallow: /images/symposium/

User-agent: Claude-Web
Allow: /
Disallow: /video/
Disallow: /images/symposium/

# Common Crawl (Used by many AI models)
User-agent: CCBot
Allow: /
Crawl-delay: 1
Disallow: /video/
Disallow: /images/symposium/

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Crawl-delay: 1
Disallow: /video/
Disallow: /images/symposium/

# Google AI (Bard, Gemini)
User-agent: Google-Extended
Allow: /
Disallow: /video/
Disallow: /images/symposium/

# Bing AI
User-agent: BingBot
Allow: /
Disallow: /video/
Disallow: /images/symposium/

# General crawlers
User-agent: *
Allow: /

# Disallow private/authenticated routes
# Allow public profile pages first (more specific rules take precedence)
Allow: /profile-research-journey
Disallow: /profile
Disallow: /events/info-session
Disallow: /account/
Disallow: /checkout/
Disallow: /admin/
Disallow: /api/auth/
Disallow: /journal/admin/
Disallow: /research/
Disallow: /my-lab/
Disallow: /labs/manage/
Disallow: /subscribe/
Disallow: /join-us

# Keep symposium media out of Google Images / Video results (don't let crawlers copy them)
Disallow: /video/
Disallow: /images/symposium/

# Allow API documentation (when available)
Allow: /api/articles.json
Allow: /api/issues.json

# Sitemaps
Sitemap: https://teenscientists.org/sitemap.xml