# Last updated: 20250618

# Block all unknown crawlers
User-agent: *
Disallow: /

# Allow major search engine crawlers
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Mobile
User-agent: Google-InspectionTool
User-agent: Bingbot
User-agent: msnbot
User-agent: Slurp
User-agent: Baiduspider
User-agent: DuckDuckBot

# Allow trusted SEO tools
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: Screaming Frog SEO Spider
User-agent: Moz

# Allow social media bots for link previews
User-agent: facebookexternalhit
User-agent: Facebot
User-agent: Twitterbot
User-agent: TelegramBot

# Allow AI and language model crawlers
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OpenAI-User
User-agent: AnthropicBot
User-agent: DeepSeekBot
User-agent: Google-Extended
User-agent: PerplexityBot
User-agent: youBot

# Allow Apple, Amazon, Alexa bots
User-agent: Applebot
User-agent: ia_archiver
User-agent: Amazonbot
User-agent: archive.org_bot

# Allowed paths for all above bots
Allow: /index.php
Allow: /welcome*.php
Allow: /images/club99_link_logo.png
Allow: /welcome-images/
Allow: /sitemap*.xml

# Sitemap declaration (not in use, remarked)
# Sitemap: https://www.club99.com/sitemap.xml