# AI Clan Today robots.txt
User-agent: *
Allow: /

# Sitemaps
Sitemap: https://aiclans.today/sitemap.xml

# Crawl-delay
Crawl-delay: 10

# Allow all major search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

# Disallow specific paths
Disallow: /api/
Disallow: /admin/
Disallow: /private/