# robots.txt for kluster.ai
# Allow all search engines to crawl the site

User-agent: *
Allow: /

# Sitemap location
Sitemap: https://kluster.ai/sitemap.xml

# Block crawling of sensitive areas (if any exist in the future)
# Disallow: /admin/
# Disallow: /api/private/

# Allow specific crawling of important areas
Allow: /blog/
Allow: /about/
Allow: /privacy-policy/
Allow: /terms-of-use/

# Crawl delay for being respectful to server resources
Crawl-delay: 1

# Google-specific directives
User-agent: Googlebot
Allow: /
Crawl-delay: 0

# Bing-specific directives  
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Block known bad bots (optional - can be removed if not needed)
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /