# StackItSmart Robots.txt
# Updated: November 12, 2025

# Allow all search engines to crawl public pages
User-agent: *
Allow: /
Allow: /cycle-builder
Allow: /research
Allow: /about
Allow: /privacy-policy

# Block private/authenticated pages
Disallow: /dashboard/
Disallow: /api/
Disallow: /auth/

# Block admin or config pages if they exist
Disallow: /admin/
Disallow: /_next/

# Sitemap location
Sitemap: https://stackitsmart.com/sitemap.xml

# Crawl-delay for aggressive bots (optional)
User-agent: GPTBot
Crawl-delay: 10

# Block AI training scrapers (optional - protect content)
User-agent: CCBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Google-Extended
Disallow: /

# Common crawlers
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /