# FolioForge robots.txt
User-agent: *
Allow: /
Allow: /portfolio/*
Allow: /api/og
Disallow: /api/
Disallow: /auth/
Crawl-delay: 1

# Sitemaps
Sitemap: https://folioforge.org/sitemap.xml

# AI crawlers (allow for better AI understanding)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: OAI-SearchCrawler
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Googlebot
Allow: /

# Social crawlers (allow OG previews)
User-agent: Twitterbot
Allow: /
Allow: /api/og

User-agent: facebookexternalhit
Allow: /
Allow: /api/og

User-agent: LinkedInBot
Allow: /
Allow: /api/og

User-agent: Slackbot-LinkExpanding
Allow: /
Allow: /api/og