# Find Feny robots.txt
# Allows AI platforms to access content for citations and visibility

# Default: Allow all crawlers
User-agent: *
Allow: /

# Search Engine Crawlers
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# Social Media Crawlers (for preview cards)
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

# AI Platform Crawlers - Explicitly Allowed
User-agent: GPTBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

# Priority Pages - Explicitly Allowed
Allow: /
Allow: /library
Allow: /library/*
Allow: /restaurants
Allow: /restaurants/*
Allow: /feny-verify
Allow: /how-it-works
Allow: /llms.txt
Allow: /feny-verify/llms.txt

# Security: Disallow admin and private areas
Disallow: /admin/
Disallow: /private/
Disallow: /api/
Disallow: /ingest/
Disallow: /*.json

# Utility pages: keep out of search index (low-value impressions)
Disallow: /privacy
Disallow: /terms
Disallow: /cookies

# Performance: Crawl delay for non-priority bots
Crawl-delay: 1

# Sitemap reference
Sitemap: https://www.findfeny.com/sitemap.xml