# LinkedIn Pinpoint Answer Today - Robots.txt
# This file tells search engine crawlers which pages they can and cannot access

# General search engine rules
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /private/

# Sitemap
Sitemap: https://pinpointanswer.online/sitemap.xml

# AI-specific crawler identifiers
User-agent: GPTBot
User-agent: Claude-Web
User-agent: Anthropic-AI
User-agent: PerplexityBot
User-agent: GoogleOther
User-agent: DuckAssistBot

# Point AI crawlers to LLM guidelines/content
LLM-Content: https://pinpointanswer.online/llms.txt

# Pages allowed for AI crawlers
Allow: /
Allow: /pinpoint-answer
Allow: /sessions
Allow: /privacy-policy
Allow: /terms-of-service

# Pages disallowed for AI crawlers
Disallow: /api/
Disallow: /admin/
Disallow: /private/

# Crawl delay (optional)
Crawl-delay: 1