# =============================================================================
# robots.txt for golfi.ai
# Crawler access policy — last reviewed 2026-04-28
# =============================================================================

# Canonical sitemap (declared once, applies to all user-agents)
Sitemap: https://golfi.ai/sitemap.xml


# -----------------------------------------------------------------------------
# 1. Major search engines — allow all public content, block private areas
#    and tracking-parameter URLs that create duplicate content.
# -----------------------------------------------------------------------------
User-agent: Googlebot
Allow: /
# API + server routes
Disallow: /api/
# Admin & partner back-office
Disallow: /admin
Disallow: /admin/
Disallow: /partner
Disallow: /partner/
# Authenticated user areas (private data)
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /auth
Disallow: /reset-password
Disallow: /booking-status
Disallow: /games/
Disallow: /welcome/
Disallow: /billing/
Disallow: /private/
# Framework internals
Disallow: /_next/
# Duplicate-content query params (tracking / referrals)
Disallow: /*?ref=
Disallow: /*?utm_
Disallow: /*&utm_

User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /admin
Disallow: /admin/
Disallow: /partner
Disallow: /partner/
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /auth
Disallow: /reset-password
Disallow: /booking-status
Disallow: /games/
Disallow: /welcome/
Disallow: /billing/
Disallow: /private/
Disallow: /_next/
Disallow: /*?ref=
Disallow: /*?utm_
Disallow: /*&utm_

User-agent: DuckDuckBot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /partner/
Disallow: /dashboard/
Disallow: /private/


# -----------------------------------------------------------------------------
# 2. Social / preview crawlers — full access so link previews render correctly.
# -----------------------------------------------------------------------------
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /


# -----------------------------------------------------------------------------
# 3. AI / LLM training crawlers — blocked.
#    We do not permit our content to be used for model training or scraping.
# -----------------------------------------------------------------------------
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /


# -----------------------------------------------------------------------------
# 4. Aggressive SEO / analytics bots — throttled to reduce server load.
# -----------------------------------------------------------------------------
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 10


# -----------------------------------------------------------------------------
# 5. Default policy — every other crawler.
#    Public content is allowed; private areas and duplicate-content URLs blocked.
# -----------------------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin
Disallow: /admin/
Disallow: /partner
Disallow: /partner/
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /auth
Disallow: /reset-password
Disallow: /booking-status
Disallow: /games/
Disallow: /welcome/
Disallow: /billing/
Disallow: /private/
Disallow: /_next/
Disallow: /*?ref=
Disallow: /*?utm_
Disallow: /*&utm_