# ServerSearcher robots.txt
# Last updated: 2026-07-03

# Main search engines
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: Slurp
User-agent: Yandex
User-agent: Baidu
Allow: /

# AI answer-engine RETRIEVAL / CITATION crawlers.
# These fetch pages live to cite them in answers, so they must be welcomed to be
# eligible for citation in ChatGPT, Perplexity, Claude, and Google/Gemini AI answers.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Google-Extended
User-agent: Applebot-Extended
Allow: /

# AI TRAINING crawlers (kept allowed for brand/entity presence in future models).
User-agent: GPTBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: CCBot
User-agent: Omgilibot
Allow: /

# Social media crawlers
User-agent: Twitterbot
Allow: /
Allow: /api/og
Allow: /api/og?*

User-agent: FacebookBot
Allow: /

# Block certain paths for all crawlers
User-agent: *
Allow: /
Allow: /go/
Allow: /api/hostingsearch/
Allow: /api/hostingsearch/search
Allow: /api/hostingsearch/getinfo
Allow: /api/hostingsearch/providers/info
Allow: /api/hostingsearch/*
Disallow: /api/
Disallow: /admin/
Disallow: /auth/
Disallow: /settings/
Disallow: /*.json$
# Authenticated provider area (and the legacy path that redirects into it).
Disallow: /portal/
Disallow: /dashboard
# Legacy team URLs; kept as a safety net for anything still linking to them.
Disallow: /teams/
Disallow: /invitations/*

# Allow specific user paths
Allow: /about
Allow: /about/*
Allow: /servers/
Allow: /servers/*
Allow: /servers/provider/
Allow: /servers/provider/*
Allow: /servers/location/
Allow: /servers/location/*
Allow: /blog/
Allow: /blog/*

# Sitemap locations
Sitemap: https://www.serversearcher.com/sitemap.xml

# Crawl-delay for allowed bots
Crawl-delay: 1