User-agent: *
Allow: /
# The console is a staff tool and never a search result. The pages
# carry `noindex` too; this line keeps a crawler from asking at all.
Disallow: /admin
# The customer dashboard is signed in and holds a person's services,
# bills and keys. It carries `noindex` and an `X-Robots-Tag` too; the
# disallow keeps a crawler from fetching a page it could never index.
Disallow: /app
Disallow: /app/api

# Per AGENTS.md §9 + the LLM-crawler discoverability rule:
# the public marketing surface is open. The /api and /dashboard
# routes are intentionally absent from this site (R220
# marketing-only cut, R221 unchanged) so no Disallow lines
# are needed for them. The signed-in dashboard at /app is covered by
# its own `noindex` and the disallow above.

Sitemap: https://www.webhost24.in/sitemap-index.xml

# The crawlers we block. The list is the one
# /robots.txt.disallow publishes, so the reference file and this file
# always name the same crawlers.
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# AI training crawlers: explicit allowlist for the well-known
# LLM discovery surfaces. These crawlers respect robots.txt
# for opt-in/opt-out semantics.
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: Bytespider
Allow: /

Host: www.webhost24.in