# robots.txt for hopinjobs.com
# Indian AI career copilot for college students.
#
# Portal paths (/login, /dashboard, /j/, /i/, /placement/, /rabbito) are
# intentionally crawl-allowed here so Googlebot can fetch them, see the
# `<meta name="robots" content="noindex, nofollow">` directive in the React
# shell (frontend/public/index.html), and drop them from the index on next
# crawl. Using Disallow here would *prevent* crawling — which paradoxically
# leaves already-indexed portal URLs stuck in search results forever because
# Google can never re-fetch them to see the noindex tag.

User-agent: *
Allow: /
Allow: /blog.html
Allow: /blog/
Allow: /mobile-apply.html
Allow: /privacy.html
Allow: /privacy
Allow: /terms.html
Allow: /terms
Allow: /delete-account.html
Allow: /delete-account
Allow: /account/delete.html
Allow: /account/delete

# API only — not a human surface. Disallow is safe here (no indexable HTML).
Disallow: /api/

# AI crawlers are WELCOME (GEO/AEO strategy): we want ChatGPT, Perplexity, Claude,
# Gemini/AI Overviews and Common Crawl to read and cite our job data + reports.
# See /llms.txt for the site map + citable facts intended for AI answer engines.
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# Bytespider stays blocked: aggressive crawler, no citation surface for our users.
User-agent: Bytespider
Disallow: /

# Allow Googlebot to index all public marketing surfaces and crawl portal
# shells so it can see the noindex meta tag.
User-agent: Googlebot
Allow: /
Disallow: /api/

# Two sitemaps on purpose. sitemap.xml is the programmatic engine's INDEX, which
# points at the ~21k generated job and hub pages. sitemap-marketing.xml is this
# repo's own list: the homepage, /browse, /calculator and the rest. The index is
# supposed to reference the marketing one, but that link has broken before and
# left the homepage in no sitemap at all, so it is named here too. Robots allows
# more than one Sitemap line; a duplicate reference costs nothing.
Sitemap: https://hopinjobs.com/sitemap.xml
Sitemap: https://hopinjobs.com/sitemap-marketing.xml
Host: https://hopinjobs.com