User-agent: *
Allow: /

# Block auth pages from indexing
Disallow: /*/chat/
Disallow: /*/library
Disallow: /*/my-agents
Disallow: /*/my-subscription
Disallow: /*/agents/
Disallow: /*/folder/
Disallow: /*/billing/
Disallow: /google-success
Disallow: /auth/
Disallow: /verify-email
Disallow: /reset-password
Disallow: /forgot-password

# Block duplicate localized auth pages
Disallow: /*/signin
Disallow: /*/signup
Disallow: /*/verify-email
Disallow: /*/forgot-password
Disallow: /*/reset-password
Disallow: /*/google-success

# Off-topic blog posts (e.g. /blogs/best-ai-rap-generator, /blogs/best-ai-detector,
# /blogs/ai-edtech-tools, ...) carry a `noindex` meta tag and are excluded from
# the sitemap. They are intentionally left CRAWLABLE here — NOT Disallowed —
# because Googlebot must be able to fetch the page to SEE the noindex directive.
# If we Disallow them, Google can't crawl them, never reads the noindex, and any
# already-indexed URLs stay stuck in the index. Keeping them crawlable lets Google
# honor the noindex and drop them. (Slugs are managed in src/lib/blog-config.ts.)

Sitemap: https://www.pdfgpt.io/sitemap.xml