# WeldForge marketing site (www.weldforge.org).
# Crawlable by search engines and AI agents.
# See also: /llms.txt, /sitemap.xml, /ai-manifest.json, /ld.json.
#
# End-user auth screens live on the SSO host, not here — both on the apex
#   https://sso.weldforge.org/{login,forgot-password,reset-password,register,verify-email}
# and on each tenant's own subdomain, {slug}.sso.weldforge.org, which are now
# provisioned and hold a valid wildcard certificate.
#
# Every response from that host carries `X-Robots-Tag: noindex, nofollow`,
# deliberately: login forms don't belong in search results, and with per-tenant
# subdomains each tenant would otherwise be another indexable copy of the same
# forms. A header rather than a robots.txt file, because that host is an API
# plus an SPA with no document root — fetching https://sso.weldforge.org/robots.txt
# returns the portal's HTML, which is not a crawl permission.
#
# AI agents that need to know how end users sign in should read /llms.txt or
# /agents.html on this site rather than crawling the auth forms.
#
# /actuator/* is not routed publicly at all and there is nothing to crawl there.

User-agent: *
Allow: /

# Major AI crawlers — explicitly allowed.
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Diffbot
Allow: /

Sitemap: https://www.weldforge.org/sitemap.xml