# ============================================================
# Ubuntu Guard — robots.txt
# Last updated: 2026-05-05
# ============================================================

# --- AI Crawlers (explicitly welcome) ---
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Amazonbot
User-agent: Google-Extended
User-agent: Gemini-crawler
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: FacebookBot
User-agent: Applebot
User-agent: Bytespider
User-agent: Diffbot
User-agent: CCBot
User-agent: cohere-ai
User-agent: AI2Bot
User-agent: Timpibot
Allow: /
Disallow: /api/
Disallow: /inc/
Disallow: /sql/
Disallow: /data/
Disallow: /scamnet/api/
Disallow: /clickguard/api/
Disallow: /clickguard/admin/
Disallow: /clickguard/logs/
Disallow: /goldphish/php/
Disallow: /goldphish/data/
Disallow: /scamnet/login.html
Disallow: /scamnet/dash.html
Disallow: /scamnet/report.html
Disallow: /old_pass/
Disallow: /.vscode/
Disallow: /.claude/
Disallow: /cdn-cgi/
Disallow: /cyber-toolkit/blog/
# --- CheckURLink V2 (deployed at /checkurlink/) ---
# Public pages (homepage, privacy, terms) are crawlable; the rest is blocked.
# /api returns JSON — never useful in search results.
# /app, /data, /log are also blocked at the server level via .htaccess; these
# entries are defence-in-depth so crawlers don't waste budget probing them.
# /r/ is the user-generated permalink resolver — each id is a per-scan snapshot
# that expires after 90 days, so indexing them would just feed Google a stream
# of stale 410s. Sharing on WhatsApp / email still works either way.
Disallow: /checkurlink/api/
Disallow: /checkurlink/app/
Disallow: /checkurlink/data/
Disallow: /checkurlink/log/
Disallow: /checkurlink/r/

# --- All other crawlers (Google, Bing, etc.) ---
User-agent: *
Allow: /
Disallow: /api/
Disallow: /inc/
Disallow: /sql/
Disallow: /data/
Disallow: /scamnet/api/
Disallow: /clickguard/api/
Disallow: /clickguard/admin/
Disallow: /clickguard/logs/
Disallow: /goldphish/php/
Disallow: /goldphish/data/
Disallow: /scamnet/login.html
Disallow: /scamnet/dash.html
Disallow: /scamnet/report.html
Disallow: /old_pass/
Disallow: /.vscode/
Disallow: /.claude/
Disallow: /cdn-cgi/
Disallow: /cyber-toolkit/blog/
# CheckURLink V2 (see comment block above for rationale)
Disallow: /checkurlink/api/
Disallow: /checkurlink/app/
Disallow: /checkurlink/data/
Disallow: /checkurlink/log/
Disallow: /checkurlink/r/

Sitemap: https://www.ubuntuguard.co.za/sitemap.xml