# ============================================================
# robots.txt — Verified Biz Exchange (www.verify.ukp.co.ke)
# ============================================================
# This file tells search engines and other automated crawlers
# which parts of the site are open for indexing and which are
# not. It works together with — not instead of — the server-
# side anti-scraping, rate-limiting, and X-Robots-Tag headers
# already enforced by every page (see Privacy Policy, Section
# on Security: /privacy#security).
#
# Disallowing a path here is a request, not a lock. Sensitive
# systems (identity documents, the admin console, payment
# callbacks) are additionally protected by authentication,
# session checks, and rate limits regardless of what this file
# says.
#
# This file is the technical counterpart to three human-facing
# pages, which explain WHY each rule below exists:
#   /sitemap#private   — "Pages We Keep Private" (Section 11)
#   /privacy#security   — "How We Protect Your Data" (Section 15)
#   /terms#ip           — "Intellectual Property" (Section 14)
# ============================================================

# ── DEFAULT: ALL CRAWLERS ─────────────────────────────────────
User-agent: *

# Public marketplace, franchise, company, and legal pages —
# openly indexable.
Allow: /
Allow: /listings
Allow: /listing
Allow: /franchises
Allow: /franchise-post
Allow: /about
Allow: /contact
Allow: /advertise
Allow: /pricing
Allow: /verify
Allow: /terms
Allow: /privacy
Allow: /sitemap

# Signed-in dashboard state — same URL as the public sign-in
# page (/login), but this query state renders private account
# data and must never be indexed.
Disallow: /login?dashboard=*
Disallow: /login?*action=logout*

# Seller/franchisor verification: the application form, payment
# step, and post-payment status screens are session-specific
# and produce no useful content for a crawler.
Disallow: /verify?page=apply*
Disallow: /verify?page=payment*
Disallow: /verify?page=status*
Disallow: /verify?page=track*

# Verification staff console and every admin sub-view — internal
# only, never meant to be discoverable.
Disallow: /verify?page=admin*
Disallow: /verify?page=admin_dashboard*
Disallow: /verify?page=admin_view*

# Backend action handlers and data endpoints — these return
# JSON or binary data, not pages, and require an authenticated
# session in any case.
Disallow: /login_api.php
Disallow: /id_verify.php
Disallow: /analytics_data.php
Disallow: /UKPMfaManager.php
Disallow: /UKPTotp.php

# Invoices are generated per-transaction for a signed-in user
# and opened via a POST request — there is nothing here for a
# crawler to fetch or index.
Disallow: /invoice

# Internal photo/logo streaming routes (serve images through
# the app rather than a static path — see listing/franchise
# detail pages). Indexing the route itself serves no purpose;
# the images are already visible on their listing pages.
Disallow: /*?action=serve_photo*
Disallow: /*?action=serve_logo*
Disallow: /*?action=serve_id_card*

# Configuration, uploaded documents, and any other server-side
# files. These are not served as static paths by this host in
# the first place, but are excluded here as defence in depth.
Disallow: /config/
Disallow: /uploads/

Crawl-delay: 5

# ── KNOWN AGGRESSIVE / NON-SEARCH CRAWLERS ────────────────────
# These bots provide no benefit to a Kenyan business marketplace
# (SEO-tool scrapers, uptime/security scanners) and are blocked
# outright, matching the user-agent list already enforced at
# the application layer.
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: RogerBot
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SeekportBot
Disallow: /

# ── SITEMAP ────────────────────────────────────────────────────
Sitemap: https://www.verify.ukp.co.ke/sitemap.xml