User-agent: * Allow: / # Disallow admin areas Disallow: /backstage/ Disallow: /django-backstage/ # Disallow search endpoints (server-side search results = duplicate content) Disallow: /search/ Disallow: /directory/search/ Disallow: /directory/lenders/search/ Disallow: /directory/certifying-bodies/search/ Disallow: /federal-spending/search/ Disallow: /suppliers/search/ # Disallow unclaimed supplier index (SAM.gov imports with no editorial value) Disallow: /suppliers/unclaimed/ # Disallow only the search query parameter (other filter params expose # legitimate long-tail content; pagination must remain crawlable so # Googlebot can discover items on page 2+). Disallow: /*?q= Disallow: /*?search= Disallow: /*?agency= # ?fy= is intentionally NOT blocked site-wide: each fiscal year on # the federal-spending dashboard and agency pages is a distinct # Dataset with different temporalCoverage. Block only the search # endpoint's fy filter (which IS true duplicate-with-pagination). Disallow: /federal-spending/search/*?fy= # Expiring contracts explorer: filtered views, the CSV export and the # alert form are tools, not pages (the unfiltered explorer stays open). Disallow: /contracts/expiring/explorer/? Disallow: /contracts/expiring/explorer.csv Disallow: /contracts/expiring/alerts/ # Disallow private/authenticated areas Disallow: /accounts/ Disallow: /messages/ Disallow: /dashboard/ Disallow: /buyers/ Disallow: /vault/ Disallow: /ops/ Disallow: /email/ Disallow: /desk/ Disallow: /consultants/ Disallow: /services/start/ Disallow: /services/checkout/ Disallow: /services/thanks/ Disallow: /services/link/ Disallow: /services/owner/ Disallow: /services/offer/ Disallow: /services/capability-statement-pro/ Disallow: /services/certification-roadmap/ Disallow: /tier2/ Disallow: /monitoring/ # Subscriptions: pricing root is public/indexable; private flows are not. Disallow: /subscriptions/billing/ Disallow: /subscriptions/upgrade/ Disallow: /subscriptions/cancel/ Disallow: /subscriptions/usage/ Disallow: /subscriptions/checkout/ Disallow: /subscriptions/portal/ Disallow: /subscriptions/test/ # CertifyAll: landing is public/indexable; the application flow is not. Disallow: /certifyall/dashboard/ Disallow: /certifyall/profile/ Disallow: /certifyall/owners/ Disallow: /certifyall/documents/ Disallow: /certifyall/order/ Disallow: /outcomes/journey/ Disallow: /go/ Disallow: /sponsor/ # Sitemap location Sitemap: https://www.supplierdiversity.com/sitemap.xml