# ===========================
# 3PL Hub Robots.txt
# Purpose: Allow full indexing by search engines and LLMs
# Block search result pages to prevent thin/duplicate content indexing
# Sitemap is the curated index served by the jobs Worker (filters BD noindex/redirect/blocked URLs)
# ===========================

User-agent: *
Disallow: /api/
# Public profile styles, scripts and logo assets needed for rendering.
Allow: /api/widget/html/json/3pl-profile-assets-v2?asset=
Allow: /api/widget/html/json/3pl-profile-logos-v1?asset=
Disallow: /admin/
Disallow: /members/
Disallow: /account/
Disallow: /passwordreset/
Disallow: /search_results
Disallow: /3pl-directory?
Disallow: /sitesearch
Disallow: /blog?
Disallow: /*/reviews$
Disallow: /*/connect$
Disallow: /*/writeareview$
Disallow: /events?
Disallow: /classifieds?
Disallow: /login/retrieval
Disallow: /join?claim
Disallow: /checkout/
Disallow: /events/*/
Allow: /
Crawl-delay: 5
Sitemap: https://www.3plhub.co/sitemap.xml

# ---- Google Crawlers ----
User-agent: Googlebot
Allow: /
Disallow: /api/
# Public profile styles, scripts and logo assets needed for rendering.
Allow: /api/widget/html/json/3pl-profile-assets-v2?asset=
Allow: /api/widget/html/json/3pl-profile-logos-v1?asset=
Disallow: /admin/
Disallow: /members/
Disallow: /account/
Disallow: /passwordreset/
Disallow: /search_results
Disallow: /3pl-directory?
Disallow: /sitesearch
Disallow: /blog?
Disallow: /*/reviews$
Disallow: /*/connect$
Disallow: /*/writeareview$
Disallow: /events?
Disallow: /classifieds?
Disallow: /login/retrieval
Disallow: /join?claim
Disallow: /checkout/
Disallow: /events/*/

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Mediapartners-Google
Allow: /

# ---- Bing / Microsoft ----
User-agent: Bingbot
Allow: /
Disallow: /api/
# Public profile styles, scripts and logo assets needed for rendering.
Allow: /api/widget/html/json/3pl-profile-assets-v2?asset=
Allow: /api/widget/html/json/3pl-profile-logos-v1?asset=
Disallow: /admin/
Disallow: /members/
Disallow: /account/
Disallow: /passwordreset/
Disallow: /search_results
Disallow: /3pl-directory?
Disallow: /sitesearch
Disallow: /blog?
Disallow: /*/reviews$
Disallow: /*/connect$
Disallow: /*/writeareview$
Disallow: /events?
Disallow: /classifieds?
Disallow: /login/retrieval
Disallow: /join?claim
Disallow: /checkout/
Disallow: /events/*/

User-agent: BingPreview
Allow: /

# ---- Yahoo ----
User-agent: Slurp
Allow: /
Disallow: /api/
# Public profile styles, scripts and logo assets needed for rendering.
Allow: /api/widget/html/json/3pl-profile-assets-v2?asset=
Allow: /api/widget/html/json/3pl-profile-logos-v1?asset=
Disallow: /admin/
Disallow: /members/
Disallow: /account/
Disallow: /passwordreset/
Disallow: /search_results
Disallow: /3pl-directory?
Disallow: /sitesearch
Disallow: /blog?
Disallow: /*/reviews$
Disallow: /*/connect$
Disallow: /*/writeareview$
Disallow: /events?
Disallow: /classifieds?
Disallow: /login/retrieval
Disallow: /join?claim
Disallow: /checkout/
Disallow: /events/*/

# ---- DuckDuckGo ----
User-agent: DuckDuckBot
Allow: /

# ---- Social / Preview Bots ----
User-agent: Twitterbot
Allow: /
User-agent: FacebookExternalHit
Allow: /
User-agent: LinkedInBot
Allow: /

# ---- AI / LLM Crawlers ----
User-agent: ChatGPT-User
Allow: /
User-agent: ChatGPT
Allow: /
User-agent: GPTBot
Allow: /
User-agent: anthropic-ai
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: YouBot
Allow: /
User-agent: Google-Extended
Allow: /
User-agent: CCBot
Allow: /

# ---- Archive / Reference Crawlers ----
User-agent: archive.org_bot
Allow: /
User-agent: ia_archiver
Allow: /

# ---- End ----