# === AI search / answer bots: allowed for GEO visibility ===
# These bots help BizNet content appear in AI search and answer experiences.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Google-Extended
Disallow: /user/
Disallow: /messages/
Disallow: /main/search/
Disallow: /main/error
Disallow: /sites/add
Disallow: /otvet/add
Disallow: /business/add
Disallow: /articles/add
Disallow: /groups/add
Disallow: /groups/create_topic/
Disallow: /cdn-cgi/
Disallow: /forums/startTopic/
Disallow: /forums/discover/unread/
Disallow: /forums/markallread/
Disallow: /forums/staff/
Disallow: /forums/online/
Disallow: /forums/discover/
Disallow: /forums/leaderboard/
Disallow: /forums/search/
Disallow: /forums/*?advancedSearchForm=
Disallow: /forums/register/
Disallow: /forums/lostpassword/
Disallow: /forums/login/
Disallow: /forums/*?sortby=
Disallow: /forums/*?filter=
Disallow: /forums/*?tab=
Disallow: /forums/*?do=
Disallow: /forums/*ref=
Disallow: /forums/*?forumId*
Disallow: /forums/profile/

# === AI training / low-value AI bots: blocked ===
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

# Legacy Claude crawler token kept blocked until verified as search/user-triggered.
User-agent: Claude-Web
Disallow: /

# === Aggressive / low-value commercial crawlers: blocked ===
User-agent: Amazonbot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: SurdotlyBot
Disallow: /

User-agent: adsbot
Disallow: /

User-agent: CriteoBot/0.1
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: trendictionbot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SiteAuditBot
Disallow: /

User-agent: SemrushBot-BA
Disallow: /

User-agent: SemrushBot-SI
Disallow: /

User-agent: SemrushBot-SWA
Disallow: /

User-agent: SemrushBot-CT
Disallow: /

User-agent: SemrushBot-BM
Disallow: /

User-agent: SplitSignalBot
Disallow: /

User-agent: *
Disallow: /user/
Disallow: /messages/
Disallow: /main/search/
Disallow: /main/error
#Disallow: /bb/add
Disallow: /sites/add
Disallow: /otvet/add
Disallow: /business/add
Disallow: /articles/add
Disallow: /groups/add
Disallow: /groups/create_topic/
Disallow: /cdn-cgi/

# Rules for Invision Community (https://invisioncommunity.com)
# Block pages with no unique content
Disallow: /forums/startTopic/
Disallow: /forums/discover/unread/
Disallow: /forums/markallread/
Disallow: /forums/staff/
Disallow: /forums/online/
Disallow: /forums/discover/
Disallow: /forums/leaderboard/
Disallow: /forums/search/
Disallow: /forums/*?advancedSearchForm=
Disallow: /forums/register/
Disallow: /forums/lostpassword/
Disallow: /forums/login/

# Block faceted pages and 301 redirect pages
Disallow: /forums/*?sortby=
Disallow: /forums/*?filter=
Disallow: /forums/*?tab=
Disallow: /forums/*?do=
Disallow: /forums/*ref=
Disallow: /forums/*?forumId*

# Block profile pages as these have little unique value, consume a lot of crawl time and contain hundreds of 301 links
Disallow: /forums/profile/

# Sitemap URLs
Sitemap: https://biznet.guru/sitemap/
Sitemap: https://biznet.guru/forums/sitemap.php

Crawl-delay: 2

User-agent: bingbot
Crawl-delay: 2

User-agent: msnbot
Crawl-delay: 2

User-agent: Slurp
Crawl-delay: 2

User-agent: MJ12bot
Crawl-delay: 2

User-agent: AhrefsBot
Crawl-delay: 2