# ============================================================
# VeteranOwnedBusiness.com Robots.txt
# ============================================================

Sitemap: https://www.veteranownedbusiness.com/sitemap-index.xml


# ------------------------------------------------------------
# Apple
# ------------------------------------------------------------

User-agent: Applebot
Crawl-delay: 20
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /search.php
Disallow: /service_comments.php

User-agent: Applebot-Extended
Crawl-delay: 20
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php


# ------------------------------------------------------------
# Amazon
# ------------------------------------------------------------

User-agent: Amazonbot
Crawl-delay: 30
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php


# ------------------------------------------------------------
# OpenAI
# ------------------------------------------------------------

User-agent: GPTBot
Crawl-delay: 30
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php

User-agent: OAI-SearchBot
Crawl-delay: 30
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php


# ------------------------------------------------------------
# Anthropic
# ------------------------------------------------------------

User-agent: ClaudeBot
Crawl-delay: 30
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php

User-agent: Claude-SearchBot
Crawl-delay: 30
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php

User-agent: Claude-User
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php

User-agent: Anthropic
Crawl-delay: 30
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php


# ------------------------------------------------------------
# Other AI / LLM Crawlers
# ------------------------------------------------------------

User-agent: PerplexityBot
Crawl-delay: 30
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php

User-agent: NeevaBot
Crawl-delay: 30
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php


# ------------------------------------------------------------
# AI / Dataset Crawlers - Heavily Throttled
# ------------------------------------------------------------

# Parallel AI / ShapBot
# Excessive crawl activity observed
User-agent: ShapBot
Crawl-delay: 60
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php

# Common Crawl
User-agent: CCBot
Crawl-delay: 30
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php

# ByteDance
User-agent: Bytespider
Crawl-delay: 30
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php

# Meta AI crawler
User-agent: meta-externalagent
Crawl-delay: 30
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php


# ------------------------------------------------------------
# Google AI
# ------------------------------------------------------------

# Google-Extended controls certain Gemini/AI uses.
# It does not control normal Google Search crawling.
User-agent: Google-Extended
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php


# ------------------------------------------------------------
# SEO / Heavy Commercial Crawlers
# ------------------------------------------------------------

User-agent: AhrefsBot
Crawl-delay: 20
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php

User-agent: SemrushBot
Crawl-delay: 20
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php

User-agent: MJ12bot
Crawl-delay: 30
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php


# ------------------------------------------------------------
# Other Search Engines
# ------------------------------------------------------------

User-agent: Baiduspider
Crawl-delay: 20
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php

User-agent: Yandex
Crawl-delay: 20
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php
Disallow: /member-share.php
Disallow: /search.php
Disallow: /service_comments.php


# ------------------------------------------------------------
# Google, Bing, Social Crawlers and Everyone Else
# ------------------------------------------------------------

User-agent: *
Allow: /
Disallow: /includes/layout/map.php
Disallow: /business_contact.php
Disallow: /business_error.php