# ═══════════════════════════════════════════════════════════════════
# AI access policy — generated by Piperic · piperic.com/tools/ai-bots/policy
# Goal: Allow AI answers & search; AI assistants; AI agents. Block model training; scrapers & data resellers.
#
# HOW TO USE: APPEND these groups to your EXISTING robots.txt — do NOT replace the file.
# Keep your current Sitemap: line and your other User-agent rules. Each crawler obeys only
# its most-specific matching group, so these AI-bot rules coexist with your Googlebot / * rules.
# robots.txt is advisory — bots that ignore it need Cloudflare / WAF / Pay-Per-Crawl (see the note on the page).
# ═══════════════════════════════════════════════════════════════════

# ─── Allowed (you WANT these AI crawlers) ───
# Anthropic
User-agent: Claude-SearchBot
User-agent: Claude-User
# Apple
User-agent: Applebot
# Google
User-agent: Gemini-Deep-Research
# Mistral
User-agent: MistralAI-User
# OpenAI
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: Operator
# Perplexity
User-agent: Perplexity-User
User-agent: PerplexityBot
Allow: /

# ─── Blocked AI crawlers ───
# Amazon
User-agent: Amazonbot
# Anthropic
User-agent: anthropic-ai
User-agent: ClaudeBot
# Apple
User-agent: Applebot-Extended
# ByteDance
User-agent: Bytespider
# Common Crawl
User-agent: CCBot
# Google
User-agent: Google-Extended
User-agent: GoogleOther
# Meta
User-agent: meta-externalagent
# OpenAI
User-agent: GPTBot
# Other crawlers
User-agent: AddThis
User-agent: payroll-bot
User-agent: ahrefssiteaudit
User-agent: AhrefsBot
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: aiHitBot
User-agent: AirsoftdbBot
User-agent: Claude
User-agent: AppcastBot
User-agent: archive-it
User-agent: arquivo-web-crawler
User-agent: WebSpiderMount
User-agent: Attracta
User-agent: Authory
User-agent: Awario
User-agent: AwarioSmartBot
User-agent: Barkrowler
User-agent: Bazqux
User-agent: bestchange
User-agent: bnf.fr_bot
User-agent: bigsur.ai
User-agent: Blogtrottr
User-agent: BorderxBot
User-agent: Botify
User-agent: Brandwatch
User-agent: BuddyBot
User-agent: buttondown
User-agent: TikTokSpider
User-agent: capitalonebot
User-agent: seo4ajax
User-agent: CMU-Cylab
User-agent: TerraCotta
User-agent: CitibotSiteCrawler
User-agent: cloudflare-healthchecks
User-agent: cloudflare-prefetch
User-agent: cloudflare-ssldetector
User-agent: Cloudflare-Traffic-Manager
User-agent: Daric2
User-agent: Daric3
User-agent: Daric4
User-agent: cohere-training-data-crawler
User-agent: CaliberBot
User-agent: ConvermaxBot
User-agent: CoostiBot
User-agent: CragCrawler
User-agent: Crawlspace
User-agent: artsdata-crawler
User-agent: Cxense
User-agent: DataForSEO
User-agent: dataforseobot
User-agent: DeepSeekBot
User-agent: Mediatoolkitbot
User-agent: Diffbot
User-agent: videootvbot
User-agent: w4mwnpbxf3mfabxokjrw
User-agent: EchoboxBot
User-agent: EzoicBot
User-agent: Factset_spyderbot
User-agent: fedreporterdatabot
User-agent: Feedbin
User-agent: FeedBlitz
User-agent: Feedly
User-agent: FeedOtter
User-agent: FirecrawlAgent
User-agent: fiscalnote
User-agent: FishBot
User-agent: FlipboardRSS
User-agent: FlyingPress
User-agent: follow.it
User-agent: fullfact
User-agent: GeedoShopProductFinder
User-agent: GhostExplore
User-agent: Chrome-Lighthouse
User-agent: google-calendar-importer
User-agent: googleassociationservice
User-agent: googleproducer
User-agent: storebot-google
User-agent: Google-AdWords-Express
User-agent: Google-Adwords-Instant
User-agent: inforoomreader
User-agent: Hatena
User-agent: HelloWork
User-agent: Navu
User-agent: hubspot-feedfetcher
User-agent: Idealo
User-agent: ifttt
User-agent: img2dataset
User-agent: IndeedJobBot
User-agent: innguma
User-agent: Innologica
User-agent: archive.org_bot
User-agent: ISSCyberRiskCrawler
User-agent: omim-litrack
User-agent: LAIONDownloader
User-agent: linerbot
User-agent: linkchecker.pro
User-agent: LinkCheckerBot
User-agent: AIWebIndex
User-agent: MagnetmeBot
User-agent: MJ12Bot
User-agent: Make.com
User-agent: marketgoo
User-agent: MediaMonitoringBot
User-agent: Trellis-Services
User-agent: centcombot
User-agent: FacebookBot
User-agent: MidasNetworkBot
User-agent: MirrorWebCrawler
User-agent: MonitoRSS
User-agent: KimiBot
User-agent: MotoMinerBot
User-agent: dotbot
User-agent: rogerbot-crawler
User-agent: Mozilla-Tabstack
User-agent: muckrack.com
User-agent: ClearscopeBot
User-agent: kb.dk_bot
User-agent: NewsBank
User-agent: NewsBlur
User-agent: ICC-Crawler
User-agent: NitroBot
User-agent: Nooshub
User-agent: Nostra
User-agent: novellum
User-agent: Observer
User-agent: OnCrawl
User-agent: online-webceo-bot
User-agent: OpenAI
User-agent: OpenRSS
User-agent: Oseox
User-agent: OurFamilyWizard
User-agent: Overcast
User-agent: pagefreezer
User-agent: Panscient
User-agent: panscient.com
User-agent: ShapBot
User-agent: PinterestBot
User-agent: prerender
User-agent: Protopage
User-agent: wpbot
User-agent: Querit-SearchBot
User-agent: QueritBot
User-agent: QuillBot
User-agent: quillbot.com
User-agent: rakutenusabot-image
User-agent: Feeder
User-agent: Revvim
User-agent: EchoFeed
User-agent: Cotoyogi
User-agent: SBIntuitionsBot
User-agent: scour.ing
User-agent: Screpy
User-agent: SERankingBacklinksBot
User-agent: selectikascraper
User-agent: Semrushbot
User-agent: SemrushBot-OCOB
User-agent: SemrushBotBacklinkAudit
User-agent: SemrushBotBacklinks
User-agent: SemrushBotLinkBuilding
User-agent: SemrushBotSI
User-agent: SemrushBotSwa
User-agent: SiteAuditBot
User-agent: seo-audit-check-bot
User-agent: SeobilityBot
User-agent: Zoombot
User-agent: SerpstatBot
User-agent: ServerHunterSpider
User-agent: SiteGuru
User-agent: LinkCheck
User-agent: sitecheck-sitecrawl
User-agent: SkroutzBot
User-agent: Spyglasses
User-agent: Statabot
User-agent: StatistikAustria
User-agent: StatsDroneBot
User-agent: AgentTimes
User-agent: PanguBot
User-agent: Thinkbot
User-agent: Timpibot
User-agent: TrustedSite
User-agent: TurnitinBot
User-agent: 1001FirmsBot
User-agent: 2ip
User-agent: Abonti
User-agent: AccessibleWebBot
User-agent: AccompanyBot
User-agent: AdAuth
User-agent: Adbeat
User-agent: Adsbot
User-agent: Adscanner
User-agent: AdsTxtCrawler
User-agent: adstxtlab.com
User-agent: Aegis
User-agent: AffiliateSpyBot
User-agent: AlltheWeb
User-agent: AlphaXCrawl
User-agent: Amazonbot-Video
User-agent: Ant
User-agent: ApifyBot
User-agent: ApifyWebsiteContentCrawler
User-agent: Aranet-SearchBot
User-agent: ArchiveBot
User-agent: ArchiveBox
User-agent: Arquivo.pt
User-agent: Asana
User-agent: AspiegelBot
User-agent: Backlink-Check.de
User-agent: BacklinkCrawler
User-agent: BacklinksExtendedBot
User-agent: BackupLand
User-agent: BDCbot
User-agent: BDFetch
User-agent: Birdcrawlerbot
User-agent: BitlyBot
User-agent: Bloglines
User-agent: Bloglovin
User-agent: BrandVerity
User-agent: Brightbot
User-agent: BrightEdge
User-agent: BUbiNG
User-agent: BuiltWith
User-agent: CareerBot
User-agent: Castopod
User-agent: Charlotte
User-agent: ChatGLM-Spider
User-agent: ChatGPT-Browser
User-agent: Choosito
User-agent: Cincraw
User-agent: CLASSLA-web
User-agent: Clickagy
User-agent: Cliqzbot
User-agent: CloudServerMarketSpider
User-agent: CloudVertexBot
User-agent: Cocolyzebot
User-agent: Cohere-Command
User-agent: colly
User-agent: CommaFeed
User-agent: Comscore
User-agent: Cookiebot
User-agent: CopyvioDetector
User-agent: Crawl4AI
User-agent: Crawldad
User-agent: CriteoBot
User-agent: CrowdTangle
User-agent: CrystalSemanticsBot
User-agent: DaspeedBot
User-agent: datagnionbot
User-agent: Datanyze
User-agent: Dataprovider
User-agent: deepnoc
User-agent: DepSpid
User-agent: Disqus
User-agent: DomainAppender
User-agent: DomainCrawler
User-agent: DomainStatsBot
User-agent: Doximity-Diffbot
User-agent: Dubbotbot
User-agent: DuplexWeb-Google
User-agent: eCairn-Grabber
User-agent: Entfer
User-agent: evc-batch
User-agent: Everyfeed
User-agent: ExaBot
User-agent: Example3
User-agent: Eyeotabot
User-agent: Ezooms
User-agent: Faveeo
User-agent: FeedBurner
User-agent: Feedspot
User-agent: Femtosearch
User-agent: Fever
User-agent: FindFiles.net
User-agent: Findxbot
User-agent: Flipboard
User-agent: FontRadar
User-agent: fragFINN
User-agent: FreshRSS
User-agent: FriendlyCrawler
User-agent: Functionize
User-agent: Gaisbot
User-agent: GDNP
User-agent: GeedoBot
User-agent: Gemini-AI
User-agent: Golfe
User-agent: Google-ilp
User-agent: Google-Read-Aloud
User-agent: Google-Safety
User-agent: Gowikibot
User-agent: Gozle
User-agent: Grapeshot
User-agent: Gregarius
User-agent: GTmetrix
User-agent: HaloBot
User-agent: Headline
User-agent: HenkBot
User-agent: Heritrix
User-agent: htmlyse
User-agent: httpx
User-agent: HuaweiWebCatBot
User-agent: HubPages
User-agent: HubSpot
User-agent: HubSpotContentSearchBot
User-agent: HypeStat
User-agent: iAskBot
User-agent: iaskspider
User-agent: IDG
User-agent: ImageSift
User-agent: imageSpider
User-agent: Infegy
User-agent: InfoTigerBot
User-agent: Inspici
User-agent: InsytfulBot
User-agent: InternetMeasurement
User-agent: IsItWP
User-agent: iTMS
User-agent: JobboerseBot
User-agent: JungleKeyThumbnail
User-agent: KadoBot
User-agent: Keys.so
User-agent: KlarnaBot
User-agent: KomodiaBot
User-agent: Konturbot
User-agent: Kouio
User-agent: Kozmonavt
User-agent: KStandBot
User-agent: KunatoCrawler
User-agent: laion-huggingface-processor
User-agent: LCC
User-agent: leak.info
User-agent: LetSearch
User-agent: LightspeedSystemsCrawler
User-agent: LinkBloom
User-agent: LinkpadBot
User-agent: LinkWalker
User-agent: LiveJournal
User-agent: Lumar
User-agent: LumeWebScan
User-agent: LumtelBot
User-agent: MaCoCu
User-agent: MADBbot
User-agent: MakeMerryBot
User-agent: Marginalia
User-agent: masscan
User-agent: masscan-ng
User-agent: Mediumbot
User-agent: MegaIndex
User-agent: MeltwaterNews
User-agent: Meta-ExternalAds
User-agent: MetaInspector
User-agent: MetaJobBot
User-agent: MicroAdBot
User-agent: Mixnode
User-agent: Monsidobot
User-agent: MTRobot
User-agent: MuckRack
User-agent: MuscatFerret
User-agent: MyCentralAIScraperBot
User-agent: najdu.s.holubem.eu
User-agent: NalezenCzBot
User-agent: NameProtectBot
User-agent: nbertaupete95
User-agent: netEstate
User-agent: NetTrack
User-agent: Netvibes
User-agent: NETZZAPPEN
User-agent: NewsGator
User-agent: Newslitbot
User-agent: NiceCrawler
User-agent: NLCrawler
User-agent: Nuzzel
User-agent: OI-Crawler
User-agent: Onalytica
User-agent: OpenGraph.io
User-agent: OpenLinkProfiler
User-agent: OpenRobotsTXT
User-agent: OpenWebSpider
User-agent: OSZKbot
User-agent: Outbrain
User-agent: OWLer
User-agent: Pageburst
User-agent: PagePeeker
User-agent: PageThing
User-agent: Pandalytics
User-agent: Paqlebot
User-agent: parse.ly
User-agent: ParticleNewsBot
User-agent: Peer39
User-agent: Pigafetta
User-agent: Pinterest
User-agent: PiplBot
User-agent: PlayStore-Google
User-agent: Plukkie
User-agent: Pompos
User-agent: PritTorrent
User-agent: Quantcast
User-agent: QuerySeekerSpider
User-agent: Rainmeter
User-agent: RavenCrawler
User-agent: RedekenBot
User-agent: ReqBin
User-agent: Rogerbot
User-agent: RSiteAuditor
User-agent: Ryowl
User-agent: SabsimBot
User-agent: SafeDNSBot
User-agent: SBIder
User-agent: ScreenerBot
User-agent: Sellers.Guide
User-agent: semaltbot
User-agent: Senuto
User-agent: Seobility
User-agent: SEOENGBot
User-agent: SEOkicks
User-agent: SeolytBot
User-agent: Seoscanners.net
User-agent: sfFeedReader
User-agent: ShopAlike
User-agent: SilverReader
User-agent: Sirdata
User-agent: Sitebulb
User-agent: SiteCheckerBotCrawler
User-agent: SitemapParser-VIPnytt
User-agent: SiteScore
User-agent: SiteSucker
User-agent: Slackbot
User-agent: Sparkler
User-agent: Spider
User-agent: SpiderLing
User-agent: Spinn3r
User-agent: SplitSignalBot
User-agent: Sprinklr
User-agent: sqalix
User-agent: StackScopeBot
User-agent: start.me
User-agent: Statista
User-agent: StatOnline.ru
User-agent: StatusNestBacklinkSpider
User-agent: Stract
User-agent: Sublinq
User-agent: SuggestBot
User-agent: SurdotlyBot
User-agent: svEye
User-agent: Swiftbot
User-agent: Synapse
User-agent: t3versions
User-agent: Taboolabot
User-agent: TactiScout
User-agent: TavilyBot
User-agent: tchelebi
User-agent: TestCrawler
User-agent: TigerBot
User-agent: TinEye
User-agent: Trendsmap
User-agent: TrustBot
User-agent: Twurly
User-agent: uMBot
User-agent: UniversalFeedParser
User-agent: URLAppendBot
User-agent: URLSuMaBot
User-agent: Vagabondo
User-agent: ValidBot
User-agent: VeryHip
User-agent: VORTEX
User-agent: WebbCrawler
User-agent: WebCEO
User-agent: WebDataStats
User-agent: Webliobot
User-agent: WebPros
User-agent: Website-info
User-agent: webtru
User-agent: Webwiki
User-agent: webzio
User-agent: Webzio-Extended
User-agent: WellKnownBot
User-agent: WeViKaBot
User-agent: WhatCMS
User-agent: WhereGoes
User-agent: Wibybot
User-agent: WireReaderBot
User-agent: Workona
User-agent: WRTNBot
User-agent: XoviBot
User-agent: xtate
User-agent: YandexBot-MirrorDetector
User-agent: ygs-scraper-bot
User-agent: Yourls
User-agent: Zaldamo
User-agent: Zao
User-agent: Zeno
User-agent: Zookabot
User-agent: ZoominfoBot
User-agent: upday
User-agent: VelenPublicWebCrawler
User-agent: BLEXBot
User-agent: WARDBot
User-agent: omgili
User-agent: Pricey
User-agent: w3c-checklink
User-agent: xy-archive-compliance
User-agent: j-brw
User-agent: YandexAdditional
User-agent: YandexAdditionalBot
User-agent: rss2tg_bot
User-agent: Scrapy
Disallow: /

# Note: 218 entries were omitted — their user-agent is a descriptive name (with spaces or
# symbols), not a standards-valid robots.txt token, so a rule for them would have no effect.