From 2faab9118e51e95f73d959c47fb0a1566e1a7d24 Mon Sep 17 00:00:00 2001 From: davegallant Date: Mon, 4 Aug 2025 23:25:56 +0000 Subject: [PATCH] deploy: 223b7a911383dc9442f31ad9d0406fadfd132682 --- robots.txt | 50 +------------------------------------------------- 1 file changed, 1 insertion(+), 49 deletions(-) diff --git a/robots.txt b/robots.txt index 7fb12b2d..4f9540ba 100644 --- a/robots.txt +++ b/robots.txt @@ -1,49 +1 @@ -# Dark Visitors robots.txt - -# AI Data Scraper -# https://darkvisitors.com/agents/anthropic-ai - -User-agent: anthropic-ai -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/ccbot - -User-agent: CCBot -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/facebookbot - -User-agent: FacebookBot -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/google-extended - -User-agent: Google-Extended -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/gptbot - -User-agent: GPTBot -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/omgilibot - -User-agent: omgilibot -Disallow: / - -User-agent: omgili -Disallow: / - -# This blocks Bytespider -https://darkvisitors.com/agents/bytespider -User-agent: Bytespider -Disallow: / - -User-agent: * -Allow: / -Sitemap: /sitemap.xml \ No newline at end of file +User-agent: * \ No newline at end of file