Delete themes/custom-theme/layouts/robots.txt

themes/custom-theme/layouts/robots.txt

@@ -1,53 +0,0 @@
-# Dark Visitors robots.txt
-
-# AI Data Scraper
-# https://darkvisitors.com/agents/anthropic-ai
-
-User-agent: anthropic-ai
-Disallow: /
-
-# AI Data Scraper
-# https://darkvisitors.com/agents/ccbot
-
-User-agent: CCBot
-Disallow: /
-
-# AI Data Scraper
-# https://darkvisitors.com/agents/facebookbot
-
-User-agent: FacebookBot
-Disallow: /
-
-# AI Data Scraper
-# https://darkvisitors.com/agents/google-extended
-
-User-agent: Google-Extended
-Disallow: /
-
-# AI Data Scraper
-# https://darkvisitors.com/agents/gptbot
-
-User-agent: GPTBot
-Disallow: /
-
-# AI Data Scraper
-# https://darkvisitors.com/agents/omgilibot
-
-User-agent: omgilibot
-Disallow: /
-
-User-agent: omgili
-Disallow: /
-
-# This blocks Bytespider
-https://darkvisitors.com/agents/bytespider
-User-agent: Bytespider
-Disallow: /
-
-User-agent: *
-{{ if eq (hugo.Environment) "production" -}}
-Allow: /
-{{ else -}}
-Disallow: /
-{{ end -}}
-Sitemap: {{ "sitemap.xml" | absURL -}}