diff --git a/themes/custom-theme/layouts/robots.txt b/themes/custom-theme/layouts/robots.txt deleted file mode 100644 index 89c1f425..00000000 --- a/themes/custom-theme/layouts/robots.txt +++ /dev/null @@ -1,53 +0,0 @@ -# Dark Visitors robots.txt - -# AI Data Scraper -# https://darkvisitors.com/agents/anthropic-ai - -User-agent: anthropic-ai -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/ccbot - -User-agent: CCBot -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/facebookbot - -User-agent: FacebookBot -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/google-extended - -User-agent: Google-Extended -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/gptbot - -User-agent: GPTBot -Disallow: / - -# AI Data Scraper -# https://darkvisitors.com/agents/omgilibot - -User-agent: omgilibot -Disallow: / - -User-agent: omgili -Disallow: / - -# This blocks Bytespider -https://darkvisitors.com/agents/bytespider -User-agent: Bytespider -Disallow: / - -User-agent: * -{{ if eq (hugo.Environment) "production" -}} -Allow: / -{{ else -}} -Disallow: / -{{ end -}} -Sitemap: {{ "sitemap.xml" | absURL -}}