diff --git a/themes/hugo-theme-gruvbox/layouts/robots.txt b/themes/hugo-theme-gruvbox/layouts/robots.txt index 62e723e3..5de09978 100644 --- a/themes/hugo-theme-gruvbox/layouts/robots.txt +++ b/themes/hugo-theme-gruvbox/layouts/robots.txt @@ -1,7 +1,48 @@ +# Dark Visitors robots.txt + +# AI Data Scraper +# https://darkvisitors.com/agents/anthropic-ai + +User-agent: anthropic-ai +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/ccbot + +User-agent: CCBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/facebookbot + +User-agent: FacebookBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/google-extended + +User-agent: Google-Extended +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/gptbot + +User-agent: GPTBot +Disallow: / + +# AI Data Scraper +# https://darkvisitors.com/agents/omgilibot + +User-agent: omgilibot +Disallow: / + +User-agent: omgili +Disallow: / + User-agent: * {{ if eq (hugo.Environment) "production" -}} Allow: / {{ else -}} Disallow: / {{ end -}} -Sitemap: {{ "sitemap.xml" | absURL -}} \ No newline at end of file +Sitemap: {{ "sitemap.xml" | absURL -}}