diff --git a/roles/httpd/website/templates/robots/lists.fedoraproject.org-robots.txt b/roles/httpd/website/templates/robots/lists.fedoraproject.org-robots.txt index a51422965a..87fec7f04a 100644 --- a/roles/httpd/website/templates/robots/lists.fedoraproject.org-robots.txt +++ b/roles/httpd/website/templates/robots/lists.fedoraproject.org-robots.txt @@ -9,3 +9,6 @@ Disallow: / User-agent: ClaudeBot Disallow: / + +User-agent: Barkrowler +Disallow: / diff --git a/roles/mailman3/templates/mailmanweb.conf.j2 b/roles/mailman3/templates/mailmanweb.conf.j2 index a6a73678ce..749dc9a444 100644 --- a/roles/mailman3/templates/mailmanweb.conf.j2 +++ b/roles/mailman3/templates/mailmanweb.conf.j2 @@ -33,9 +33,11 @@ ProxyPassReverse / http://127.0.0.1:8000/ # Redirecting to hyperkitty if nothing is specified RewriteEngine on RewriteRule ^/$ /archives [R,L] + # Spiders-gone-wild -# These spiders do not follow robots.txt -RewriteCond %{HTTP_USER_AGENT} ^.*(Bytespider|ClaudeBot).*$ [NC] +# These spiders may not follow robots.txt and will +# hit admin sections which consume large amounts of CPU +RewriteCond %{HTTP_USER_AGENT} ^.*(Bytespider|ClaudeBot|Amazonbot|YandexBot|claudebot|ChatGLM-Spider|GPTBot|Barkrowler|YisouSpider|MJ12bot).*$ [NC] RewriteRule .* - [F,L] # Old static archives