diff --git a/roles/download/files/httpd/dl.fedoraproject.org/rewrite.conf b/roles/download/files/httpd/dl.fedoraproject.org/rewrite.conf index 581012ed97..6ca112f2f6 100644 --- a/roles/download/files/httpd/dl.fedoraproject.org/rewrite.conf +++ b/roles/download/files/httpd/dl.fedoraproject.org/rewrite.conf @@ -3,8 +3,13 @@ RewriteEngine On RewriteCond %{HTTP_USER_AGENT} "lftp" RewriteRule ^.*$ https://fedoraproject.org/wiki/Infrastructure/Mirroring#Tools_to_avoid [R,L] -RewriteRule ^/$ /pub [R=302,L] +# Spiders-gone-wild +# These spiders may not follow robots.txt and will +# hit admin sections which consume large amounts of CPU +RewriteCond %{HTTP_USER_AGENT} ^.*(Bytespider|ClaudeBot|Amazonbot|YandexBot|ChatGLM-Spider|GPTBot|Barkrowler|YisouSpider|MJ12bot).*$ [NC] +RewriteRule .* - [F,L] +RewriteRule ^/$ /pub [R=302,L] RedirectMatch 302 ^/pub/fedora/linux/atomic/(.*$) https://kojipkgs.fedoraproject.org/atomic/$1 RedirectMatch 302 ^/pub/fedora/linux/atomic https://kojipkgs.fedoraproject.org/atomic/ diff --git a/roles/mailman3/templates/mailmanweb.conf.j2 b/roles/mailman3/templates/mailmanweb.conf.j2 index 749dc9a444..87d3e99eff 100644 --- a/roles/mailman3/templates/mailmanweb.conf.j2 +++ b/roles/mailman3/templates/mailmanweb.conf.j2 @@ -37,7 +37,7 @@ RewriteRule ^/$ /archives [R,L] # Spiders-gone-wild # These spiders may not follow robots.txt and will # hit admin sections which consume large amounts of CPU -RewriteCond %{HTTP_USER_AGENT} ^.*(Bytespider|ClaudeBot|Amazonbot|YandexBot|claudebot|ChatGLM-Spider|GPTBot|Barkrowler|YisouSpider|MJ12bot).*$ [NC] +RewriteCond %{HTTP_USER_AGENT} ^.*(Bytespider|ClaudeBot|Amazonbot|YandexBot|ChatGLM-Spider|GPTBot|Barkrowler|YisouSpider|MJ12bot).*$ [NC] RewriteRule .* - [F,L] # Old static archives diff --git a/roles/pagure/templates/0_pagure.conf b/roles/pagure/templates/0_pagure.conf index 5fdd9cba72..8ed899f6de 100644 --- a/roles/pagure/templates/0_pagure.conf +++ b/roles/pagure/templates/0_pagure.conf @@ -138,10 +138,13 @@ MaxConnectionsPerChild 1000 # RewriteEngine On # RewriteCond %{REQUEST_URI} ^/fedora-web/websites$ # RewriteRule .* - [F] - # Reject Bytespider spider + RewriteEngine On - RewriteCond %{HTTP_USER_AGENT} .*Bytespider.* - RewriteRule .* - [F] +# Spiders-gone-wild +# These spiders may not follow robots.txt and will +# hit admin sections which consume large amounts of CPU + RewriteCond %{HTTP_USER_AGENT} ^.*(Bytespider|ClaudeBot|Amazonbot|YandexBot|ChatGLM-Spider|GPTBot|Barkrowler|YisouSpider|MJ12bot).*$ [NC] + RewriteRule .* - [F,L] SetHandler server-status