diff --git a/roles/httpd/reverseproxy/templates/reversepassproxy.fp-wiki.conf b/roles/httpd/reverseproxy/templates/reversepassproxy.fp-wiki.conf index a59d73c160..4209ac7f22 100644 --- a/roles/httpd/reverseproxy/templates/reversepassproxy.fp-wiki.conf +++ b/roles/httpd/reverseproxy/templates/reversepassproxy.fp-wiki.conf @@ -8,6 +8,11 @@ RewriteCond %{HTTPS} off RewriteCond %{QUERY_STRING} action= [NC] RewriteRule .* https://%{HTTP_HOST}%{REQUEST_URI} [R=301,NE,L] +# Spiders-gone-wild +# These spiders may not follow robots.txt and will +# hit admin sections which consume large amounts of CPU +RewriteCond %{HTTP_USER_AGENT} ^.*(Bytespider|ClaudeBot|Amazonbot|YandexBot|ChatGLM-Spider|GPTBot|Barkrowler|YisouSpider|MJ12bot|SemrushBot).*$ [NC] +RewriteRule .* - [F,L] # /wiki must come before /w due to prefix matching. ProxyPass {{wikipath}} {{proxyurl}}{{wikipath}}