From 4937c5be2a00b2858cc5965d8d9132c881824042 Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Wed, 3 Jul 2024 17:18:06 +0000 Subject: [PATCH] Attempt to stop various spiders which do not follow rules. The ByteSpider and ClaudeBot do not follow robots.txt so need to be dealt with in other ways. Signed-off-by: Stephen Smoogen --- roles/mailman3/templates/mailmanweb.conf.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/mailman3/templates/mailmanweb.conf.j2 b/roles/mailman3/templates/mailmanweb.conf.j2 index 763e4395ae..82430119b4 100644 --- a/roles/mailman3/templates/mailmanweb.conf.j2 +++ b/roles/mailman3/templates/mailmanweb.conf.j2 @@ -33,6 +33,10 @@ ProxyPassReverse / http://127.0.0.1:8000/ # Redirecting to hyperkitty if nothing is specified RewriteEngine on RewriteRule ^/$ /archives [R,L] +# Spiders-gone-wild +# These spiders do not follow robots.txt +RewriteCond %{HTTP_USER_AGENT ^.*(Bytespider|ClaudeBot).*$ [NC] +RewriteRule .* - [F,L] # Old static archives