From 6c2f16eb9a0cad3cc88f60f4b0bcbdc66061882d Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 3 Jul 2024 10:00:01 -0700 Subject: [PATCH] src: adjust robots.txt and fix formatting Signed-off-by: Kevin Fenzi --- .../website/templates/robots/src.fedoraproject.org-robots.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/httpd/website/templates/robots/src.fedoraproject.org-robots.txt b/roles/httpd/website/templates/robots/src.fedoraproject.org-robots.txt index fcbbd0321a..4121c50c7a 100644 --- a/roles/httpd/website/templates/robots/src.fedoraproject.org-robots.txt +++ b/roles/httpd/website/templates/robots/src.fedoraproject.org-robots.txt @@ -1,4 +1,8 @@ User-agent: * Crawl-delay: 5 + User-agent: Bytespider Disallow: / + +User-agent: ClaudeBot +Disallow: /