diff --git a/roles/httpd/website/templates/robots/lists.fedoraproject.org-robots.txt b/roles/httpd/website/templates/robots/lists.fedoraproject.org-robots.txt index 8e817aaa29..a51422965a 100644 --- a/roles/httpd/website/templates/robots/lists.fedoraproject.org-robots.txt +++ b/roles/httpd/website/templates/robots/lists.fedoraproject.org-robots.txt @@ -2,5 +2,10 @@ User-agent: * Disallow: /accounts/ Disallow: /django-admin/ Disallow: /archives/*/vote$ +Crawl-Delay: 10 + User-agent: Bytespider Disallow: / + +User-agent: ClaudeBot +Disallow: /