Drop crawl-delay down to 1

2015-01-29 00:00:21 +00:00 · 2015-01-29 00:00:21 +00:00 · 09fb09d20d
commit 09fb09d20d
parent 4fe887e988
2 changed files with 3 additions and 2 deletions
--- a/roles/httpd/website/files/robots/robots.txt
+++ b/roles/httpd/website/files/robots/robots.txt
@ -1,2 +1,2 @@
 User-agent: *
-Crawl-delay: 10
+Crawl-delay: 1
--- a/roles/httpd/website/files/robots/robots.txt.lockbox01
+++ b/roles/httpd/website/files/robots/robots.txt.lockbox01
@ -1,4 +1,5 @@
 User-agent: *
-Crawl-delay: 10
+Crawl-delay: 1
 Allow: /infra/docs/
 Disallow: /infra
+Allow: /cgit/