From 09fb09d20d4acb50d4b9587c1f992f608c806cc0 Mon Sep 17 00:00:00 2001
From: Kevin Fenzi <kevin@scrye.com>
Date: Thu, 29 Jan 2015 00:00:21 +0000
Subject: [PATCH] Drop crawl-delay down to 1

---
 roles/httpd/website/files/robots/robots.txt           | 2 +-
 roles/httpd/website/files/robots/robots.txt.lockbox01 | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/roles/httpd/website/files/robots/robots.txt b/roles/httpd/website/files/robots/robots.txt
index 93d096db6c..64e9e1c2ae 100644
--- a/roles/httpd/website/files/robots/robots.txt
+++ b/roles/httpd/website/files/robots/robots.txt
@@ -1,2 +1,2 @@
 User-agent: *
-Crawl-delay: 10
+Crawl-delay: 1
diff --git a/roles/httpd/website/files/robots/robots.txt.lockbox01 b/roles/httpd/website/files/robots/robots.txt.lockbox01
index ce52b44584..06a741dd49 100644
--- a/roles/httpd/website/files/robots/robots.txt.lockbox01
+++ b/roles/httpd/website/files/robots/robots.txt.lockbox01
@@ -1,4 +1,5 @@
 User-agent: *
-Crawl-delay: 10
+Crawl-delay: 1
 Allow: /infra/docs/
 Disallow: /infra
+Allow: /cgit/