diff --git a/roles/distgit/files/robots-pkgs.txt b/roles/distgit/files/robots-pkgs.txt
new file mode 100644
index 0000000000..1f53798bb4
--- /dev/null
+++ b/roles/distgit/files/robots-pkgs.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /
diff --git a/roles/distgit/files/robots-src.txt b/roles/distgit/files/robots-src.txt
new file mode 100644
index 0000000000..437658ade1
--- /dev/null
+++ b/roles/distgit/files/robots-src.txt
@@ -0,0 +1,8 @@
+User-agent: *
+Disallow: /cgit/
+
+User-agent: *
+Disallow: /git/
+
+User-agent: *
+Disallow: /repo/
diff --git a/roles/distgit/tasks/main.yml b/roles/distgit/tasks/main.yml
index ac3972171b..9d8c949bf0 100644
--- a/roles/distgit/tasks/main.yml
+++ b/roles/distgit/tasks/main.yml
@@ -114,6 +114,14 @@
tags:
- distgit
+- name: Install robots.txt files
+ copy: src={{item}} dest=/var/www/{{item}}
+ with_items:
+ - robots-pkgs.txt
+ - robots-src.txt
+ tags:
+ - distgit
+
- name: install the DistGit related httpd config
copy: src=git-smart-http.conf dest=/etc/httpd/conf.d/dist-git/git-smart-http.conf
notify:
diff --git a/roles/distgit/templates/lookaside-upload.conf b/roles/distgit/templates/lookaside-upload.conf
index ec9afabc35..dc2b8826c1 100644
--- a/roles/distgit/templates/lookaside-upload.conf
+++ b/roles/distgit/templates/lookaside-upload.conf
@@ -12,13 +12,32 @@ SSLRandomSeed startup file:/dev/urandom 256
SSLRandomSeed connect builtin
SSLCryptoDevice builtin
+Alias /robots.txt /var/www/robots-src.txt
+
+ Require all granted
+
+
ServerName pkgs{{ env_suffix }}.fedoraproject.org
- RewriteCond expr "! -R '192.168.0.0/16'"
- RewriteCond expr "! -R '10.0.0.0/8'"
- RewriteRule ^(.*)$ https://src.fedoraproject.org/$1 [L,R]
+ #RewriteCond expr "! -R '192.168.0.0/16'"
+ #RewriteCond expr "! -R '10.0.0.0/8'"
+ #RewriteRule ^(.*)$ https://src.fedoraproject.org/$1 [L,R]
Alias /repo/ /srv/cache/lookaside/
+
+ Require ip 127.0.0.1
+ Require ip ::1
+ Require ip 10.0.0.0/8
+ Require ip 192.168.0.0/16
+
+
+ CustomLog "logs/pkgs-access.log" combined
+ ErrorLog "logs/pkgs-error.log"
+ Alias /robots.txt /var/www/robots-pkgs.txt
+
+ Require all granted
+
+
RewriteEngine on
RewriteRule "^/$" "https://src{{ env_suffix }}.fedoraproject.org/"
RewriteRule "^/login/$" "https://src{{ env_suffix }}.fedoraproject.org/login/"