Block all but internal from pkgs.fp.o and set up robots.txt files

Signed-off-by: Patrick Uiterwijk <puiterwijk@redhat.com>

roles/distgit/files/robots-pkgs.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

roles/distgit/files/robots-src.txt

@@ -0,0 +1,8 @@
+User-agent: *
+Disallow: /cgit/
+
+User-agent: *
+Disallow: /git/
+
+User-agent: *
+Disallow: /repo/

roles/distgit/tasks/main.yml

@@ -114,6 +114,14 @@
   tags:
   - distgit
 
+- name: Install robots.txt files
+  copy: src={{item}} dest=/var/www/{{item}}
+  with_items:
+  - robots-pkgs.txt
+  - robots-src.txt
+  tags:
+  - distgit
+
 - name: install the DistGit related httpd config
   copy: src=git-smart-http.conf dest=/etc/httpd/conf.d/dist-git/git-smart-http.conf
   notify:

roles/distgit/templates/lookaside-upload.conf

@@ -12,13 +12,32 @@ SSLRandomSeed startup file:/dev/urandom  256
 SSLRandomSeed connect builtin
 SSLCryptoDevice builtin
 
+Alias /robots.txt /var/www/robots-src.txt
+<Location /robots.txt>
+    Require all granted
+</Location>
+
 <VirtualHost _default_:80>
     ServerName pkgs{{ env_suffix }}.fedoraproject.org
-    RewriteCond expr "! -R '192.168.0.0/16'"        
-    RewriteCond expr "! -R '10.0.0.0/8'"            
-    RewriteRule ^(.*)$ https://src.fedoraproject.org/$1 [L,R]
+    #RewriteCond expr "! -R '192.168.0.0/16'"        
+    #RewriteCond expr "! -R '10.0.0.0/8'"            
+    #RewriteRule ^(.*)$ https://src.fedoraproject.org/$1 [L,R]
     Alias /repo/ /srv/cache/lookaside/
 
+    <Location />
+        Require ip 127.0.0.1
+        Require ip ::1
+        Require ip 10.0.0.0/8
+        Require ip 192.168.0.0/16
+    </Location>
+
+    CustomLog "logs/pkgs-access.log" combined
+    ErrorLog "logs/pkgs-error.log"
+    Alias /robots.txt /var/www/robots-pkgs.txt
+    <Location /robots.txt>
+        Require all granted
+    </Location>
+
     RewriteEngine on
     RewriteRule "^/$" "https://src{{ env_suffix }}.fedoraproject.org/"
     RewriteRule "^/login/$" "https://src{{ env_suffix }}.fedoraproject.org/login/"