Add robots.txt for Pagure

Signed-off-by: Patrick Uiterwijk <puiterwijk@redhat.com>

roles/pagure/frontend/files/robots.txt

@@ -0,0 +1,5 @@
+User-agent: *
+Disallow: /api
+Disallow: /login
+Disallow: /*/raw
+Crawl-Delay: 2 

roles/pagure/frontend/tasks/main.yml

@@ -30,6 +30,11 @@
   tags:
   - pagure
 
+- name: Put in robots.txt
+  copy: src=robots.txt dest=/var/www/html/robots.txt
+  tags:
+  - pagure
+
 
 # Set-up gitolite
 

roles/pagure/frontend/templates/0_pagure.conf

@@ -59,6 +59,8 @@ WSGIDaemonProcess paguredocs user=git group=git maximum-requests=1000 display-na
   ServerName pagure.io
 {% endif %}
 
+  Alias "/robots.txt" "/var/www/html/robots.txt"
+
   WSGIScriptAlias / /var/www/pagure.wsgi
 
   ServerAdmin admin@fedoraproject.org