diff --git a/roles/pagure/frontend/files/robots.txt b/roles/pagure/frontend/files/robots.txt deleted file mode 100644 index a70291b52e..0000000000 --- a/roles/pagure/frontend/files/robots.txt +++ /dev/null @@ -1,5 +0,0 @@ -User-agent: * -Disallow: /api -Disallow: /login -Disallow: /*/raw -Crawl-Delay: 2 diff --git a/roles/pagure/frontend/tasks/main.yml b/roles/pagure/frontend/tasks/main.yml index ddb4248e5c..6d2f4128a1 100644 --- a/roles/pagure/frontend/tasks/main.yml +++ b/roles/pagure/frontend/tasks/main.yml @@ -31,7 +31,7 @@ - pagure - name: Put in robots.txt - copy: src=robots.txt dest=/var/www/html/robots.txt + template: src=robots.txt.j2 dest=/var/www/html/robots.txt tags: - pagure diff --git a/roles/pagure/frontend/templates/robots.txt.j2 b/roles/pagure/frontend/templates/robots.txt.j2 new file mode 100644 index 0000000000..f7f174d8e3 --- /dev/null +++ b/roles/pagure/frontend/templates/robots.txt.j2 @@ -0,0 +1,9 @@ +User-agent: * +{% if env == 'pagure-staging' %} +Disallow: / +{% else %} +Disallow: /api +Disallow: /login +Disallow: /*/raw +Crawl-Delay: 2 +{% endif %}