From 77644a51834f3043a4c296868faede26a5fe547c Mon Sep 17 00:00:00 2001 From: Rick Elrod Date: Mon, 28 Jan 2019 22:43:00 +0000 Subject: [PATCH] Attempt to make crawlers stop scanning stg.fp.o (infra #7514) Signed-off-by: Rick Elrod --- .../httpd/website/files/robots/robots.txt.fedoraproject.org | 2 -- roles/httpd/website/tasks/main.yml | 2 +- roles/httpd/website/{files => templates}/robots/robots.txt | 0 .../robots/robots.txt.admin.fedoraproject.org | 0 .../robots/robots.txt.apps.fedoraproject.org | 0 .../website/templates/robots/robots.txt.fedoraproject.org | 5 +++++ .../robots/robots.txt.greenwave.fedoraproject.org | 0 .../website/{files => templates}/robots/robots.txt.lockbox01 | 0 .../robots/robots.txt.paste.fedoraproject.org | 0 .../robots/robots.txt.translate.fedoraproject.org | 0 10 files changed, 6 insertions(+), 3 deletions(-) delete mode 100644 roles/httpd/website/files/robots/robots.txt.fedoraproject.org rename roles/httpd/website/{files => templates}/robots/robots.txt (100%) rename roles/httpd/website/{files => templates}/robots/robots.txt.admin.fedoraproject.org (100%) rename roles/httpd/website/{files => templates}/robots/robots.txt.apps.fedoraproject.org (100%) create mode 100644 roles/httpd/website/templates/robots/robots.txt.fedoraproject.org rename roles/httpd/website/{files => templates}/robots/robots.txt.greenwave.fedoraproject.org (100%) rename roles/httpd/website/{files => templates}/robots/robots.txt.lockbox01 (100%) rename roles/httpd/website/{files => templates}/robots/robots.txt.paste.fedoraproject.org (100%) rename roles/httpd/website/{files => templates}/robots/robots.txt.translate.fedoraproject.org (100%) diff --git a/roles/httpd/website/files/robots/robots.txt.fedoraproject.org b/roles/httpd/website/files/robots/robots.txt.fedoraproject.org deleted file mode 100644 index dbf8929435..0000000000 --- a/roles/httpd/website/files/robots/robots.txt.fedoraproject.org +++ /dev/null @@ -1,2 +0,0 @@ -User-agent: * -Disallow: /wikiold diff --git a/roles/httpd/website/tasks/main.yml b/roles/httpd/website/tasks/main.yml index d0f88c8111..399392ea94 100644 --- a/roles/httpd/website/tasks/main.yml +++ b/roles/httpd/website/tasks/main.yml @@ -57,7 +57,7 @@ - httpd/website - name: And lastly, the robots.txt file - copy: > + template: > src={{item}} dest=/srv/web/robots.txt.{{site_name}} mode=0644 diff --git a/roles/httpd/website/files/robots/robots.txt b/roles/httpd/website/templates/robots/robots.txt similarity index 100% rename from roles/httpd/website/files/robots/robots.txt rename to roles/httpd/website/templates/robots/robots.txt diff --git a/roles/httpd/website/files/robots/robots.txt.admin.fedoraproject.org b/roles/httpd/website/templates/robots/robots.txt.admin.fedoraproject.org similarity index 100% rename from roles/httpd/website/files/robots/robots.txt.admin.fedoraproject.org rename to roles/httpd/website/templates/robots/robots.txt.admin.fedoraproject.org diff --git a/roles/httpd/website/files/robots/robots.txt.apps.fedoraproject.org b/roles/httpd/website/templates/robots/robots.txt.apps.fedoraproject.org similarity index 100% rename from roles/httpd/website/files/robots/robots.txt.apps.fedoraproject.org rename to roles/httpd/website/templates/robots/robots.txt.apps.fedoraproject.org diff --git a/roles/httpd/website/templates/robots/robots.txt.fedoraproject.org b/roles/httpd/website/templates/robots/robots.txt.fedoraproject.org new file mode 100644 index 0000000000..01aa679090 --- /dev/null +++ b/roles/httpd/website/templates/robots/robots.txt.fedoraproject.org @@ -0,0 +1,5 @@ +User-agent: * +{% if env == "staging" %} +Disallow: / +{% endif %} +Disallow: /wikiold diff --git a/roles/httpd/website/files/robots/robots.txt.greenwave.fedoraproject.org b/roles/httpd/website/templates/robots/robots.txt.greenwave.fedoraproject.org similarity index 100% rename from roles/httpd/website/files/robots/robots.txt.greenwave.fedoraproject.org rename to roles/httpd/website/templates/robots/robots.txt.greenwave.fedoraproject.org diff --git a/roles/httpd/website/files/robots/robots.txt.lockbox01 b/roles/httpd/website/templates/robots/robots.txt.lockbox01 similarity index 100% rename from roles/httpd/website/files/robots/robots.txt.lockbox01 rename to roles/httpd/website/templates/robots/robots.txt.lockbox01 diff --git a/roles/httpd/website/files/robots/robots.txt.paste.fedoraproject.org b/roles/httpd/website/templates/robots/robots.txt.paste.fedoraproject.org similarity index 100% rename from roles/httpd/website/files/robots/robots.txt.paste.fedoraproject.org rename to roles/httpd/website/templates/robots/robots.txt.paste.fedoraproject.org diff --git a/roles/httpd/website/files/robots/robots.txt.translate.fedoraproject.org b/roles/httpd/website/templates/robots/robots.txt.translate.fedoraproject.org similarity index 100% rename from roles/httpd/website/files/robots/robots.txt.translate.fedoraproject.org rename to roles/httpd/website/templates/robots/robots.txt.translate.fedoraproject.org