From 00269dc0a8d33d6aa4157da918424bdb940ccbb9 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sun, 7 Dec 2014 23:39:44 +0000 Subject: [PATCH] Working on the proxies-websites.yml playbook. --- inventory/group_vars/all | 4 ++ inventory/group_vars/staging | 3 + playbooks/groups/proxies-redirects.yml | 1 - playbooks/groups/proxies-websites.yml | 31 ++++++++ playbooks/groups/proxies.yml | 2 - roles/httpd/website/files/robots/robots.txt | 2 + .../robots/robots.txt.admin.fedoraproject.org | 8 +++ .../files/robots/robots.txt.fedoraproject.org | 2 + .../website/files/robots/robots.txt.lockbox01 | 4 ++ .../robots.txt.translate.fedoraproject.org | 4 ++ roles/httpd/website/tasks/main.yml | 70 +++++++++++++++++++ roles/httpd/website/templates/logs.conf | 2 + roles/httpd/website/templates/robots.conf | 1 + roles/httpd/website/vars/main.yml | 8 +++ 14 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 playbooks/groups/proxies-websites.yml create mode 100644 roles/httpd/website/files/robots/robots.txt create mode 100644 roles/httpd/website/files/robots/robots.txt.admin.fedoraproject.org create mode 100644 roles/httpd/website/files/robots/robots.txt.fedoraproject.org create mode 100644 roles/httpd/website/files/robots/robots.txt.lockbox01 create mode 100644 roles/httpd/website/files/robots/robots.txt.translate.fedoraproject.org create mode 100644 roles/httpd/website/tasks/main.yml create mode 100644 roles/httpd/website/templates/logs.conf create mode 100644 roles/httpd/website/templates/robots.conf create mode 100644 roles/httpd/website/vars/main.yml diff --git a/inventory/group_vars/all b/inventory/group_vars/all index 1f8a5beb70..e413d0b9e2 100644 --- a/inventory/group_vars/all +++ b/inventory/group_vars/all @@ -44,6 +44,10 @@ virt_install_command: virt-install -n {{ inventory_hostname }} -r {{ mem_size }} --network bridge=br0,model=virtio --autostart --noautoconsole +# This is the wildcard certname for our proxies. It has a different name for +# the staging group and is used in the proxies.yml playbook. +wildcard_ssl_cert: wildcard-2014.fedoraproject.org + # By default, nodes get no fedmsg certs. They need to declare them explicitly. fedmsg_certs: [] diff --git a/inventory/group_vars/staging b/inventory/group_vars/staging index c1cf3ff8f2..60a6d58673 100644 --- a/inventory/group_vars/staging +++ b/inventory/group_vars/staging @@ -2,3 +2,6 @@ freezes: false env: staging host_group: staging + +# This is the wildcard certname for our stg proxies. +wildcard_ssl_cert: wildcard-2014.stg.fedoraproject.org diff --git a/playbooks/groups/proxies-redirects.yml b/playbooks/groups/proxies-redirects.yml index c5d3f12338..3164e83c01 100644 --- a/playbooks/groups/proxies-redirects.yml +++ b/playbooks/groups/proxies-redirects.yml @@ -2,7 +2,6 @@ hosts: proxies-stg user: root gather_facts: True - accelerate: "{{ accelerated }}" vars_files: - /srv/web/infra/ansible/vars/global.yml diff --git a/playbooks/groups/proxies-websites.yml b/playbooks/groups/proxies-websites.yml new file mode 100644 index 0000000000..334bd946b3 --- /dev/null +++ b/playbooks/groups/proxies-websites.yml @@ -0,0 +1,31 @@ +- name: Set up those proxy websites. My, my.. + hosts: proxies-stg + user: root + gather_facts: True + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + handlers: + - include: "{{ handlers }}/restart_services.yml" + + vars: + - ssl: true + - SSLCertificateChainFile: wildcard-2014.fedoraproject.org.intermediate.cert + + # wildcard_cert_name is defined in group_vars + + roles: + + - role: httpd/website + name: fedoraproject.org + server_aliases: [stg.fedoraproject.org] + cert_name: {{wildcard_cert_name}} + + - role: httpd/website + name: admin.fedoraproject.org + server_aliases: [admin.stg.fedoraproject.org] + cert_name: {{wildcard_cert_name}} + sslonly: true diff --git a/playbooks/groups/proxies.yml b/playbooks/groups/proxies.yml index 5b67f23baa..ce9bbc0c77 100644 --- a/playbooks/groups/proxies.yml +++ b/playbooks/groups/proxies.yml @@ -4,7 +4,6 @@ hosts: proxies-stg user: root gather_facts: False - accelerate: "{{ accelerated }}" vars_files: - /srv/web/infra/ansible/vars/global.yml @@ -21,7 +20,6 @@ hosts: proxies-stg user: root gather_facts: True - accelerate: "{{ accelerated }}" vars_files: - /srv/web/infra/ansible/vars/global.yml diff --git a/roles/httpd/website/files/robots/robots.txt b/roles/httpd/website/files/robots/robots.txt new file mode 100644 index 0000000000..93d096db6c --- /dev/null +++ b/roles/httpd/website/files/robots/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Crawl-delay: 10 diff --git a/roles/httpd/website/files/robots/robots.txt.admin.fedoraproject.org b/roles/httpd/website/files/robots/robots.txt.admin.fedoraproject.org new file mode 100644 index 0000000000..aeb1c2cc4c --- /dev/null +++ b/roles/httpd/website/files/robots/robots.txt.admin.fedoraproject.org @@ -0,0 +1,8 @@ +User-agent: * +Disallow: /voting +Disallow: /mirrormanager +Disallow: /pkgdb/packages/name + +# Temp block http://www.80legs.com/webcrawler.html +User-agent: 008 +Disallow: / diff --git a/roles/httpd/website/files/robots/robots.txt.fedoraproject.org b/roles/httpd/website/files/robots/robots.txt.fedoraproject.org new file mode 100644 index 0000000000..dbf8929435 --- /dev/null +++ b/roles/httpd/website/files/robots/robots.txt.fedoraproject.org @@ -0,0 +1,2 @@ +User-agent: * +Disallow: /wikiold diff --git a/roles/httpd/website/files/robots/robots.txt.lockbox01 b/roles/httpd/website/files/robots/robots.txt.lockbox01 new file mode 100644 index 0000000000..ce52b44584 --- /dev/null +++ b/roles/httpd/website/files/robots/robots.txt.lockbox01 @@ -0,0 +1,4 @@ +User-agent: * +Crawl-delay: 10 +Allow: /infra/docs/ +Disallow: /infra diff --git a/roles/httpd/website/files/robots/robots.txt.translate.fedoraproject.org b/roles/httpd/website/files/robots/robots.txt.translate.fedoraproject.org new file mode 100644 index 0000000000..24a40d99f5 --- /dev/null +++ b/roles/httpd/website/files/robots/robots.txt.translate.fedoraproject.org @@ -0,0 +1,4 @@ +User-agent: * +Crawl-delay: 10 +Disallow: /*download_zip/$ +Disallow: /*download_targz/$ diff --git a/roles/httpd/website/tasks/main.yml b/roles/httpd/website/tasks/main.yml new file mode 100644 index 0000000000..25c186dd56 --- /dev/null +++ b/roles/httpd/website/tasks/main.yml @@ -0,0 +1,70 @@ +# Expected vars +# - name... +# - ips: [] +# - server_aliases: [] +# - server_admin: webmaster@fedoraproject.org +# - ssl: true +# - sslonly: false +# - cert_name: "" +# - SSLCertificateChainFile: "" +# - gzip: false + +- name: Create site directory in httpd/conf.d/{{name}} + file: > + dest=/etc/httpd/conf.d/{{name}} + state=directory + owner=root + group=root + mode=0755 + notify: + - restart httpd + tags: + - httpd + - httpd/website + +# TODO - copy in Httpd::Certificate + +- name: Copy over primary template for {{name}} + template: > + src=website.conf + dest=/etc/httpd/conf.d/{{name}}.conf + owner=root + group=root + mode=0644 + notify: + - restart httpd + tags: + - httpd + - httpd/redirect + +- name: Copy over some subordinate templates for {{name}} + template: > + src={{item}}.conf + dest=/etc/httpd/conf.d/{{name}}/{{item}}.conf + owner=root + group=root + mode=0644 + with_items: + - logs + - robots + notify: + - restart httpd + tags: + - httpd + - httpd/redirect + +- name: And lastly, the robots.txt file + copy: > + src={{item}} + dest=/srv/web/robots.txt.{{name}} + mode=0644 + owner=root + group=root + with_first_found: + - robots/robots.txt.{{name}} + - robots/robots.txt + notify: + - restart httpd + tags: + - httpd + - httpd/redirect diff --git a/roles/httpd/website/templates/logs.conf b/roles/httpd/website/templates/logs.conf new file mode 100644 index 0000000000..f4b06d9711 --- /dev/null +++ b/roles/httpd/website/templates/logs.conf @@ -0,0 +1,2 @@ +CustomLog "logs/{{ name }}-access.log" combined +ErrorLog "logs/{{ name }}-error.log" diff --git a/roles/httpd/website/templates/robots.conf b/roles/httpd/website/templates/robots.conf new file mode 100644 index 0000000000..040f48d397 --- /dev/null +++ b/roles/httpd/website/templates/robots.conf @@ -0,0 +1 @@ +Alias /robots.txt /srv/web/robots.txt.{{ name }} diff --git a/roles/httpd/website/vars/main.yml b/roles/httpd/website/vars/main.yml new file mode 100644 index 0000000000..ac7609056a --- /dev/null +++ b/roles/httpd/website/vars/main.yml @@ -0,0 +1,8 @@ +ips: [] +server_aliases: [] +server_admin: webmaster@fedoraproject.org +ssl: true +sslonly: false +cert_name: "" +SSLCertificateChainFile: "" +gzip: false