Merge branch 'master' of /git/ansible

inventory/builders

@@ -241,6 +241,9 @@ arm04-builder00.arm.fedoraproject.org
 arm04-builder01.arm.fedoraproject.org
 arm02-builder21.arm.fedoraproject.org
 arm02-builder23.arm.fedoraproject.org
+aarch64-02a.arm.fedoraproject.org
+buildvm-ppc64-01.ppc.fedoraproject.org
+buildvm-ppc64le-01.ppc.fedoraproject.org
 
 [builders:children]
 buildhw

inventory/group_vars/koschei-stg

@@ -12,7 +12,7 @@ koschei_pgsql_hostname: db01.stg.phx2.fedoraproject.org
 koschei_koji_hub: koji01.stg.phx2.fedoraproject.org
 koschei_kojipkgs: koji01.stg.phx2.fedoraproject.org
 koschei_koji_web: koji.stg.fedoraproject.org
-koschei_koji_tag: f24
+koschei_koji_tag: f25
 koschei_openid_provider: id.stg.fedoraproject.org
 koschei_bugzilla: partner-bugzilla.redhat.com
 

inventory/group_vars/openqa

@@ -28,6 +28,30 @@ checkcompose_smtp: bastion.phx2.fedoraproject.org
 checkcompose_url: "https://{{ external_hostname }}"
 
 deployment_type: prod
+freezes: false
 
 # http and NFS
 tcp_ports: [ 80, 2049 ]
+
+# These are consumed by a task in roles/fedmsg/base/main.yml
+fedmsg_certs:
+- service: shell
+  owner: root
+  group: sysadmin
+  can_send:
+  - logger.log
+- service: openqa
+  owner: root
+  group: geekotest
+  can_send:
+  - openqa.job.create
+  - openqa.job.delete
+  - openqa.job.cancel
+  - openqa.job.duplicate
+  - openqa.job.restart
+  - openqa.jobs.restart
+  - openqa.job.update.result
+  - openqa.job.done
+
+# we need this to log with fedmsg-logger
+fedmsg_active: True

inventory/group_vars/openqa-stg

@@ -35,6 +35,33 @@ wikitcms_password: "{{ stg_wikitcms_password }}"
 checkcompose_url: "https://{{ external_hostname }}"
 
 deployment_type: stg
+freezes: false
 
 # http and NFS
 tcp_ports: [80, 2049]
+
+# These are consumed by a task in roles/fedmsg/base/main.yml
+fedmsg_certs:
+- service: shell
+  owner: root
+  group: sysadmin
+  can_send:
+  - logger.log
+- service: openqa
+  owner: root
+  group: geekotest
+  can_send:
+  - openqa.job.create
+  - openqa.job.delete
+  - openqa.job.cancel
+  - openqa.job.duplicate
+  - openqa.job.restart
+  - openqa.jobs.restart
+  - openqa.job.update.result
+  - openqa.job.done
+
+# we need this to log with fedmsg-logger
+fedmsg_active: True
+
+# makes sure it sends stg not prod fedmsgs
+fedmsg_env: stg

inventory/group_vars/openqa-stg-workers

@@ -6,3 +6,4 @@ openqa_key: "{{ stg_openqa_apikey }}"
 openqa_secret: "{{ stg_openqa_apisecret }}"
 
 deployment_type: stg
+freezes: false

inventory/group_vars/openqa-workers

@@ -6,3 +6,4 @@ openqa_key: "{{ prod_openqa_apikey }}"
 openqa_secret: "{{ prod_openqa_apisecret }}"
 
 deployment_type: prod
+freezes: false

inventory/group_vars/pdc-backend

@@ -13,4 +13,6 @@ fas_client_groups: sysadmin-noc,sysadmin-releng,sysadmin-datanommer
 
 # These people get told when something goes wrong.
 fedmsg_error_recipients:
-- releng-cron@lists.fedoraproject.org
+- ralph@fedoraproject.org
+- ausil@fedoraproject.org
+- adamwill@fedoraproject.org

inventory/group_vars/pdc-backend-stg

@@ -13,4 +13,6 @@ fas_client_groups: sysadmin-noc,sysadmin-releng,sysadmin-datanommer
 
 # These people get told when something goes wrong.
 fedmsg_error_recipients:
-- releng-cron@lists.fedoraproject.org
+- ralph@fedoraproject.org
+- ausil@fedoraproject.org
+- adamwill@fedoraproject.org

inventory/group_vars/proxies

@@ -63,6 +63,8 @@ custom_rules: [
 
     # Allow resultsdb talk to the inbound fedmsg relay.
     '-A INPUT -p tcp -m tcp --dport 9941 -s 10.5.124.207 -j ACCEPT',
+    # Allow openqa01 to talk to the inbound fedmsg relay.
+    '-A INPUT -p tcp -m tcp --dport 9941 -s 10.5.131.71 -j ACCEPT',
  ]
 
 fas_client_groups: sysadmin-noc,fi-apprentice

inventory/group_vars/proxies-stg

@@ -67,6 +67,9 @@ custom_rules: [
 
     # Allow resultsdb talk to the inbound fedmsg relay.
     '-A INPUT -p tcp -m tcp --dport 9941 -s 10.5.124.147 -j ACCEPT',
+
+    # Allow openqa to talk to the inbound fedmsg relay.
+    '-A INPUT -p tcp -m tcp --dport 9941 -s 10.5.131.72 -j ACCEPT',
  ]
 
 fas_client_groups: sysadmin-noc,fi-apprentice

inventory/group_vars/qa-stg

@@ -15,10 +15,11 @@ extra_enablerepos: 'infrastructure-testing'
 
 sshd_config: ssh/sshd_config.qa-stg
 sshd_port: 222
-external_hostname: qadevel-stg.cloud.fedoraproject.org
+external_hostname: qa.stg.fedoraproject.org
 
-sslcertfile: qa-stg.qa.fedoraproject.org.cert
+# not needed with new setup
-sslkeyfile: qa-stg.qa.fedoraproject.org.key
+#sslcertfile: qa-stg.qa.fedoraproject.org.cert
+#sslkeyfile: qa-stg.qa.fedoraproject.org.key
 sslintermediatecertfile: ''
 
 mariadb_host: localhost

inventory/host_vars/openqa-stg01.qa.fedoraproject.org

@@ -30,5 +30,3 @@ num_cpus: 4
 
 nrpe_procs_warn: 250
 nrpe_procs_crit: 300
-
-freezes: false

inventory/host_vars/openqa01.qa.fedoraproject.org

@@ -30,5 +30,3 @@ num_cpus: 4
 
 nrpe_procs_warn: 250
 nrpe_procs_crit: 300
-
-freezes: false

inventory/host_vars/qa05.qa.fedoraproject.org

@@ -1,5 +1,4 @@
 ---
-freezes: false
 fas_client_groups: sysadmin-qa,sysadmin-main
 sudoers: "{{ private }}/files/sudo/qavirt-sudoers"
 eth0_ip: 10.5.124.155

inventory/host_vars/qa06.qa.fedoraproject.org

@@ -1,5 +1,4 @@
 ---
-freezes: false
 fas_client_groups: sysadmin-qa,sysadmin-main
 sudoers: "{{ private }}/files/sudo/qavirt-sudoers"
 eth0_ip: 10.5.124.156

inventory/host_vars/qa07.qa.fedoraproject.org

@@ -1,5 +1,4 @@
 ---
-freezes: false
 fas_client_groups: sysadmin-qa,sysadmin-main
 sudoers: "{{ private }}/files/sudo/qavirt-sudoers"
 eth0_ip: 10.5.124.157

inventory/inventory

@@ -803,9 +803,11 @@ retrace02.qa.fedoraproject.org
 s390-koji01.qa.fedoraproject.org
 arm-koji01.qa.fedoraproject.org
 resultsdb01.qa.fedoraproject.org
+openqa01.qa.fedoraproject.org
 
 [fedmsg-qa-network-stg]
 resultsdb-stg01.qa.fedoraproject.org
+openqa-stg01.qa.fedoraproject.org
 
 
 # assorted categories of fedmsg services, for convenience

playbooks/groups/openqa.yml

@@ -43,14 +43,14 @@
   roles:
    - { role: openqa/server, tags: ['openqa_server'] }
    - { role: openqa/dispatcher, tags: ['openqa_dispatcher'] }
-   - { role: fedmsg/hub, tags: ['fedmsg_hub'] }
    - { role: check-compose, tags: ['check-compose'] }
+   - { role: fedmsg/base, tags: ['fedmsg_base', 'fedmsg'] }
+   - { role: fedmsg/hub, tags: ['fedmsg_hub', 'fedmsg'] }
 
   handlers:
    - include: "{{ handlers }}/restart_services.yml"
 
-# for now just stg while we're testing
+- name: set up openQA server data NFS mounts (staging)
-- name: set up openQA server data NFS mounts
   hosts: openqa-stg
   roles:
   - role: nfs/client
@@ -58,22 +58,22 @@
     nfs_src_dir: 'fedora_openqa_stg/testresults'
     nfs_mount_opts: 'rw,bg,nfsvers=3'
     tags: ['nfs_client']
-
-- name: set up openQA server data NFS mounts
-  hosts: openqa-stg
-  roles:
   - role: nfs/client
     mnt_dir: '/var/lib/openqa/images'
     nfs_src_dir: 'fedora_openqa_stg/images'
     nfs_mount_opts: 'rw,bg,nfsvers=3'
     tags: ['nfs_client']
 
-# set up prod temp mount
+- name: set up openQA server data NFS mounts (prod)
-- name: set up openQA server data NFS mounts
   hosts: openqa
   roles:
   - role: nfs/client
-    mnt_dir: '/mnt/temp'
+    mnt_dir: '/var/lib/openqa/testresults'
-    nfs_src_dir: 'fedora_openqa'
+    nfs_src_dir: 'fedora_openqa/testresults'
+    nfs_mount_opts: 'rw,bg,nfsvers=3'
+    tags: ['nfs_client']
+  - role: nfs/client
+    mnt_dir: '/var/lib/openqa/images'
+    nfs_src_dir: 'fedora_openqa/images'
     nfs_mount_opts: 'rw,bg,nfsvers=3'
     tags: ['nfs_client']

playbooks/include/proxies-fedora-web.yml

@@ -41,7 +41,6 @@
     website: arm.fedoraproject.org
   - role: fedora-web/budget
     website: budget.fedoraproject.org
-    when: env == "staging"
 
   # Some other static content, not strictly part of "fedora-web" goes below here
   - role: fedora-docs/proxy

playbooks/include/proxies-reverseproxy.yml

@@ -455,6 +455,7 @@
     destname: qa-stg-phab
     # Talk directly to the app server, not haproxy
     proxyurl: http://phab.qa-stg01.qa.fedoraproject.org
+    keephost: true
     when: env == "staging"
 
   - role: httpd/reverseproxy

playbooks/include/proxies-websites.yml

@@ -203,7 +203,6 @@
     - budget.stg.fedoraproject.org
     sslonly: true
     cert_name: "{{wildcard_cert_name}}"
-    when: env == "staging"
 
   - role: httpd/website
     name: boot.fedoraproject.org

roles/base/templates/iptables/iptables.osuosl

@@ -29,7 +29,6 @@
 -A INPUT -p tcp -m tcp --dport 5666  -s 209.132.181.35 -j ACCEPT
 -A INPUT -p tcp -m tcp --dport 5666 -s 10.5.126.41 -j ACCEPT
 
-
 # if the host/group defines incoming tcp_ports - allow them
 {% if tcp_ports is defined %}
 {% for port in tcp_ports %}

roles/fas_server/templates/fas.cfg.j2

@@ -54,8 +54,8 @@ default_language = 'en'
 country_blacklist = ["--", "A1", "A2", "AN", "AS", "AX", "BI", "BL", "BV", "CC", "CU", "CV", "CX", "DM", "FK", "FO", "GF", "GG", "GP", "GS", "GW", "HM", "IO", "IR", "IQ", "JE", "KI", "KP", "MF", "MP", "MS", "MW", "NF", "NR", "NU", "PM", "PN", "RE", "SB", "SD", "SH", "SJ", "SY", "TC", "TF", "TK", "TL", "TV", "UM", "VC", "VG", "WF", "YT"]
 
 # Captcha
-tgcaptcha.key = '{{ fasCaptchaSecret }}'
+tgcaptcha2.key = '{{ fasCaptchaSecret }}'
-tgcaptcha.jpeg_generator = 'vanasco_dowty'
+tgcaptcha2.jpeg_generator = 'vanasco_dowty'
 
 ###
 ### Administrative settings

roles/fedmsg/base/tasks/main.yml

@@ -119,7 +119,7 @@
   - relay.py
   - logging.py
   - base.py
-  when: "'persistent-cloud' not in group_names"
+  when: "'persistent-cloud' not in group_names and 'qa-isolated' not in group_names"
   tags:
   - config
   - fedmsgdconfig
@@ -152,7 +152,7 @@
   - restart fedmsg-irc
   - restart fedmsg-relay
 
-- name: setup basic /etc/fedmsg.d/ contents for cloud hosts
+- name: setup basic /etc/fedmsg.d/ contents for firewalled/external hosts
   template: >
     src="{{ item }}.j2"
     dest="/etc/fedmsg.d/{{ item }}"
@@ -165,7 +165,7 @@
   - relay.py
   - logging.py
   - base.py
-  when: "'persistent-cloud' in group_names"
+  when: "'persistent-cloud' in group_names or 'qa-isolated' in group_names"
   tags:
   - config
   - fedmsgdconfig

roles/fedmsg/relay/tasks/main.yml

@@ -1,11 +1,19 @@
 # This is a *very* simple role.  The config needed for fedmsg-relay to operate
 # correctly is actually included as part of the fedmsg/base role.
 
-- name: install fedmsg-relay
+- name: install fedmsg-relay (yum)
   yum: pkg=fedmsg-relay state=present
   tags:
   - packages
   - fedmsg/relay
+  when: ansible_distribution_major_version|int < 22
+
+- name: install fedmsg-relay (dnf)
+  dnf: pkg=fedmsg-relay state=present
+  tags:
+  - packages
+  - fedmsg/relay
+  when: ansible_distribution_major_version|int > 21
 
 - name: ensure that nrpe has rights to monitor us
   file: >

roles/httpd/reverseproxy/tasks/main.yml

@@ -4,6 +4,7 @@
 # - remotepath..
 # - proxyurl
 # - rewrite
+# - keephost
 
 - name: Copy in ProxyPassReverse for {{destname}} ({{website}}{{remotepath}})
   template: >

roles/httpd/reverseproxy/templates/reversepassproxy.conf

@@ -9,5 +9,10 @@ RequestHeader set X-Scheme https early
 RequestHeader set X-Forwarded-Proto https early
 
 {% endif %}
+
+{% if keephost %}
+ProxyPreserveHost On
+{% endif %}
+
 ProxyPass {{ localpath }} {{ proxyurl }}{{remotepath}}
 ProxyPassReverse {{ localpath }} {{ proxyurl }}{{remotepath}}

roles/httpd/reverseproxy/vars/main.yml

@@ -3,3 +3,4 @@ localpath: /
 destname: reversepassproxy
 rewrite: false
 header_scheme: false
+keephost: false

roles/koji_builder/templates/kojid.conf

@@ -85,7 +85,7 @@ plugins = runroot
 
 {% else %}
 
-{% if env == 'staging' %}
+{% if env == 'staging' and not inventory_hostname.startswith('arm') %}
 plugins = builder_containerbuild
 {% else %}
 plugins =

roles/openqa/server/tasks/main.yml

@@ -36,18 +36,8 @@
 # need to ensure geekotest always has the same uid/gid if we re-deploy
 # the servers. So we create the account here with uid/gid 601.
 
-# this first one is just to make sure I don't screw up prod while I'm
-# fiddling with this stuff on stg
-- name: Check if geekotest already exists (temp)
-  command: "getent passwd geekotest"
-  register: gotgeeko
-  failed_when: "1 != 1"
-  changed_when: "1 != 1"
-  always_run: true
-
 - name: Create geekotest group with static GID 601
   group: name=geekotest gid=601 system=yes
-  when: "gotgeeko.rc > 0"
 
 - name: Create geekotest user with static UID 601
   user:
@@ -59,15 +49,15 @@
     createhome: no
     system: yes
     shell: /sbin/nologin
-  when: "gotgeeko.rc > 0"
 
-# note: we need updates-testing until fedfind 2.x and openQA 4.3-18 go stable
+# note: we need updates-testing until fedfind 2.x and openQA 4.3-21 go stable
 - name: Install required packages (testing)
   dnf: name={{ item }} state=present enablerepo="updates-testing"
   with_items:
   - python2-fedfind
   - openqa
   - openqa-httpd
+  - openqa-plugin-fedmsg
   tags:
   - packages
 
@@ -75,7 +65,6 @@
   dnf: name={{ item }} state=present enablerepo=adamwill-openQA
   with_items:
   - libselinux-python
-  - openqa
   - git
   - json_diff
   - libselinux-utils
@@ -88,7 +77,6 @@
   - libguestfs-xfs
   - python2-pexpect
   - python-libguestfs
-  - python2-fedfind
   tags:
   - packages
 

roles/openqa/server/templates/openqa.ini.j2

@@ -2,6 +2,7 @@
 branding = plain
 base_url = https://{{ external_hostname }}
 download_domains = fedoraproject.org
+plugins = Fedmsg
 
 [auth]
 method=OpenID

roles/phabricator/tasks/main.yml

@@ -173,6 +173,7 @@
       owner: root
       group: root
       mode: 0600
+  when: deployment_type == 'qadevel-prod'
 
 - name: Install the SSL key
   copy:
@@ -181,6 +182,7 @@
       owner: root
       group: root
       mode: 0600
+  when: deployment_type == 'qadevel-prod'
 
 - name: Install the SSL intermediate cert
   copy:
@@ -189,7 +191,7 @@
       owner: root
       group: root
       mode: 0600
-  when: sslintermediatecertfile != ''
+  when: sslintermediatecertfile != '' and deployment_type == 'qadevel-prod'
 
 
 - name: copy phabricator httpd config

roles/phabricator/templates/local.json.j2

@@ -1,5 +1,6 @@
 {
   "phabricator.base-uri" : "https:\/\/phab.{{ external_hostname }}\/",
+  "phabricator.allowed-uris": ["http:\/\/phab.{{ inventory_hostname }}\/"],
   "log.ssh.path"  : "\/var\/log\/phabricator\/ssh.log",
   "log.access.path" : "\/var\/log\/phabricator\/access.log",
   "mysql.host" : "{{ mariadb_host }}",

roles/phabricator/templates/phabricator.conf.j2

@@ -1,3 +1,4 @@
+{% if  deployment_type == "qadevel-prod" %}
 <VirtualHost *:443>
   # Change this to the domain which points to your host.
   ServerName phab.{{external_hostname}}
@@ -75,7 +76,20 @@
   RewriteEngine on
   RewriteRule ^/?(.*) https://%{SERVER_NAME}/$1 [R,L]
 </VirtualHost>
+{% else %}
+<VirtualHost *:80>
+  # Change this to the domain which points to your host.
+  ServerName phab.{{external_hostname}}
 
+  # Make sure you include "/webroot" at the end!
+  DocumentRoot {{phabroot}}/phabricator/webroot
+
+  RewriteEngine on
+  RewriteRule ^/rsrc/(.*)     -                       [L,QSA]
+  RewriteRule ^/favicon.ico   -                       [L,QSA]
+  RewriteRule ^(.*)$          /index.php?__path__=$1  [B,L,QSA]
+</VirtualHost>
+{% endif %}
 
 <Directory "{{ phabroot }}/phabricator/webroot">
   AllowOverride None