Merge branch 'master' of /git/ansible

playbooks/groups/mailman.yml

@@ -117,6 +117,7 @@
     - httpd
     - mailman3
     - postfix
+    when: inventory_hostname.startswith('mailman01.phx2') or inventory_hostname.startswith('lists-dev')
 
   handlers:
   - include: "{{ handlers }}/restart_services.yml"

playbooks/manual/rebuild/hotspot.txt

@@ -0,0 +1 @@
+OK

playbooks/manual/rebuild/hotspot.yml

@@ -0,0 +1,11 @@
+- name: Put a hotspot.txt file in place.
+  hosts: proxies;proxies-stg
+  user: root
+  vars_files: 
+   - /srv/web/infra/ansible/vars/global.yml
+   - "/srv/private/ansible/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+
+  tasks:
+  - file: dest=/srv/web/fedoraproject.org/static/ state=directory
+  - copy: src=hotspot.txt dest=/srv/web/fedoraproject.org/static/hotspot.txt

playbooks/manual/rebuild/websites.yml

@@ -0,0 +1,37 @@
+- name: Force a rebuild of website content on the backend builder
+  hosts: sundries01*
+  user: root
+  become: true
+  become_user: apache
+  vars_files: 
+   - /srv/web/infra/ansible/vars/global.yml
+   - "/srv/private/ansible/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+
+  tasks:
+  - name: Run syncStatic (this takes a while)...
+    command: /usr/local/bin/lock-wrapper syncStatic /usr/local/bin/syncStatic
+
+- name: Tell the proxies to sync that freshness
+  hosts: proxies;proxies-stg
+  user: root
+  vars_files: 
+   - /srv/web/infra/ansible/vars/global.yml
+   - "/srv/private/ansible/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+
+  tasks:
+  - name: rsync each site in to place
+    command: /usr/bin/rsync --delete -a --no-owner --no-group sundries01::{{item}}/ /srv/web/{{item}}/
+    with_items:
+    - getfedora.org
+    - arm.fedoraproject.org
+    - boot.fedoraproject.org
+    - fedoracommunity.org
+    - developer.fedoraproject.org
+    - fedoraproject.org
+    - fudcon.fedoraproject.org
+    - labs.fedoraproject.org
+    - mirrors.fedoraproject.org
+    - spins.fedoraproject.org
+    - start.fedoraproject.org

playbooks/vhost_reboot.yml

@@ -35,7 +35,7 @@
 
 # Call out to another playbook.  Disable any proxies that may live here
 - include: update-proxy-dns.yml status=disable proxies=myvms_new:&proxies
-  when: nodns is not defined or not "true" in nodns
+  when: nodns is not defined or not nodns
 
 - name: halt instances
   hosts: myvms_new
@@ -48,7 +48,7 @@
     nagios: action=downtime minutes=30 service=host host={{ inventory_hostname_short }}{{ env_suffix }}
     delegate_to: noc01.phx2.fedoraproject.org
     ignore_errors: true
-    when: nonagios is not defined or not "true" in nonagios
+    when: nonagios is not defined or not nonagios
 
   - name: halt the vm instances - to poweroff
     command: /sbin/shutdown -h 1
@@ -74,7 +74,7 @@
     nagios: action=downtime minutes=60 service=host host={{ inventory_hostname_short }}{{ env_suffix }}
     delegate_to: noc01.phx2.fedoraproject.org
     ignore_errors: true
-    when: nonagios is not defined or not "true" in nonagios
+    when: nonagios is not defined or not nonagios
 
   - name: reboot the virthost
     command: /sbin/shutdown -r 1
@@ -100,11 +100,11 @@
     nagios: action=unsilence service=host host={{ inventory_hostname_short }}{{ env_suffix }}
     delegate_to: noc01.phx2.fedoraproject.org
     ignore_errors: true
-    when: nonagios is not defined or not "true" in nonagios
+    when: nonagios is not defined or not nonagios
 
 # Call out to that dns playbook.  Put proxies back in now that they're back
 - include: update-proxy-dns.yml status=enable proxies=myvms_new:&proxies
-  when: not "true" in nodns
+  when: nodns is not defined or not nodns
 
 - name: Fix unbound if necessary
   # intersection - hosts that are in our dynamic group and also in unbound-dns

roles/check-compose/tasks/main.yml

@@ -40,10 +40,12 @@
     repo: https://github.com/os-autoinst/openQA-python-client.git
     dest: /root/openQA-python-client
   register: gitclient
+  always_run: true
 
 - name: Check if python-client has ever been installed
   command: "python -c 'import openqa_client'"
   register: instclient
+  always_run: true
   changed_when: "1 != 1"
   failed_when: "1 != 1"
 
@@ -58,6 +60,7 @@
     repo: https://git.fedorahosted.org/git/fedora-qa.git
     dest: /root/fedora-qa
   register: gitqa
+  always_run: true
 
 - name: Build check-compose
   command: "make prefix=/usr"

roles/distgit/templates/pkgdb_sync_git_branches.py

@@ -217,6 +217,10 @@ def branch_package(ns, pkgname, requested_branches, existing_branches):
     if not exists or 'master' not in existing_branches:
         if not TEST_ONLY:
             _invoke(SETUP_PACKAGE, [os.path.join(ns, pkgname)])
+            if ns == 'rpms':
+                old_place = os.path.exists(os.path.join(
+                    GIT_FOLDER, '%s.git' % pkgname))
+                os.symlink(exists, old_place)
             # SETUP_PACKAGE creates master
             if 'master' in requested_branches:
                 requested_branches.remove('master')

roles/fedora-web/build/files/syncStatic.sh

@@ -68,3 +68,4 @@ build boot.fedoraproject.org
 build fedoracommunity.org
 build fudcon.fedoraproject.org
 build start.fedoraproject.org
+build fedoraproject.org

roles/fedora-web/main/files/hotspot.conf

@@ -0,0 +1 @@
+Alias /static/hotspot.txt /srv/web/hotspot.txt

roles/fedora-web/main/tasks/main.yml

@@ -13,6 +13,7 @@
   - languages.conf
   - cache.conf
   - persona.conf
+  - hotspot.conf
   - expires.conf
   - deflate.conf
   notify:
@@ -20,6 +21,7 @@
   tags:
   - fedora-web
   - fedora-web/main
+  - fedora-web/hotspot
 
 - name: And one template (for {{website}})
   template: >
@@ -33,6 +35,15 @@
   - fedora-web
   - fedora-web/main
 
+- name: Create a file for hotspot
+  copy: dest=/srv/web/hotspot.txt content="OK"
+        owner=root group=root mode=0644
+        setype=httpd_sys_content_t
+  tags:
+  - fedora-web
+  - fedora-web/main
+  - fedora-web/hotspot
+
 - name: And, copy over a template for browserid
   template: >
     src=browserid.fedoraproject.org dest=/srv/web/browserid.fedoraproject.org

roles/mailman/tasks/main.yml

@@ -426,3 +426,4 @@
   - mailman3
   - postfix
   tags: mailman
+  when: inventory_hostname.startswith('mailman01.phx2') or inventory_hostname.startswith('lists-dev')

roles/nagios_server/files/nagios/misccommands.cfg

@@ -86,13 +86,13 @@ define command{
 # 'host-notify-by-fedmsg' command definition
 define command{
 	command_name	host-notify-by-fedmsg
-    command_line    /usr/bin/echo '{"type": "$NOTIFICATIONTYPE$", "host": "$HOSTALIAS$", "state": "$HOSTSTATE$", "host_ack_author": "$HOSTACKAUTHOR$", "service_ack_author": "$SERVICEACKAUTHOR$"}' | fedmsg-logger --cert-prefix nagios --modname nagios --topic host.state.change --json-input
+    command_line    /usr/bin/echo '{"type": "$NOTIFICATIONTYPE$", "host": "$HOSTALIAS$", "state": "$HOSTSTATE$", "output": "$HOSTOUTPUT$", "host_ack_author": "$HOSTACKAUTHOR$", "service_ack_author": "$SERVICEACKAUTHOR$"}' | fedmsg-logger --cert-prefix nagios --modname nagios --topic host.state.change --json-input
 	}
 
 # 'notify-by-fedmsg' command definition
 define command{
 	command_name	notify-by-fedmsg
-    command_line    /usr/bin/echo '{"type": "$NOTIFICATIONTYPE$", "host": "$HOSTALIAS$", "state": "$SERVICESTATE$", "host_ack_author": "$HOSTACKAUTHOR$", "service_ack_author": "$SERVICEACKAUTHOR$"}' | fedmsg-logger --cert-prefix nagios --modname nagios --topic service.state.change --json-input
+    command_line    /usr/bin/echo '{"type": "$NOTIFICATIONTYPE$", "host": "$HOSTALIAS$", "state": "$SERVICESTATE$", "service": "$SERVICEDESC$", "output": "$SERVICEOUTPUT$", "host_ack_author": "$HOSTACKAUTHOR$", "service_ack_author": "$SERVICEACKAUTHOR$"}' | fedmsg-logger --cert-prefix nagios --modname nagios --topic service.state.change --json-input
 	}
 
 # 'notify-by-xmpp' command definition

roles/nagios_server/files/nagios/services/websites.cfg

@@ -48,6 +48,7 @@ define service {
   service_description   mm-publiclist-internal
   check_command         check_website_publiclist!localhost!/mirrormanager/api/mirroradmins/?name=dl.fedoraproject.org
   use                   internalwebsitetemplate
+  event_handler         restart_httpd
 }
 
 define service {

roles/openqa/dispatcher/tasks/main.yml

@@ -37,12 +37,14 @@
     repo: https://github.com/os-autoinst/openQA-python-client.git
     dest: /root/openQA-python-client
   register: gitclient
+  always_run: true
 
 - name: Check if python-client has ever been installed
   command: "python -c 'import openqa_client'"
   register: instclient
   changed_when: "1 != 1"
   failed_when: "1 != 1"
+  always_run: true
 
 - name: Install openQA-python-client
   command: "python setup.py install"
@@ -57,12 +59,14 @@
     repo: https://bitbucket.org/rajcze/openqa_fedora_tools
     dest: /root/openqa_fedora_tools-dispatcher
   register: gittools
+  always_run: true
 
 - name: Check if openqa_fedora_tools has ever been installed
   stat: path=/usr/bin/fedora-openqa-schedule
   register: insttools
   changed_when: "1 != 1"
   failed_when: "1 != 1"
+  always_run: true
 
 - name: Install openqa_fedora_tools
   command: "python setup.py install"

roles/openqa/server/tasks/main.yml

@@ -50,6 +50,7 @@
   - libselinux-python
   - openqa
   - git
+  - json_diff
   - libselinux-utils
   - libsemanage-python
   - nfs-utils
@@ -66,9 +67,9 @@
     repo: https://bitbucket.org/rajcze/openqa_fedora
     dest: /var/lib/openqa/share/tests/fedora
   register: gittests
-
+  sudo: true
-- name: Have tests owned by geekotest
+  sudo_user: geekotest
-  file: path=/var/lib/openqa/share/tests/fedora owner=geekotest recurse=yes
+  always_run: true
 
 - name: Check out openqa_fedora_tools
   git:
@@ -82,38 +83,42 @@
   - /var/lib/openqa/share/factory/hdd
   - /var/lib/openqa/share/factory/repo
 
-- name: Start libvirt service
-  service: name=libvirtd state=started
-
 # NOTE: this is very hacky, but we can't do much better with the current
 # disk creation script, I will try and make it better. We'll have to bump
-# this hardcoded release number every so often.
+# this hardcoded release number every so often. Much better createhdds
+# is in review: https://phab.qadevel.cloud.fedoraproject.org/D687
+# LIBGUESTFS_BACKEND is to avoid the need to run libvirt just for this
 - name: Create hard disk images (this may take a long time!)
   command: "/root/openqa_fedora_tools/tools/createhdds.sh 23"
   args:
     creates: /var/lib/openqa/share/factory/hdd/disk_full_mbr.img
     chdir: /var/lib/openqa/share/factory/hdd/
+  environment:
+    LIBGUESTFS_BACKEND: direct
 
 - name: Create N-2 desktop x86_64 disk image (this may take a long time!)
   command: "/root/openqa_fedora_tools/tools/createhdds.sh 22 desktop_64bit"
   args:
     creates: /var/lib/openqa/share/factory/hdd/disk_f22_desktop_x86_64.img
     chdir: /var/lib/openqa/share/factory/hdd/
+  environment:
+    LIBGUESTFS_BACKEND: direct
 
 - name: Create N-2 desktop i686 disk image (this may take a long time!)
   command: "/root/openqa_fedora_tools/tools/createhdds.sh 22 desktop_32bit"
   args:
     creates: /var/lib/openqa/share/factory/hdd/disk_f22_desktop_i686.img
     chdir: /var/lib/openqa/share/factory/hdd/
+  environment:
+    LIBGUESTFS_BACKEND: direct
 
 - name: Create N-2 minimal x86_64 disk image (this may take a long time!)
   command: "/root/openqa_fedora_tools/tools/createhdds.sh 22 minimal_64bit"
   args:
     creates: /var/lib/openqa/share/factory/hdd/disk_f22_minimal_x86_64.img
     chdir: /var/lib/openqa/share/factory/hdd/
-
+  environment:
-- name: Stop libvirt service
+    LIBGUESTFS_BACKEND: direct
-  service: name=libvirtd state=stopped
 
 - name: Create exports file
   copy: src=exports dest=/etc/exports.d/openqa.exports owner=root group=root mode=0644
@@ -160,6 +165,7 @@
   register: initdb
   changed_when: "initdb.rc == 0"
   failed_when: "(initdb.rc > 0) and (initdb.stderr is not defined or initdb.stderr.find('already exists') == -1)"
+  always_run: true
 
 - name: Enable and start services
   service: name={{ item }} enabled=yes state=started
@@ -169,6 +175,7 @@
   - openqa-webui
   - openqa-websockets
   - openqa-gru
+  always_run: true
 
 # This is using a big hammer until #1277312 is resolved
 - name: Allow Apache to connect to openQA
@@ -188,14 +195,30 @@
   register: admin
   changed_when: "admin.rc == 0"
   failed_when: "(admin.rc > 0) and (admin.stderr is not defined or admin.stderr.find('already exists') == -1)"
+  always_run: true
 
 - name: Patch repo URLs in templates
   shell: "cp /var/lib/openqa/share/tests/fedora/templates /tmp && sed -i -e 's,dl.fedoraproject,dl.phx2.fedoraproject,g' /tmp/templates"
   when: "gittests|changed"
   changed_when: "1 != 1"
 
-# This will always show as changed, even when nothing in the templates
+- name: Dump existing config for checking changes
-# changed; this is a bit hard to fix
+  shell: "/usr/share/openqa/script/dump_templates --json > /tmp/tmpl-old.json"
+  when: "gittests|changed"
+  changed_when: "1 != 1"
+
+# Because of the boring details of how template loading works, getting
+# a correct 'changed' for this step is too difficult. Instead we have
+# the prior and following steps; when the templates actually changed,
+# the *following* step will register as changed.
 - name: Load tests
   shell: "/tmp/templates --clean"
   when: "gittests|changed"
+  changed_when: "1 != 1"
+
+- name: Check if the tests changed in previous step
+  command: "/usr/share/openqa/script/dump_templates --json > /tmp/tmpl-new.json && json_diff /tmp/tmpl-old.json /tmp/tmpl-new.json"
+  when: "gittests|changed"
+  register: testsdiff
+  changed_when: "testsdiff.rc > 0"
+  failed_when: "1 != 1"