openqa/{server,worker}: enhance package handling

This provides a mechanism for deploying scratch builds, and also
for controlling whether or not to install openQA and os-autoinst
from updates-testing.

I have been doing the scratch build thing for years already, just
manually by ssh'ing into the boxes. This is getting tiring now
we have like 15 worker hosts.

The scratch build mechanism isn't properly idempotent, but fixing
that would be hard and I really only intend to use it transiently
when I'm updating the packages, so I don't think it's worth the
effort.

This also adds a notification for restarting openQA worker
services when the packages or config are updated, and fixes the
worker playbook to enable the last worker service.

Signed-off-by: Adam Williamson <awilliam@redhat.com>

inventory/group_vars/openqa_stg

@@ -36,6 +36,10 @@ openqa_env: staging
 # for now, we have the 'extra' arches (ppc64 and aarch64) on stg
 openqa_extraarches: true
 
+# install openQA from updates-testing - this is staging, we live
+# ON THE EDGE (radical guitar riff)
+openqa_repo: updates-testing
+
 wikitcms_token: "{{ private }}/files/openidc/staging/wikitcms.json"
 openqa_wikitcms_hostname: stg.fedoraproject.org
 openqa_resultsdb_url: http://resultsdb-stg01.qa.fedoraproject.org/resultsdb_api/api/v2.0/

inventory/group_vars/openqa_stg_workers

@@ -11,7 +11,9 @@ openqa_secret: "{{ stg_openqa_apisecret }}"
 openqa_env_suffix: .stg
 openqa_env: staging
 
-# These boxes are F30+, so we need Python 3 ansible
+# install openQA and os-autoinst from updates-testing - this is
+# staging, we live ON THE EDGE (radical guitar riff)
+openqa_repo: updates-testing
 
 deployment_type: stg
 freezes: false

roles/openqa/server/defaults/main.yml

@@ -2,3 +2,4 @@ openqa_webapi_plugins: FedoraUpdateRestart
 openqa_amqp_publisher_prefix: 
 openqa_amqp_publisher_url: amqp://test:@localhost/%2Fpubsub
 openqa_amqp_publisher_exchange: amq.topic
+openqa_repo: updates

roles/openqa/server/files/scratchrepo.conf

@@ -0,0 +1,6 @@
+[scratchrepo]
+name=Scratch buld repo
+baseurl=file:///var/tmp/scratchrepo
+enabled=1
+metadata_expire=30
+gpgcheck=0

roles/openqa/server/tasks/main.yml

@@ -22,6 +22,10 @@
 ##                also set openqa_amqp_publisher_prefix, openqa_amqp_publisher_url
 ##                and openqa_amqp_publisher_exchange
 ##       default - FedoraUpdateRestart
+# - openqa_repo
+##      string - Repo to enable when updating openQA packages. Set to
+##               'updates-testing' to use packages from updates-testing
+##      default - 'updates', which is effectively a no-op
 
 # Optional vars
 # - openqa_static_uid
@@ -68,6 +72,14 @@
 ##      string - Fedora Infrastructure thing; for this role, applies an
 ##               infra-specific tweak to httpd config. Don't set it outside
 ##               Fedora infra.
+# - openqa_scratch
+##      list -   A list of task IDs of scratch builds to install. If set, we
+##               will download the scratch build(s) and set up a side repo
+##               containing them. We do this because openQA staging is our
+##               main platform for testing new openQA releases; we need to
+##               be able to run scratch builds to see if a new release is
+##               really bad before submitting it as an update, we don't want
+##               to send new builds to updates-testing if they have problems
 
 # If openqa_dbhost is set, the other openqa_db* variables must be too,
 # and the server will be configured to use a pgsql database accordingly.
@@ -90,14 +102,44 @@
     shell: /sbin/nologin
   when: "openqa_static_uid is defined"
 
-# this is separate from the step below so we can easily flip it between
-# stable and testing
+- name: Remove scratch repo directory
+  file: path=/var/tmp/scratchrepo state=absent
+
+- name: (Re-)create scratch repo directory
+  file: path=/var/tmp/scratchrepo state=directory owner=root group=root
+  when: "openqa_scratch is defined"
+
+- name: Install Koji CLI client
+  package:
+    name: koji
+    state: present
+  tags:
+  - packages
+  when: "openqa_scratch is defined"
+
+- name: Download scratch builds
+  command: "koji download-task --arch=noarch --arch={{ ansible_architecture }} {{ item }}"
+  args:
+    chdir: /var/tmp/scratchrepo
+  loop: "{{ openqa_scratch|flatten(levels=1) }}"
+  when: "openqa_scratch is defined"
+
+- name: Write scratch build repo config
+  copy: src=scratchrepo.conf dest=/etc/yum.repos.d/scratchrepo.conf owner=root group=root mode=0644
+  when: "openqa_scratch is defined"
+
+- name: Delete scratch build repo config
+  file: path=/etc/yum.repos.d/scratchrepo.conf state=absent
+  when: "openqa_scratch is not defined"
+
+# this is separate from the step below so we can use openqa_repo just
+# for these packages
 - name: Install openQA packages
   package:
     name: ['openqa', 'openqa-httpd', 'openqa-plugin-fedora-messaging', 'openqa-plugin-fedoraupdaterestart',
            'python3-fedfind']
-    state: present
-#    enablerepo: "updates-testing"
+    state: latest
+    enablerepo: "{{ openqa_repo }}"
   tags:
   - packages
 

roles/openqa/worker/defaults/main.yml

@@ -1 +1,2 @@
 openqa_hostname: localhost
+openqa_repo: updates

roles/openqa/worker/files/scratchrepo.conf

@@ -0,0 +1,6 @@
+[scratchrepo]
+name=Scratch buld repo
+baseurl=file:///var/tmp/scratchrepo
+enabled=1
+metadata_expire=30
+gpgcheck=0

roles/openqa/worker/handlers/main.yml

@@ -0,0 +1,6 @@
+# Restart handler for worker services
+- name: Conditionally restart openQA workers
+  command: /usr/local/bin/conditional-restart.sh openqa-worker@{{ item }}
+  loop: "{{ range(1, {{ openqa_workers }} + 1)|list }}"
+  listen:
+  - restart openqa workers

roles/openqa/worker/tasks/main.yml

@@ -6,6 +6,10 @@
 # - openqa_hostname
 ##      string - hostname of openQA server to run jobs for
 ##      default - localhost
+# - openqa_repo
+##      string - Repo to enable when updating openQA packages. Set to
+##               'updates-testing' to use packages from updates-testing
+##      default - 'updates', which is effectively a no-op
 
 # Optional vars
 # - openqa_tap
@@ -16,12 +20,52 @@
 ##               installed and rngd.service enabled/started
 # - openqa_worker_class
 ##      string - custom WORKER_CLASS value for workers.ini
+# - openqa_scratch
+##      list -   A list of task IDs of scratch builds to install. If set, we
+##               will download the scratch build(s) and set up a side repo
+##               containing them. We do this because openQA staging is our
+##               main platform for testing new openQA releases; we need to
+##               be able to run scratch builds to see if a new release is
+##               really bad before submitting it as an update, we don't want
+##               to send new builds to updates-testing if they have problems
 
-- name: Install required packages (testing)
+- name: Remove scratch repo directory
+  file: path=/var/tmp/scratchrepo state=absent
+
+- name: (Re-)create scratch repo directory
+  file: path=/var/tmp/scratchrepo state=directory owner=root group=root
+  when: "openqa_scratch is defined"
+
+- name: Install Koji CLI client
   package:
-    name: ['openqa-worker']
+    name: koji
     state: present
-    enablerepo: "updates-testing"
+  tags:
+  - packages
+  when: "openqa_scratch is defined"
+
+- name: Download scratch builds
+  command: "koji download-task --arch=noarch --arch={{ ansible_architecture }} {{ item }}"
+  args:
+    chdir: /var/tmp/scratchrepo
+  loop: "{{ openqa_scratch|flatten(levels=1) }}"
+  when: "openqa_scratch is defined"
+
+- name: Write scratch build repo config
+  copy: src=scratchrepo.conf dest=/etc/yum.repos.d/scratchrepo.conf owner=root group=root mode=0644
+  when: "openqa_scratch is defined"
+
+- name: Delete scratch build repo config
+  file: path=/etc/yum.repos.d/scratchrepo.conf state=absent
+  when: "openqa_scratch is not defined"
+
+- name: Install required packages
+  package:
+    name: ['openqa-worker', 'os-autoinst']
+    state: latest
+    enablerepo: "{{ openqa_repo }}"
+  notify:
+  - restart openqa workers
   tags:
   - packages
 
@@ -91,6 +135,8 @@
 
 - name: openQA worker config
   template: src=workers.ini.j2 dest=/etc/openqa/workers.ini owner=_openqa-worker group=root mode=0644
+  notify:
+  - restart openqa workers
   tags:
   - config
 
@@ -99,4 +145,4 @@
 
 - name: Enable and start worker services
   service: name=openqa-worker@{{ item }} enabled=yes state=started
-  with_sequence: "count={{ openqa_workers }}"
+  loop: "{{ range(1, {{ openqa_workers }} + 1)|list }}"

roles/openqa/worker/tasks/tap-setup.yml

@@ -1,7 +1,8 @@
 - name: Install packages
   package:
     name: ['os-autoinst-openvswitch', 'tunctl', 'network-scripts']
-    state: present
+    state: latest
+    enablerepo: "{{ openqa_repo }}"
   tags:
   - packages