From bb1525bdef1d446e4d040e0444b6b6bba7f1d750 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Thu, 30 Apr 2020 12:18:32 -0700 Subject: [PATCH] openqa/{server,worker}: enhance package handling This provides a mechanism for deploying scratch builds, and also for controlling whether or not to install openQA and os-autoinst from updates-testing. I have been doing the scratch build thing for years already, just manually by ssh'ing into the boxes. This is getting tiring now we have like 15 worker hosts. The scratch build mechanism isn't properly idempotent, but fixing that would be hard and I really only intend to use it transiently when I'm updating the packages, so I don't think it's worth the effort. This also adds a notification for restarting openQA worker services when the packages or config are updated, and fixes the worker playbook to enable the last worker service. Signed-off-by: Adam Williamson --- inventory/group_vars/openqa_stg | 4 ++ inventory/group_vars/openqa_stg_workers | 4 +- roles/openqa/server/defaults/main.yml | 1 + roles/openqa/server/files/scratchrepo.conf | 6 +++ roles/openqa/server/tasks/main.yml | 50 ++++++++++++++++++-- roles/openqa/worker/defaults/main.yml | 1 + roles/openqa/worker/files/scratchrepo.conf | 6 +++ roles/openqa/worker/handlers/main.yml | 6 +++ roles/openqa/worker/tasks/main.yml | 54 ++++++++++++++++++++-- roles/openqa/worker/tasks/tap-setup.yml | 3 +- 10 files changed, 125 insertions(+), 10 deletions(-) create mode 100644 roles/openqa/server/files/scratchrepo.conf create mode 100644 roles/openqa/worker/files/scratchrepo.conf create mode 100644 roles/openqa/worker/handlers/main.yml diff --git a/inventory/group_vars/openqa_stg b/inventory/group_vars/openqa_stg index 5bb19e4460..c494ff3e22 100644 --- a/inventory/group_vars/openqa_stg +++ b/inventory/group_vars/openqa_stg @@ -36,6 +36,10 @@ openqa_env: staging # for now, we have the 'extra' arches (ppc64 and aarch64) on stg openqa_extraarches: true +# install openQA from updates-testing - this is staging, we live +# ON THE EDGE (radical guitar riff) +openqa_repo: updates-testing + wikitcms_token: "{{ private }}/files/openidc/staging/wikitcms.json" openqa_wikitcms_hostname: stg.fedoraproject.org openqa_resultsdb_url: http://resultsdb-stg01.qa.fedoraproject.org/resultsdb_api/api/v2.0/ diff --git a/inventory/group_vars/openqa_stg_workers b/inventory/group_vars/openqa_stg_workers index 1d4f3e4a2f..599a980339 100644 --- a/inventory/group_vars/openqa_stg_workers +++ b/inventory/group_vars/openqa_stg_workers @@ -11,7 +11,9 @@ openqa_secret: "{{ stg_openqa_apisecret }}" openqa_env_suffix: .stg openqa_env: staging -# These boxes are F30+, so we need Python 3 ansible +# install openQA and os-autoinst from updates-testing - this is +# staging, we live ON THE EDGE (radical guitar riff) +openqa_repo: updates-testing deployment_type: stg freezes: false diff --git a/roles/openqa/server/defaults/main.yml b/roles/openqa/server/defaults/main.yml index f45215d474..534491ffe2 100644 --- a/roles/openqa/server/defaults/main.yml +++ b/roles/openqa/server/defaults/main.yml @@ -2,3 +2,4 @@ openqa_webapi_plugins: FedoraUpdateRestart openqa_amqp_publisher_prefix: openqa_amqp_publisher_url: amqp://test:@localhost/%2Fpubsub openqa_amqp_publisher_exchange: amq.topic +openqa_repo: updates diff --git a/roles/openqa/server/files/scratchrepo.conf b/roles/openqa/server/files/scratchrepo.conf new file mode 100644 index 0000000000..d25cc78137 --- /dev/null +++ b/roles/openqa/server/files/scratchrepo.conf @@ -0,0 +1,6 @@ +[scratchrepo] +name=Scratch buld repo +baseurl=file:///var/tmp/scratchrepo +enabled=1 +metadata_expire=30 +gpgcheck=0 diff --git a/roles/openqa/server/tasks/main.yml b/roles/openqa/server/tasks/main.yml index 760b69ee65..89e6f2e1dc 100644 --- a/roles/openqa/server/tasks/main.yml +++ b/roles/openqa/server/tasks/main.yml @@ -22,6 +22,10 @@ ## also set openqa_amqp_publisher_prefix, openqa_amqp_publisher_url ## and openqa_amqp_publisher_exchange ## default - FedoraUpdateRestart +# - openqa_repo +## string - Repo to enable when updating openQA packages. Set to +## 'updates-testing' to use packages from updates-testing +## default - 'updates', which is effectively a no-op # Optional vars # - openqa_static_uid @@ -68,6 +72,14 @@ ## string - Fedora Infrastructure thing; for this role, applies an ## infra-specific tweak to httpd config. Don't set it outside ## Fedora infra. +# - openqa_scratch +## list - A list of task IDs of scratch builds to install. If set, we +## will download the scratch build(s) and set up a side repo +## containing them. We do this because openQA staging is our +## main platform for testing new openQA releases; we need to +## be able to run scratch builds to see if a new release is +## really bad before submitting it as an update, we don't want +## to send new builds to updates-testing if they have problems # If openqa_dbhost is set, the other openqa_db* variables must be too, # and the server will be configured to use a pgsql database accordingly. @@ -90,14 +102,44 @@ shell: /sbin/nologin when: "openqa_static_uid is defined" -# this is separate from the step below so we can easily flip it between -# stable and testing +- name: Remove scratch repo directory + file: path=/var/tmp/scratchrepo state=absent + +- name: (Re-)create scratch repo directory + file: path=/var/tmp/scratchrepo state=directory owner=root group=root + when: "openqa_scratch is defined" + +- name: Install Koji CLI client + package: + name: koji + state: present + tags: + - packages + when: "openqa_scratch is defined" + +- name: Download scratch builds + command: "koji download-task --arch=noarch --arch={{ ansible_architecture }} {{ item }}" + args: + chdir: /var/tmp/scratchrepo + loop: "{{ openqa_scratch|flatten(levels=1) }}" + when: "openqa_scratch is defined" + +- name: Write scratch build repo config + copy: src=scratchrepo.conf dest=/etc/yum.repos.d/scratchrepo.conf owner=root group=root mode=0644 + when: "openqa_scratch is defined" + +- name: Delete scratch build repo config + file: path=/etc/yum.repos.d/scratchrepo.conf state=absent + when: "openqa_scratch is not defined" + +# this is separate from the step below so we can use openqa_repo just +# for these packages - name: Install openQA packages package: name: ['openqa', 'openqa-httpd', 'openqa-plugin-fedora-messaging', 'openqa-plugin-fedoraupdaterestart', 'python3-fedfind'] - state: present -# enablerepo: "updates-testing" + state: latest + enablerepo: "{{ openqa_repo }}" tags: - packages diff --git a/roles/openqa/worker/defaults/main.yml b/roles/openqa/worker/defaults/main.yml index 42aacce60c..651151c9d9 100644 --- a/roles/openqa/worker/defaults/main.yml +++ b/roles/openqa/worker/defaults/main.yml @@ -1 +1,2 @@ openqa_hostname: localhost +openqa_repo: updates diff --git a/roles/openqa/worker/files/scratchrepo.conf b/roles/openqa/worker/files/scratchrepo.conf new file mode 100644 index 0000000000..d25cc78137 --- /dev/null +++ b/roles/openqa/worker/files/scratchrepo.conf @@ -0,0 +1,6 @@ +[scratchrepo] +name=Scratch buld repo +baseurl=file:///var/tmp/scratchrepo +enabled=1 +metadata_expire=30 +gpgcheck=0 diff --git a/roles/openqa/worker/handlers/main.yml b/roles/openqa/worker/handlers/main.yml new file mode 100644 index 0000000000..d5a1ff4cce --- /dev/null +++ b/roles/openqa/worker/handlers/main.yml @@ -0,0 +1,6 @@ +# Restart handler for worker services +- name: Conditionally restart openQA workers + command: /usr/local/bin/conditional-restart.sh openqa-worker@{{ item }} + loop: "{{ range(1, {{ openqa_workers }} + 1)|list }}" + listen: + - restart openqa workers diff --git a/roles/openqa/worker/tasks/main.yml b/roles/openqa/worker/tasks/main.yml index 778cf23d76..8a53909560 100644 --- a/roles/openqa/worker/tasks/main.yml +++ b/roles/openqa/worker/tasks/main.yml @@ -6,6 +6,10 @@ # - openqa_hostname ## string - hostname of openQA server to run jobs for ## default - localhost +# - openqa_repo +## string - Repo to enable when updating openQA packages. Set to +## 'updates-testing' to use packages from updates-testing +## default - 'updates', which is effectively a no-op # Optional vars # - openqa_tap @@ -16,12 +20,52 @@ ## installed and rngd.service enabled/started # - openqa_worker_class ## string - custom WORKER_CLASS value for workers.ini +# - openqa_scratch +## list - A list of task IDs of scratch builds to install. If set, we +## will download the scratch build(s) and set up a side repo +## containing them. We do this because openQA staging is our +## main platform for testing new openQA releases; we need to +## be able to run scratch builds to see if a new release is +## really bad before submitting it as an update, we don't want +## to send new builds to updates-testing if they have problems -- name: Install required packages (testing) +- name: Remove scratch repo directory + file: path=/var/tmp/scratchrepo state=absent + +- name: (Re-)create scratch repo directory + file: path=/var/tmp/scratchrepo state=directory owner=root group=root + when: "openqa_scratch is defined" + +- name: Install Koji CLI client package: - name: ['openqa-worker'] + name: koji state: present - enablerepo: "updates-testing" + tags: + - packages + when: "openqa_scratch is defined" + +- name: Download scratch builds + command: "koji download-task --arch=noarch --arch={{ ansible_architecture }} {{ item }}" + args: + chdir: /var/tmp/scratchrepo + loop: "{{ openqa_scratch|flatten(levels=1) }}" + when: "openqa_scratch is defined" + +- name: Write scratch build repo config + copy: src=scratchrepo.conf dest=/etc/yum.repos.d/scratchrepo.conf owner=root group=root mode=0644 + when: "openqa_scratch is defined" + +- name: Delete scratch build repo config + file: path=/etc/yum.repos.d/scratchrepo.conf state=absent + when: "openqa_scratch is not defined" + +- name: Install required packages + package: + name: ['openqa-worker', 'os-autoinst'] + state: latest + enablerepo: "{{ openqa_repo }}" + notify: + - restart openqa workers tags: - packages @@ -91,6 +135,8 @@ - name: openQA worker config template: src=workers.ini.j2 dest=/etc/openqa/workers.ini owner=_openqa-worker group=root mode=0644 + notify: + - restart openqa workers tags: - config @@ -99,4 +145,4 @@ - name: Enable and start worker services service: name=openqa-worker@{{ item }} enabled=yes state=started - with_sequence: "count={{ openqa_workers }}" + loop: "{{ range(1, {{ openqa_workers }} + 1)|list }}" diff --git a/roles/openqa/worker/tasks/tap-setup.yml b/roles/openqa/worker/tasks/tap-setup.yml index 730ed13aea..14ec0aafe0 100644 --- a/roles/openqa/worker/tasks/tap-setup.yml +++ b/roles/openqa/worker/tasks/tap-setup.yml @@ -1,7 +1,8 @@ - name: Install packages package: name: ['os-autoinst-openvswitch', 'tunctl', 'network-scripts'] - state: present + state: latest + enablerepo: "{{ openqa_repo }}" tags: - packages