From b3c2a39e1fee83f058a50c07cceff708e98ae702 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 6 Dec 2022 14:39:13 -0800 Subject: [PATCH] koji_builder / osbuild: adjustments for osbuild api scripting Right now we run a script on all builders once a minute to update the api/auth ip's for osbuild. This has a number of problems: * Sometimes osbuild jobs land on s390x builders that have no internet access and hang or fail. * Sometimes the update script hangs or takes a long time to run because the builder is heavily loaded with builds, resulting in locking emails to sysadmin-main folks. So, in this commit we: * make a new koji channel called 'osbuild' with all the buildhw-x86's in it. They are usually not too overloaded and there are 16 of them so it should be available all the time. * Leave the cron job on all builders for now in case, but make them only update once a day since they won't be getting jobs. If this works out we can remove it entirely there. * Make the buildhw-x86s only update every 5min. This opens a larger window for it being wrong, but it's still pretty small and should reduce the number of emails for stalled processes we get. See https://pagure.io/fedora-infrastructure/issue/10982 Signed-off-by: Kevin Fenzi --- roles/koji_builder/files/osbuildapi-update.cron | 1 - roles/koji_builder/tasks/main.yml | 2 +- roles/koji_builder/templates/osbuildapi-update.cron | 5 +++++ roles/koji_hub/templates/hub.conf.j2 | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 roles/koji_builder/files/osbuildapi-update.cron create mode 100644 roles/koji_builder/templates/osbuildapi-update.cron diff --git a/roles/koji_builder/files/osbuildapi-update.cron b/roles/koji_builder/files/osbuildapi-update.cron deleted file mode 100644 index e352405041..0000000000 --- a/roles/koji_builder/files/osbuildapi-update.cron +++ /dev/null @@ -1 +0,0 @@ -* * * * * root /usr/local/bin/lock-wrapper osbuildapi "/usr/local/bin/osbuildapi-update.sh" 2>&1 | /usr/local/bin/nag-once osbuildapi-update.sh 1d 2>&1 diff --git a/roles/koji_builder/tasks/main.yml b/roles/koji_builder/tasks/main.yml index d8498ec363..9800f62ede 100644 --- a/roles/koji_builder/tasks/main.yml +++ b/roles/koji_builder/tasks/main.yml @@ -366,7 +366,7 @@ - koji_builder/osbuildapi - name: install cron job to run osbuild api ip update script every minute. - copy: src=osbuildapi-update.cron dest=/etc/cron.d/osbuildapi-update.cron mode=644 + template: src=osbuildapi-update.cron dest=/etc/cron.d/osbuildapi-update.cron mode=644 tags: - koji_builder - koji_builder/osbuildapi diff --git a/roles/koji_builder/templates/osbuildapi-update.cron b/roles/koji_builder/templates/osbuildapi-update.cron new file mode 100644 index 0000000000..9872862d63 --- /dev/null +++ b/roles/koji_builder/templates/osbuildapi-update.cron @@ -0,0 +1,5 @@ +{% if inventory_hostname.startswith('buildhw-x86') } +*/5 * * * * root /usr/local/bin/lock-wrapper osbuildapi "/usr/local/bin/osbuildapi-update.sh" 2>&1 | /usr/local/bin/nag-once osbuildapi-update.sh 1d 2>&1 +{% else } +23 17 * * * root /usr/local/bin/lock-wrapper osbuildapi "/usr/local/bin/osbuildapi-update.sh" 2>&1 | /usr/local/bin/nag-once osbuildapi-update.sh 1d 2>&1 +{% endif %} diff --git a/roles/koji_hub/templates/hub.conf.j2 b/roles/koji_hub/templates/hub.conf.j2 index ad034a5940..96a9ca78da 100644 --- a/roles/koji_hub/templates/hub.conf.j2 +++ b/roles/koji_hub/templates/hub.conf.j2 @@ -119,6 +119,7 @@ tag = all :: allow channel = + method osbuildImage :: use osbuild method newRepo distRepo buildSRPMFromSCM :: use createrepo method buildContainer :: use container has req_channel && has_perm customchannel :: req