From 68982b36e04b6dd1249484fffd2385109fbb3855 Mon Sep 17 00:00:00 2001 From: Adam Saleh Date: Thu, 13 May 2021 17:20:43 +0200 Subject: [PATCH 001/189] Moved bodhi in prod to use rpm for building the image. --- .../bodhi/templates/dockerfile-base | 67 +++---------------- 1 file changed, 9 insertions(+), 58 deletions(-) diff --git a/roles/openshift-apps/bodhi/templates/dockerfile-base b/roles/openshift-apps/bodhi/templates/dockerfile-base index 67aff7af84..ce7ae6435a 100644 --- a/roles/openshift-apps/bodhi/templates/dockerfile-base +++ b/roles/openshift-apps/bodhi/templates/dockerfile-base @@ -1,64 +1,15 @@ FROM fedora:34 LABEL \ - name="bodhi-base" \ - vendor="Fedora Infrastructure" \ - license="MIT" - -RUN curl -o /etc/yum.repos.d/infra-tags.repo https://pagure.io/fedora-infra/ansible/raw/main/f/files/common/fedora-infra-tags.repo - -RUN dnf install -y \ - git \ - python3-pip \ - fedora-messaging \ - httpd \ - intltool \ - origin-clients \ - python3-alembic \ - python3-arrow \ - python3-backoff \ - python3-bleach \ - python3-celery \ - python3-click \ - python3-colander \ - python3-cornice \ - python3-dogpile-cache \ - python3-fedora-messaging \ - python3-feedgen \ - python3-jinja2 \ - python3-markdown \ - python3-psycopg2 \ - python3-prometheus_client \ - python3-py3dns \ - python3-pyasn1-modules \ - python3-pylibravatar \ - python3-pyramid \ - python3-pyramid-fas-openid \ - python3-pyramid-mako \ - python3-pyramid_sawing \ - python3-bugzilla \ - python3-fedora \ - python3-pyyaml \ - python3-simplemediawiki \ - python3-sqlalchemy \ - python3-waitress \ - python3-dnf \ - python3-koji \ - python3-librepo \ - python3-mod_wsgi \ - python3-gunicorn \ - python3-whitenoise \ - && dnf --setopt='tsflags=' install -y bodhi-docs \ - && pip install koji graphene graphene-sqlalchemy WebOb-GraphQL \ - && dnf clean all - -RUN git clone -b {{bodhi_version}} https://github.com/fedora-infra/bodhi.git /srv/bodhi && \ - cd /srv/bodhi && \ - python3 -m pip install . --no-use-pep517 && \ - mkdir -p /usr/share/bodhi && \ - cp /srv/bodhi/apache/bodhi.wsgi /usr/share/bodhi/bodhi.wsgi + name="bodhi-base" \ + vendor="Fedora Infrastructure" \ + license="MIT" +RUN curl -o /etc/yum.repos.d/infra-tags.repo https://infrastructure.fedoraproject.org/infra/ansible/files/common/fedora-infra-tags.repo +# While dnf has a --nodocs, it doesen't have a --docs... +RUN sed -i '/nodocs/d' /etc/dnf/dnf.conf +RUN dnf install -y bodhi-server-{{bodhi_version}} bodhi-docs-{{bodhi_version}} python3-pyramid_sawing python3-gunicorn +# Set up krb5 RUN rm -f /etc/krb5.conf && \ ln -sf /etc/bodhi/krb5.conf /etc/krb5.conf && \ - ln -sf /etc/keytabs/koji-keytab /etc/krb5.bodhi_bodhi.fedoraproject.org.keytab - + ln -sf /etc/keytabs/koji-keytab /etc/krb5.bodhi_bodhi{{ env_suffix }}.fedoraproject.org.keytab ENV USER=openshift From 1cb04fe7b4dbc07581e00a743d7bf067ee29b964 Mon Sep 17 00:00:00 2001 From: Adam Saleh Date: Thu, 13 May 2021 17:34:28 +0200 Subject: [PATCH 002/189] Celery beat changed the order of args in bodhi. --- roles/openshift-apps/bodhi/templates/deploymentconfig.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/openshift-apps/bodhi/templates/deploymentconfig.yml b/roles/openshift-apps/bodhi/templates/deploymentconfig.yml index 2696a4446f..ede4c908c6 100644 --- a/roles/openshift-apps/bodhi/templates/deploymentconfig.yml +++ b/roles/openshift-apps/bodhi/templates/deploymentconfig.yml @@ -283,7 +283,7 @@ items: - name: bodhi-celery image: bodhi-base:latest command: ["/usr/bin/celery"] - args: ["worker", "-A", "bodhi.server.tasks.app", "-l", "info", "-Q", "celery"] + args: ["-A", "bodhi.server.tasks.app", "worker", "-l", "info", "-Q", "celery"] resources: {} volumeMounts: - name: config-volume @@ -378,7 +378,7 @@ items: image: bodhi-base:latest workingDir: "/tmp" command: ["/usr/bin/celery"] - args: ["beat", "-A", "bodhi.server.tasks.app", "-l", "info"] + args: ["-A", "bodhi.server.tasks.app", "beat", "-l", "info"] resources: {} volumeMounts: - name: config-volume From e204d829940229835c9f21e63a4d2159effb76dc Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 13 May 2021 12:39:42 -0700 Subject: [PATCH 003/189] rkhunter: add releng_compose_stg for podman use Signed-off-by: Kevin Fenzi --- roles/rkhunter/templates/rkhunter.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/rkhunter/templates/rkhunter.conf.j2 b/roles/rkhunter/templates/rkhunter.conf.j2 index 3f0d29c576..60faf47701 100644 --- a/roles/rkhunter/templates/rkhunter.conf.j2 +++ b/roles/rkhunter/templates/rkhunter.conf.j2 @@ -393,7 +393,7 @@ ALLOWDEVFILE=/dev/shm/spice.* {% if inventory_hostname in groups['ipa'] or inventory_hostname in groups['ipa_stg'] %} ALLOWDEVFILE=/dev/shm/sem.slapd*.stats {% endif %} -{% if inventory_hostname in groups['retrace'] or inventory_hostname in groups['releng_compose'] %} +{% if inventory_hostname in groups['retrace'] or inventory_hostname in groups['releng_compose'] or inventory_hostname in groups['releng_compose_stg]' %} ALLOWDEVFILE=/dev/shm/libpod_* {% endif %} {% if inventory_hostname in groups['dbserver'] or inventory_hostname in groups['dbserver_stg'] or inventory_hostname in groups['pkgs'] or inventory_hostname in groups['pagure'] or inventory_hostname in groups['pagure_stg'] or inventory_hostname in groups['zabbix_stg'] or inventory_hostname in groups['retrace'] %} From 18f4d49a9d09b86c33a35710f01ff68c12f57810 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 13 May 2021 12:41:05 -0700 Subject: [PATCH 004/189] rkhunter: fix bit of dyslexia Signed-off-by: Kevin Fenzi --- roles/rkhunter/templates/rkhunter.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/rkhunter/templates/rkhunter.conf.j2 b/roles/rkhunter/templates/rkhunter.conf.j2 index 60faf47701..6d6539eb09 100644 --- a/roles/rkhunter/templates/rkhunter.conf.j2 +++ b/roles/rkhunter/templates/rkhunter.conf.j2 @@ -393,7 +393,7 @@ ALLOWDEVFILE=/dev/shm/spice.* {% if inventory_hostname in groups['ipa'] or inventory_hostname in groups['ipa_stg'] %} ALLOWDEVFILE=/dev/shm/sem.slapd*.stats {% endif %} -{% if inventory_hostname in groups['retrace'] or inventory_hostname in groups['releng_compose'] or inventory_hostname in groups['releng_compose_stg]' %} +{% if inventory_hostname in groups['retrace'] or inventory_hostname in groups['releng_compose'] or inventory_hostname in groups['releng_compose_stg'] %} ALLOWDEVFILE=/dev/shm/libpod_* {% endif %} {% if inventory_hostname in groups['dbserver'] or inventory_hostname in groups['dbserver_stg'] or inventory_hostname in groups['pkgs'] or inventory_hostname in groups['pagure'] or inventory_hostname in groups['pagure_stg'] or inventory_hostname in groups['zabbix_stg'] or inventory_hostname in groups['retrace'] %} From 9bbef456ed57e1d12ff4f096c160b998266fc4eb Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 13 May 2021 13:01:05 -0700 Subject: [PATCH 005/189] bodhi / backend: adjust bodhi-celery service for new version The new celery version wants -A as a global option, then command, so make it happy and re-arrange the service file. Signed-off-by: Kevin Fenzi --- roles/bodhi2/backend/files/bodhi-celery.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/bodhi2/backend/files/bodhi-celery.service b/roles/bodhi2/backend/files/bodhi-celery.service index e17933f2a9..21a3d246b8 100644 --- a/roles/bodhi2/backend/files/bodhi-celery.service +++ b/roles/bodhi2/backend/files/bodhi-celery.service @@ -6,7 +6,7 @@ Wants = network-online.target [Service] User = apache Group = apache -ExecStart = /usr/bin/celery-3 worker -A bodhi.server.tasks.app -l info -Q has_koji_mount +ExecStart = /usr/bin/celery-3 A bodhi.server.tasks.app worker -l info -Q has_koji_mount [Install] WantedBy = multi-user.target From fb395d74a2eb02988229f2177e61994b6b0d6745 Mon Sep 17 00:00:00 2001 From: Tomas Hrcka Date: Thu, 13 May 2021 17:35:21 +0200 Subject: [PATCH 006/189] Update sssd config to filter users bodhi and ftpsync Signed-off-by: Tomas Hrcka --- roles/ipa/client/files/fedora-nss-ignore.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/ipa/client/files/fedora-nss-ignore.conf b/roles/ipa/client/files/fedora-nss-ignore.conf index d1b5e0ce17..a4214b99a9 100644 --- a/roles/ipa/client/files/fedora-nss-ignore.conf +++ b/roles/ipa/client/files/fedora-nss-ignore.conf @@ -2,5 +2,5 @@ ## creating restricted accounts but we want to make sure the id in ## /etc/passwd and /etc/group are used. [nss] -filter_users = root,bin,daemon,adm,lp,sync,shutdown,halt,mail,operator,games,ftp,nobody,avahi-autoipd,dbus,polkitd,rpc,tss,ntp,rpcuser,nfsnobody,postfix,sshd,nagios,nrpe,openvpn,,chrony,sssd,named,mock,apache +filter_users = root,bin,daemon,adm,lp,sync,shutdown,halt,mail,operator,games,ftp,nobody,avahi-autoipd,dbus,polkitd,rpc,tss,ntp,rpcuser,nfsnobody,postfix,sshd,nagios,nrpe,openvpn,,chrony,sssd,named,mock,apache,bodhi,ftpsync filter_groups = root,bin,daemon,sys,adm,tty,disk,lp,mem,kmem,wheel,cdrom,mail,man,dialout,floppy,games,tape,video,ftp,lock,audio,nobody,users,utmp,utempter,avahi-autoipd,ssh_keys,systemd-journal,dbus,rpc,tss,ntp,dip,rpcuser,nfsnobody,postdrop,postfix,sshd,screen,nagios,nrpe,openvpn,input,systemd-bus-proxy,systemd-network,cgred,chrony,printadmin,sssd,named,mock,apache From d9d0b9d92691dcaea032c8f3394e8494ebdd2277 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 13 May 2021 15:28:23 -0700 Subject: [PATCH 007/189] bodhi-backend: disable updates-sync and auto pushes for now Disable these until we sort out the ostree repo issues in releng 10114. Please check with me before re-enabling. :) Signed-off-by: Kevin Fenzi --- roles/bodhi2/backend/files/new-updates-sync.cron | 4 ++-- roles/bodhi2/backend/templates/bodhi-automated-pushes.cron.j2 | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/roles/bodhi2/backend/files/new-updates-sync.cron b/roles/bodhi2/backend/files/new-updates-sync.cron index f9dd3818b0..bac38e8c5e 100644 --- a/roles/bodhi2/backend/files/new-updates-sync.cron +++ b/roles/bodhi2/backend/files/new-updates-sync.cron @@ -1,3 +1,3 @@ MAILTO=releng-cron@lists.fedoraproject.org -* * * * * ftpsync /usr/local/bin/lock-wrapper updates-push '/usr/local/bin/new-updates-sync' --silent - +# disabled by kevin until we sort out releng 10114 +#* * * * * ftpsync /usr/local/bin/lock-wrapper updates-push '/usr/local/bin/new-updates-sync' --silent diff --git a/roles/bodhi2/backend/templates/bodhi-automated-pushes.cron.j2 b/roles/bodhi2/backend/templates/bodhi-automated-pushes.cron.j2 index 4dc15ebd15..59255be002 100644 --- a/roles/bodhi2/backend/templates/bodhi-automated-pushes.cron.j2 +++ b/roles/bodhi2/backend/templates/bodhi-automated-pushes.cron.j2 @@ -2,5 +2,6 @@ MAILTO=sysadmin-releng-members@fedoraproject.org {% if RelEngFrozen %} {{ '#' }}14 00 * * * apache /usr/local/bin/bodhi-automated-pushes.py {% else %} -14 00 * * * apache /usr/local/bin/bodhi-automated-pushes.py +# disabled by kevin until we sort out releng 10114 +#14 00 * * * apache /usr/local/bin/bodhi-automated-pushes.py {% endif %} From 2465791d08ec00d8f68ebc8c3c632fa63c87ea2e Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Fri, 14 May 2021 08:48:23 -0700 Subject: [PATCH 008/189] Revert "bodhi-backend: disable updates-sync and auto pushes for now" This reverts commit d9d0b9d92691dcaea032c8f3394e8494ebdd2277. --- roles/bodhi2/backend/files/new-updates-sync.cron | 4 ++-- roles/bodhi2/backend/templates/bodhi-automated-pushes.cron.j2 | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/roles/bodhi2/backend/files/new-updates-sync.cron b/roles/bodhi2/backend/files/new-updates-sync.cron index bac38e8c5e..f9dd3818b0 100644 --- a/roles/bodhi2/backend/files/new-updates-sync.cron +++ b/roles/bodhi2/backend/files/new-updates-sync.cron @@ -1,3 +1,3 @@ MAILTO=releng-cron@lists.fedoraproject.org -# disabled by kevin until we sort out releng 10114 -#* * * * * ftpsync /usr/local/bin/lock-wrapper updates-push '/usr/local/bin/new-updates-sync' --silent +* * * * * ftpsync /usr/local/bin/lock-wrapper updates-push '/usr/local/bin/new-updates-sync' --silent + diff --git a/roles/bodhi2/backend/templates/bodhi-automated-pushes.cron.j2 b/roles/bodhi2/backend/templates/bodhi-automated-pushes.cron.j2 index 59255be002..4dc15ebd15 100644 --- a/roles/bodhi2/backend/templates/bodhi-automated-pushes.cron.j2 +++ b/roles/bodhi2/backend/templates/bodhi-automated-pushes.cron.j2 @@ -2,6 +2,5 @@ MAILTO=sysadmin-releng-members@fedoraproject.org {% if RelEngFrozen %} {{ '#' }}14 00 * * * apache /usr/local/bin/bodhi-automated-pushes.py {% else %} -# disabled by kevin until we sort out releng 10114 -#14 00 * * * apache /usr/local/bin/bodhi-automated-pushes.py +14 00 * * * apache /usr/local/bin/bodhi-automated-pushes.py {% endif %} From 671a46c299c526f292a8b7990bb273864d4f6f1f Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Fri, 14 May 2021 15:35:57 -0700 Subject: [PATCH 009/189] bodhi / backend: add missing dash in front of A Signed-off-by: Kevin Fenzi --- roles/bodhi2/backend/files/bodhi-celery.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/bodhi2/backend/files/bodhi-celery.service b/roles/bodhi2/backend/files/bodhi-celery.service index 21a3d246b8..497bb19160 100644 --- a/roles/bodhi2/backend/files/bodhi-celery.service +++ b/roles/bodhi2/backend/files/bodhi-celery.service @@ -6,7 +6,7 @@ Wants = network-online.target [Service] User = apache Group = apache -ExecStart = /usr/bin/celery-3 A bodhi.server.tasks.app worker -l info -Q has_koji_mount +ExecStart = /usr/bin/celery-3 -A bodhi.server.tasks.app worker -l info -Q has_koji_mount [Install] WantedBy = multi-user.target From 08448c6cd129c1115f4fbceb1416a9b78ef285a5 Mon Sep 17 00:00:00 2001 From: Brendan Early Date: Sat, 15 May 2021 09:05:32 -0500 Subject: [PATCH 010/189] packages-static: add project delete action to playbook --- playbooks/openshift-apps/fedora-packages-static.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/playbooks/openshift-apps/fedora-packages-static.yml b/playbooks/openshift-apps/fedora-packages-static.yml index 15923e177d..701782c689 100644 --- a/playbooks/openshift-apps/fedora-packages-static.yml +++ b/playbooks/openshift-apps/fedora-packages-static.yml @@ -56,3 +56,13 @@ app: fedora-packages-static template: deploymentconfig.yml objectname: deploymentconfig.yml + +############################################### +# actions to delete the project from OpenShift +############################################### +# to run: sudo rbac-playbook -l staging -t delete openshift-apps/fedora-packages-static.yml + - role: openshift/object-delete + app: fedora-packages-static + objecttype: project + objectname: fedora-packages-static + tags: [ never, delete ] \ No newline at end of file From 19d2e4c67c6393f48c55c865e8549bf6510aa10f Mon Sep 17 00:00:00 2001 From: Brendan Early Date: Sat, 15 May 2021 13:35:39 -0500 Subject: [PATCH 011/189] solr: use a route instead of joining pod networks --- .../openshift-apps/fedora-packages-static.yml | 2 +- playbooks/openshift-apps/solr.yml | 28 ++++++++++--------- .../templates/configmap.yml | 2 +- .../templates/deploymentconfig.yml | 6 ++-- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/playbooks/openshift-apps/fedora-packages-static.yml b/playbooks/openshift-apps/fedora-packages-static.yml index 701782c689..56b774a48a 100644 --- a/playbooks/openshift-apps/fedora-packages-static.yml +++ b/playbooks/openshift-apps/fedora-packages-static.yml @@ -65,4 +65,4 @@ app: fedora-packages-static objecttype: project objectname: fedora-packages-static - tags: [ never, delete ] \ No newline at end of file + tags: [ never, delete ] diff --git a/playbooks/openshift-apps/solr.yml b/playbooks/openshift-apps/solr.yml index f6a81c0b59..b371e88b5d 100644 --- a/playbooks/openshift-apps/solr.yml +++ b/playbooks/openshift-apps/solr.yml @@ -35,21 +35,23 @@ file: service.yml objectname: service.yml + - role: openshift/route + app: solr + routename: solr + host: "solr{{ env_suffix }}.fedoraproject.org" + serviceport: 8983-tcp + servicename: solr + annotations: + haproxy.router.openshift.io/ip_whitelist: 10.0.0.0/8 + - role: openshift/object app: solr template: deploymentconfig.yml objectname: deploymentconfig.yml -- name: Link solr and fedora-packages-static networks - hosts: os_masters_stg[0] - user: root - gather_facts: False - - vars_files: - - /srv/web/infra/ansible/vars/global.yml - - "/srv/private/ansible/vars.yml" - - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml - - tasks: - - name: Run oc adm command to link solr to other projects - command: "oc adm pod-network join-projects --to=solr fedora-packages-static" +# to run: sudo rbac-playbook -l staging -t delete openshift-apps/solr.yml + - role: openshift/object-delete + app: solr + objecttype: project + objectname: solr + tags: [ never, delete ] diff --git a/roles/openshift-apps/fedora-packages-static/templates/configmap.yml b/roles/openshift-apps/fedora-packages-static/templates/configmap.yml index 37044ebf68..55d358db6c 100644 --- a/roles/openshift-apps/fedora-packages-static/templates/configmap.yml +++ b/roles/openshift-apps/fedora-packages-static/templates/configmap.yml @@ -6,5 +6,5 @@ metadata: labels: app: fedora-packages-static data: - solr-url: "http://solr.solr.svc.cluster.local:8983/" + solr-url: "https://solr{{ env_suffix }}.fedoraproject.org:8983/" sitemap-url: "https://packages{{ env_suffix }}.fedoraproject.org" diff --git a/roles/openshift-apps/fedora-packages-static/templates/deploymentconfig.yml b/roles/openshift-apps/fedora-packages-static/templates/deploymentconfig.yml index a442976270..14f6706c54 100644 --- a/roles/openshift-apps/fedora-packages-static/templates/deploymentconfig.yml +++ b/roles/openshift-apps/fedora-packages-static/templates/deploymentconfig.yml @@ -22,7 +22,7 @@ spec: initContainers: - name: init-solr image: busybox - command: ['sh', '-c', 'until nslookup solr.solr.svc.cluster.local; do echo waiting for solr; sleep 2; done;'] + command: ['sh', '-c', 'until ping -c1 solr{{ env_suffix }}.fedoraproject.org >/dev/null 2>&1; do echo waiting for solr; sleep 2; done'] containers: - name: fedora-packages-static image: fedora-packages-static:latest @@ -30,12 +30,12 @@ spec: - name: SOLR_URL valueFrom: configMapKeyRef: - name: fedora-packages-static-configmap + name: fedora-packages-static-configmap key: solr-url - name: SITEMAP_URL valueFrom: configMapKeyRef: - name: fedora-packages-static-configmap + name: fedora-packages-static-configmap key: sitemap-url ports: - containerPort: 80 From 4fdda9ba302ddc8d3f3fad87238405126e07de90 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Sun, 16 May 2021 16:48:40 +0200 Subject: [PATCH 012/189] copr-be-dev: f34 builders --- inventory/group_vars/copr_back_dev_aws | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inventory/group_vars/copr_back_dev_aws b/inventory/group_vars/copr_back_dev_aws index 05945f8d25..229130fc5c 100644 --- a/inventory/group_vars/copr_back_dev_aws +++ b/inventory/group_vars/copr_back_dev_aws @@ -19,8 +19,8 @@ copr_builder_images: hypervisor: x86_64: copr-builder-20210511_210127 aws: - x86_64: ami-05655b44ed8d4f869 # copr-builder-x86_64-f33-20210119_150254 - aarch64: ami-0e26990bd41c19eba # copr-builder-aarch64-f33-20210119_145252 + x86_64: ami-05766cb7551777d09 # copr-builder-x86_64-f34-20210516_142224 + aarch64: ami-0b7d49fecf0c393a9 # copr-builder-aarch64-f34-20210516_143216 do_sign: "true" From 3ad3dcab97b9753f23112d4dcd47d70619375b29 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Sun, 16 May 2021 11:04:53 -0700 Subject: [PATCH 013/189] fasjson: add matrixadmin alias Signed-off-by: Kevin Fenzi --- roles/fasjson/files/aliases.static | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/fasjson/files/aliases.static b/roles/fasjson/files/aliases.static index 368d206772..dc2f9a9d7d 100644 --- a/roles/fasjson/files/aliases.static +++ b/roles/fasjson/files/aliases.static @@ -174,6 +174,9 @@ program-manager: fpgm diversity: fpl,fcaic,siddharthvipul1 diversity-inclusion: diversity +# Fedora Matrix admin +matrixadmin@fedoraproject.org: mattdm,riecatnor,kevin,nb + # FESCo fesco-chair: kevin fesco: fesco@lists.fedoraproject.org From 9587670a363b0c0becf8053a60dcb2cf3ae18159 Mon Sep 17 00:00:00 2001 From: Brendan Early Date: Sun, 16 May 2021 15:12:39 -0500 Subject: [PATCH 014/189] packages-static: bring solr and packages-static into the same os project packages-static: add solr gid to supplementalGroups packages-static: delete solr playbook packages-static: change old solr configmap names fedora-packages: move solr files to old folder No idea why Ansible thinks it should be here packages-static: move solr start --- .../openshift-apps/fedora-packages-static.yml | 7 ++ playbooks/openshift-apps/solr.yml | 57 ---------------- .../files/imagestream.yml | 11 +++ .../fedora-packages-static/files/service.yml | 16 +++++ .../templates/configmap.yml | 25 ++++++- .../templates/deploymentconfig.yml | 68 ++++++++++++++++++- .../templates/storage.yml | 14 +++- .../openshift-apps/solr/files/imagestream.yml | 11 --- roles/openshift-apps/solr/files/service.yml | 16 ----- .../solr/templates/configmap.yml | 23 ------- .../solr/templates/deploymentconfig.yml | 63 ----------------- .../openshift-apps/solr/templates/storage.yml | 12 ---- 12 files changed, 138 insertions(+), 185 deletions(-) delete mode 100644 playbooks/openshift-apps/solr.yml delete mode 100644 roles/openshift-apps/solr/files/imagestream.yml delete mode 100644 roles/openshift-apps/solr/files/service.yml delete mode 100644 roles/openshift-apps/solr/templates/configmap.yml delete mode 100644 roles/openshift-apps/solr/templates/deploymentconfig.yml delete mode 100644 roles/openshift-apps/solr/templates/storage.yml diff --git a/playbooks/openshift-apps/fedora-packages-static.yml b/playbooks/openshift-apps/fedora-packages-static.yml index 56b774a48a..227e5e2011 100644 --- a/playbooks/openshift-apps/fedora-packages-static.yml +++ b/playbooks/openshift-apps/fedora-packages-static.yml @@ -66,3 +66,10 @@ objecttype: project objectname: fedora-packages-static tags: [ never, delete ] + + # to run: sudo rbac-playbook -l staging -t deletesolr openshift-apps/fedora-packages-static.yml + - role: openshift/object-delete + app: solr + objecttype: project + objectname: solr + tags: [ never, deletesolr ] diff --git a/playbooks/openshift-apps/solr.yml b/playbooks/openshift-apps/solr.yml deleted file mode 100644 index b371e88b5d..0000000000 --- a/playbooks/openshift-apps/solr.yml +++ /dev/null @@ -1,57 +0,0 @@ -- name: Make solr - hosts: os_masters_stg[0] - user: root - gather_facts: False - - vars_files: - - /srv/web/infra/ansible/vars/global.yml - - "/srv/private/ansible/vars.yml" - - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml - - roles: - - role: openshift/project - app: solr - description: Fedora solr apps - appowners: - - mymindstorm - - - role: openshift/object - app: solr - file: imagestream.yml - objectname: imagestream.yml - - - role: openshift/object - app: solr - template: storage.yml - objectname: storage.yml - - - role: openshift/object - app: solr - template: configmap.yml - objectname: configmap.yml - - - role: openshift/object - app: solr - file: service.yml - objectname: service.yml - - - role: openshift/route - app: solr - routename: solr - host: "solr{{ env_suffix }}.fedoraproject.org" - serviceport: 8983-tcp - servicename: solr - annotations: - haproxy.router.openshift.io/ip_whitelist: 10.0.0.0/8 - - - role: openshift/object - app: solr - template: deploymentconfig.yml - objectname: deploymentconfig.yml - -# to run: sudo rbac-playbook -l staging -t delete openshift-apps/solr.yml - - role: openshift/object-delete - app: solr - objecttype: project - objectname: solr - tags: [ never, delete ] diff --git a/roles/openshift-apps/fedora-packages-static/files/imagestream.yml b/roles/openshift-apps/fedora-packages-static/files/imagestream.yml index b04fcd453e..828a8f3866 100644 --- a/roles/openshift-apps/fedora-packages-static/files/imagestream.yml +++ b/roles/openshift-apps/fedora-packages-static/files/imagestream.yml @@ -8,3 +8,14 @@ spec: local: false status: dockerImageRepository: "" +--- +apiVersion: v1 +kind: ImageStream +metadata: + name: solr +spec: + tags: + - name: 8 + from: + kind: DockerImage + name: solr:8 diff --git a/roles/openshift-apps/fedora-packages-static/files/service.yml b/roles/openshift-apps/fedora-packages-static/files/service.yml index ce64397474..a4dd6b1fb3 100644 --- a/roles/openshift-apps/fedora-packages-static/files/service.yml +++ b/roles/openshift-apps/fedora-packages-static/files/service.yml @@ -14,3 +14,19 @@ spec: selector: app: fedora-packages-static deploymentconfig: fedora-packages-static +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: fedora-packages-static + name: solr +spec: + ports: + - name: 8983-tcp + port: 8983 + protocol: TCP + targetPort: 8983 + selector: + app: fedora-packages-static + deploymentconfig: fedora-packages-static diff --git a/roles/openshift-apps/fedora-packages-static/templates/configmap.yml b/roles/openshift-apps/fedora-packages-static/templates/configmap.yml index 55d358db6c..0cd8c85dfc 100644 --- a/roles/openshift-apps/fedora-packages-static/templates/configmap.yml +++ b/roles/openshift-apps/fedora-packages-static/templates/configmap.yml @@ -1,3 +1,4 @@ +{% macro load_file(filename) %}{% include filename %}{%- endmacro -%} --- apiVersion: v1 kind: ConfigMap @@ -6,5 +7,27 @@ metadata: labels: app: fedora-packages-static data: - solr-url: "https://solr{{ env_suffix }}.fedoraproject.org:8983/" + solr-url: "http://solr.fedora-packages-static.svc.cluster.local:8983/" sitemap-url: "https://packages{{ env_suffix }}.fedoraproject.org" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: fedora-packages-static-solr-configmap + labels: + app: fedora-packages-static +data: + solrconfig.xml: |- + {{ load_file('packages/solrconfig.xml') | indent }} + schema.xml: |- + {{ load_file('packages/schema.xml') | indent }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: fedora-packages-static-solr-start-configmap + labels: + app: fedora-packages-static +data: + solr-start.sh: |- + {{ load_file('solr-start.sh') | indent }} diff --git a/roles/openshift-apps/fedora-packages-static/templates/deploymentconfig.yml b/roles/openshift-apps/fedora-packages-static/templates/deploymentconfig.yml index 14f6706c54..89a0723aeb 100644 --- a/roles/openshift-apps/fedora-packages-static/templates/deploymentconfig.yml +++ b/roles/openshift-apps/fedora-packages-static/templates/deploymentconfig.yml @@ -22,7 +22,7 @@ spec: initContainers: - name: init-solr image: busybox - command: ['sh', '-c', 'until ping -c1 solr{{ env_suffix }}.fedoraproject.org >/dev/null 2>&1; do echo waiting for solr; sleep 2; done'] + command: ['sh', '-c', 'until nslookup solr.fedora-packages-static.svc.cluster.local; do echo waiting for solr; sleep 2; done;'] containers: - name: fedora-packages-static image: fedora-packages-static:latest @@ -76,3 +76,69 @@ spec: from: kind: ImageStreamTag name: fedora-packages-static:latest +--- +apiVersion: v1 +kind: DeploymentConfig +metadata: + labels: + app: fedora-packages-static + service: solr + name: solr +spec: + replicas: 1 # Only 1! Solr is not configured in cloud mode yet. + selector: + app: fedora-packages-static + deploymentconfig: fedora-packages-static + strategy: + type: Rolling + template: + metadata: + labels: + app: fedora-packages-static + deploymentconfig: fedora-packages-static + spec: + containers: + - name: solr + image: solr:8 + command: + - bash + - /opt/solr-start/solr-start.sh + ports: + - containerPort: 8983 + resources: {} + volumeMounts: + - name: data-volume + mountPath: /var/solr + - name: config-volume + mountPath: /opt/solr/server/solr/configsets/packages + readOnly: true + - name: script-volume + mountPath: /opt/solr-start + readinessProbe: + timeoutSeconds: 1 + initialDelaySeconds: 30 + httpGet: + path: / + port: 8983 + livenessProbe: + timeoutSeconds: 1 + initialDelaySeconds: 40 + httpGet: + path: / + port: 8983 + # Add solr's default gid to nfs group + securityContext: + supplementalGroups: [8983] + volumes: + - name: data-volume + persistentVolumeClaim: + claimName: solr-storage{{ '-stg' if env == 'staging' else '' }} + - name: config-volume + configMap: + name: fedora-packages-static-solr-configmap + - name: script-volume + configMap: + name: fedora-packages-static-solr-start-configmap + + triggers: + - type: ConfigChange diff --git a/roles/openshift-apps/fedora-packages-static/templates/storage.yml b/roles/openshift-apps/fedora-packages-static/templates/storage.yml index a0c38f09be..c972424a09 100644 --- a/roles/openshift-apps/fedora-packages-static/templates/storage.yml +++ b/roles/openshift-apps/fedora-packages-static/templates/storage.yml @@ -21,4 +21,16 @@ spec: resources: requests: storage: 10Gi - storageClassName: "" \ No newline at end of file + storageClassName: "" +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: solr-storage{{ '-stg' if env == 'staging' else '' }} +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 10Gi + storageClassName: "" diff --git a/roles/openshift-apps/solr/files/imagestream.yml b/roles/openshift-apps/solr/files/imagestream.yml deleted file mode 100644 index 4513e1d0ab..0000000000 --- a/roles/openshift-apps/solr/files/imagestream.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: v1 -kind: ImageStream -metadata: - name: solr -spec: - tags: - - name: 8 - from: - kind: DockerImage - name: solr:8 \ No newline at end of file diff --git a/roles/openshift-apps/solr/files/service.yml b/roles/openshift-apps/solr/files/service.yml deleted file mode 100644 index 4dfda2080d..0000000000 --- a/roles/openshift-apps/solr/files/service.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: solr - name: solr -spec: - ports: - - name: 8983-tcp - port: 8983 - protocol: TCP - targetPort: 8983 - selector: - app: solr - deploymentconfig: solr \ No newline at end of file diff --git a/roles/openshift-apps/solr/templates/configmap.yml b/roles/openshift-apps/solr/templates/configmap.yml deleted file mode 100644 index 58d0b727f6..0000000000 --- a/roles/openshift-apps/solr/templates/configmap.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: solr-configmap - labels: - app: solr -{% macro load_file(filename) %}{% include filename %}{%- endmacro -%} -data: - solrconfig.xml: |- - {{ load_file('packages/solrconfig.xml') | indent }} - schema.xml: |- - {{ load_file('packages/schema.xml') | indent }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: solr-start-configmap - labels: - app: solr -data: - solr-start.sh: |- - {{ load_file('solr-start.sh') | indent }} diff --git a/roles/openshift-apps/solr/templates/deploymentconfig.yml b/roles/openshift-apps/solr/templates/deploymentconfig.yml deleted file mode 100644 index 57d3995775..0000000000 --- a/roles/openshift-apps/solr/templates/deploymentconfig.yml +++ /dev/null @@ -1,63 +0,0 @@ ---- -apiVersion: v1 -kind: DeploymentConfig -metadata: - labels: - app: solr - service: solr - name: solr -spec: - replicas: 1 # Only 1! Solr is not configured in cloud mode yet. - selector: - app: solr - deploymentconfig: solr - strategy: - type: Rolling - template: - metadata: - labels: - app: solr - deploymentconfig: solr - spec: - containers: - - name: solr - image: solr:8 - command: - - bash - - /opt/solr-start/solr-start.sh - ports: - - containerPort: 8983 - resources: {} - volumeMounts: - - name: data-volume - mountPath: /var/solr - - name: config-volume - mountPath: /opt/solr/server/solr/configsets/packages - readOnly: true - - name: script-volume - mountPath: /opt/solr-start - readinessProbe: - timeoutSeconds: 1 - initialDelaySeconds: 30 - httpGet: - path: / - port: 8983 - livenessProbe: - timeoutSeconds: 1 - initialDelaySeconds: 40 - httpGet: - path: / - port: 8983 - volumes: - - name: data-volume - persistentVolumeClaim: - claimName: solr-storage{{ '-stg' if env == 'staging' else '' }} - - name: config-volume - configMap: - name: solr-configmap - - name: script-volume - configMap: - name: solr-start-configmap - - triggers: - - type: ConfigChange diff --git a/roles/openshift-apps/solr/templates/storage.yml b/roles/openshift-apps/solr/templates/storage.yml deleted file mode 100644 index a641b48992..0000000000 --- a/roles/openshift-apps/solr/templates/storage.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: solr-storage{{ '-stg' if env == 'staging' else '' }} -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 10Gi - storageClassName: "" \ No newline at end of file From c50c0b1537374f56cebca00185995b4dee451787 Mon Sep 17 00:00:00 2001 From: Brendan Early Date: Sun, 16 May 2021 16:33:31 -0500 Subject: [PATCH 015/189] packages-static: quote ImageStream tag name --- .../openshift-apps/fedora-packages-static/files/imagestream.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/openshift-apps/fedora-packages-static/files/imagestream.yml b/roles/openshift-apps/fedora-packages-static/files/imagestream.yml index 828a8f3866..7ef1338716 100644 --- a/roles/openshift-apps/fedora-packages-static/files/imagestream.yml +++ b/roles/openshift-apps/fedora-packages-static/files/imagestream.yml @@ -15,7 +15,7 @@ metadata: name: solr spec: tags: - - name: 8 + - name: "8" from: kind: DockerImage name: solr:8 From 55250064d644ee32309ea92a43855f35af66e3bc Mon Sep 17 00:00:00 2001 From: Brendan Early Date: Sun, 16 May 2021 16:37:59 -0500 Subject: [PATCH 016/189] packages-static: move files back to new directory Zuul seems to be incorrect. --- .../templates/packages/schema.xml | 0 .../templates/packages/solrconfig.xml | 0 .../{solr => fedora-packages-static}/templates/solr-start.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename roles/openshift-apps/{solr => fedora-packages-static}/templates/packages/schema.xml (100%) rename roles/openshift-apps/{solr => fedora-packages-static}/templates/packages/solrconfig.xml (100%) rename roles/openshift-apps/{solr => fedora-packages-static}/templates/solr-start.sh (100%) diff --git a/roles/openshift-apps/solr/templates/packages/schema.xml b/roles/openshift-apps/fedora-packages-static/templates/packages/schema.xml similarity index 100% rename from roles/openshift-apps/solr/templates/packages/schema.xml rename to roles/openshift-apps/fedora-packages-static/templates/packages/schema.xml diff --git a/roles/openshift-apps/solr/templates/packages/solrconfig.xml b/roles/openshift-apps/fedora-packages-static/templates/packages/solrconfig.xml similarity index 100% rename from roles/openshift-apps/solr/templates/packages/solrconfig.xml rename to roles/openshift-apps/fedora-packages-static/templates/packages/solrconfig.xml diff --git a/roles/openshift-apps/solr/templates/solr-start.sh b/roles/openshift-apps/fedora-packages-static/templates/solr-start.sh similarity index 100% rename from roles/openshift-apps/solr/templates/solr-start.sh rename to roles/openshift-apps/fedora-packages-static/templates/solr-start.sh From 0b8c6907023d352fbba47cc8e724c0d5e125123f Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Mon, 17 May 2021 08:45:04 +0200 Subject: [PATCH 017/189] copr-be: resalloc VM prod/dev suffix, and hypervisor tags --- roles/copr/backend/templates/resalloc/pools.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/copr/backend/templates/resalloc/pools.yaml b/roles/copr/backend/templates/resalloc/pools.yaml index 36bf18f62f..4b202e258c 100644 --- a/roles/copr/backend/templates/resalloc/pools.yaml +++ b/roles/copr/backend/templates/resalloc/pools.yaml @@ -47,7 +47,7 @@ aws_aarch64_normal_{% if devel %}dev{% else %}prod{% endif %}: # x86_64 hypervisors {% if devel %} {% for hv in ["02", "03", "04"] %} -copr_hv_x86_64_{{ hv }}: +copr_hv_x86_64_{{ hv }}_{% if devel %}dev{% else %}prod{% endif %}: max: 3 max_starting: 2 max_prealloc: 2 @@ -65,6 +65,8 @@ copr_hv_x86_64_{{ hv }}: - arch_armhfp_emulated - arch_s390x - arch_s390x_emulated + - hypervisor + - hypervisor_{{ hv }} cmd_new: "/var/lib/resallocserver/provision/libvirt-new --swap-vol-size 168" cmd_delete: "/var/lib/resallocserver/provision/libvirt-new --swap-vol-size 168" cmd_delete: "/var/lib/resallocserver/provision/libvirt-delete" From 94e5f21bc18ec755ca8c96fe8090688ec3558899 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Mon, 17 May 2021 08:55:31 +0200 Subject: [PATCH 018/189] copr-be: tag AWS resalloc instances with "aws" --- roles/copr/backend/templates/resalloc/pools.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/copr/backend/templates/resalloc/pools.yaml b/roles/copr/backend/templates/resalloc/pools.yaml index 4b202e258c..d2f097a059 100644 --- a/roles/copr/backend/templates/resalloc/pools.yaml +++ b/roles/copr/backend/templates/resalloc/pools.yaml @@ -17,6 +17,7 @@ aws_x86_64_normal_{% if devel %}dev{% else %}prod{% endif %}: - arch_armhfp_emulated - arch_s390x - arch_s390x_emulated + - aws cmd_new: "/var/lib/resallocserver/resalloc_provision/vm-aws-new --arch=x86_64" cmd_delete: "/var/lib/resallocserver/resalloc_provision/vm-aws-delete" cmd_livecheck: "/var/lib/resallocserver/resalloc_provision/vm-aws-check" @@ -35,6 +36,7 @@ aws_aarch64_normal_{% if devel %}dev{% else %}prod{% endif %}: - copr_builder - arch_aarch64 - arch_aarch64_native + - aws cmd_new: "/var/lib/resallocserver/resalloc_provision/vm-aws-new --arch=aarch64" cmd_delete: "/var/lib/resallocserver/resalloc_provision/vm-aws-delete" cmd_livecheck: "/var/lib/resallocserver/resalloc_provision/vm-aws-check" From 7afba71b54368e79f76dd2792f56f79a257dab39 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Mon, 17 May 2021 09:56:05 +0200 Subject: [PATCH 019/189] copr-be-dev: lower the amount of AWS instances --- inventory/group_vars/copr_dev_aws | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/copr_dev_aws b/inventory/group_vars/copr_dev_aws index bc512328d7..156a7fdc4c 100644 --- a/inventory/group_vars/copr_dev_aws +++ b/inventory/group_vars/copr_dev_aws @@ -29,7 +29,7 @@ nm_controlled_resolv: True builders: # max|max_spawn|max_prealloc aws: - x86_64: [20,5,5] + x86_64: [10,1,1] armhfp: [3,1,1] aarch64: [5,2,2] From f47fe96f95ca776955f77a8ae828cb2bccb91efa Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Mon, 17 May 2021 09:59:36 +0200 Subject: [PATCH 020/189] copr-be-dev: lower the amount of preallocated vms on HV --- roles/copr/backend/templates/resalloc/pools.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/copr/backend/templates/resalloc/pools.yaml b/roles/copr/backend/templates/resalloc/pools.yaml index d2f097a059..9a5a29f145 100644 --- a/roles/copr/backend/templates/resalloc/pools.yaml +++ b/roles/copr/backend/templates/resalloc/pools.yaml @@ -51,8 +51,8 @@ aws_aarch64_normal_{% if devel %}dev{% else %}prod{% endif %}: {% for hv in ["02", "03", "04"] %} copr_hv_x86_64_{{ hv }}_{% if devel %}dev{% else %}prod{% endif %}: max: 3 - max_starting: 2 - max_prealloc: 2 + max_starting: 1 + max_prealloc: 1 tags: - copr_builder - arch_x86_64 From fa5db1e73d98c4b945ba9012488a87a200007f6a Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Mon, 17 May 2021 10:13:13 +0200 Subject: [PATCH 021/189] copr-be-dev: fix IPv6 pattern for devel stack --- roles/copr/backend/templates/provision/libvirt-new | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/roles/copr/backend/templates/provision/libvirt-new b/roles/copr/backend/templates/provision/libvirt-new index f17acd4a5e..91355d349c 100755 --- a/roles/copr/backend/templates/provision/libvirt-new +++ b/roles/copr/backend/templates/provision/libvirt-new @@ -391,8 +391,14 @@ def _main(): _arange_default("resalloc_pool_id", "RESALLOC_POOL_ID") _arange_default("resalloc_id_in_pool", "RESALLOC_ID_IN_POOL") + devel = True + if "prod" in args.name: + devel = False + + ip6_a, ip6_g = get_fedora_ipv6_address(args.resalloc_pool_id, - args.resalloc_id_in_pool) + args.resalloc_id_in_pool, + devel) spawner = LibvirtSpawner(args.resalloc_pool_id, log) spawner.vm_name = args.name From 90ebfe5620ec0637cf88eb003c098ebd4efca55f Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Mon, 17 May 2021 10:54:59 +0200 Subject: [PATCH 022/189] copr-be: promote tested builder images to production And turn on the builders on hypervisors. --- inventory/group_vars/copr_back_aws | 6 ++++-- roles/copr/backend/templates/resalloc/pools.yaml | 8 ++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/inventory/group_vars/copr_back_aws b/inventory/group_vars/copr_back_aws index 9e3151c9f0..d67b6a7d92 100644 --- a/inventory/group_vars/copr_back_aws +++ b/inventory/group_vars/copr_back_aws @@ -16,9 +16,11 @@ copr_builder_images: x86_64: copr-builder-x86_64-f32-20200914_072608 ppc64le: copr-builder-ppc64le-f31-20200117_132023 aarch64: copr-builder-aarch64-f32-20200914_073754 + hypervisor: + x86_64: copr-builder-20210511_210127 aws: - x86_64: ami-05655b44ed8d4f869 # copr-builder-x86_64-f33-20210119_150254 - aarch64: ami-0e26990bd41c19eba # copr-builder-aarch64-f33-20210119_145252 + x86_64: ami-05766cb7551777d09 # copr-builder-x86_64-f34-20210516_142224 + aarch64: ami-0b7d49fecf0c393a9 # copr-builder-aarch64-f34-20210516_143216 nrpe_procs_warn: 2200 nrpe_procs_crit: 2500 diff --git a/roles/copr/backend/templates/resalloc/pools.yaml b/roles/copr/backend/templates/resalloc/pools.yaml index 9a5a29f145..c3e7b0624b 100644 --- a/roles/copr/backend/templates/resalloc/pools.yaml +++ b/roles/copr/backend/templates/resalloc/pools.yaml @@ -47,12 +47,17 @@ aws_aarch64_normal_{% if devel %}dev{% else %}prod{% endif %}: {% endmacro %} # x86_64 hypervisors -{% if devel %} {% for hv in ["02", "03", "04"] %} copr_hv_x86_64_{{ hv }}_{% if devel %}dev{% else %}prod{% endif %}: +{% if devel %} max: 3 max_starting: 1 max_prealloc: 1 +{% else %} + max: 20 + max_starting: 4 + max_prealloc: 20 +{% endif %} tags: - copr_builder - arch_x86_64 @@ -78,7 +83,6 @@ copr_hv_x86_64_{{ hv }}_{% if devel %}dev{% else %}prod{% endif %}: reuse_max_count: 8 reuse_max_time: 1800 {% endfor %} -{% endif %} {% macro hw_aarch64(id, inst, max, max_starting, max_prealloc) %} aarch64_{{ id }}_{{ inst }}: From a9df677cf3bdde5190b948a9174844017aacc53d Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 17 May 2021 09:20:42 -0700 Subject: [PATCH 023/189] fasjson / aliases: aliases have to be local Signed-off-by: Kevin Fenzi --- roles/fasjson/files/aliases.static | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/fasjson/files/aliases.static b/roles/fasjson/files/aliases.static index dc2f9a9d7d..cee79962d3 100644 --- a/roles/fasjson/files/aliases.static +++ b/roles/fasjson/files/aliases.static @@ -175,7 +175,7 @@ diversity: fpl,fcaic,siddharthvipul1 diversity-inclusion: diversity # Fedora Matrix admin -matrixadmin@fedoraproject.org: mattdm,riecatnor,kevin,nb +matrixadmin: mattdm,riecatnor,kevin,nb # FESCo fesco-chair: kevin From ec210427ecc99558af94dd9e427dcb10d4a19c63 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 17 May 2021 09:31:27 -0700 Subject: [PATCH 024/189] batcave: allow sysadmin to read rbac-playbook config This should allow folks in the sysadmin group to read (but not write) the rbac-playbook config. This should allow folks to more easily tell who is granted rights to run some playbook. Signed-off-by: Kevin Fenzi --- roles/batcave/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/batcave/tasks/main.yml b/roles/batcave/tasks/main.yml index bdf78d20a1..68e3ddc8c0 100644 --- a/roles/batcave/tasks/main.yml +++ b/roles/batcave/tasks/main.yml @@ -430,7 +430,7 @@ # - name: install the ansible_utils/rbac config - copy: src={{ private }}/files/rbac/rbac.yaml dest=/etc/ansible_utils/rbac.yaml mode=0500 + copy: src={{ private }}/files/rbac/rbac.yaml dest=/etc/ansible_utils/rbac.yaml mode=0540 group=sysadmin tags: - rbac - batcave From 80079bec373975e18d7fde34ceaf7e6a6f161b95 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 5 May 2021 11:48:15 -0700 Subject: [PATCH 025/189] 503: adjust wording on the service not available doc Signed-off-by: Kevin Fenzi --- roles/haproxy/files/503.http | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/roles/haproxy/files/503.http b/roles/haproxy/files/503.http index 2a49026658..8e90b84e08 100644 --- a/roles/haproxy/files/503.http +++ b/roles/haproxy/files/503.http @@ -54,10 +54,8 @@ Content-Type: text/html; charset=UTF-8

Sorry! This service is currently unavailable.

The service that you are trying to access is currently unavailable. Please try refreshing this page in a couple of minutes. If you still see this message, then please follow the steps below:

    -
  1. Check on the status page if there are any known outages for our services.
    2. -
  2. Check the fedora-infrastructure pagure instance for an outage notification.
    3. -
  3. Ask around in #fedora-admin on irc.freenode.net.
    4. -
  4. If it is accessible, check the Outage SOP for more information.
    5. +
  5. Check on the status page if there are any known outages for our services.
    6. +
  6. If it is accessible, please check the how to work with our team document for more information.
From 368bfaef0119c89b383705de9e354343ceb1ec55 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 17 May 2021 09:43:45 -0700 Subject: [PATCH 026/189] haproxy: adjust content length for new text in 503 message Signed-off-by: Kevin Fenzi --- roles/haproxy/files/503.http | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/haproxy/files/503.http b/roles/haproxy/files/503.http index 8e90b84e08..ffb0f1e9a9 100644 --- a/roles/haproxy/files/503.http +++ b/roles/haproxy/files/503.http @@ -1,5 +1,5 @@ HTTP/1.1 503 Service Temporarily Unavailable -Content-Length: 3552 +Content-Length: 3349 Connection: close Cache-Control: no-cache Pragma: no-cache From 3a266118418a9a94c36aafe516b54f584fec187a Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 17 May 2021 09:55:58 -0700 Subject: [PATCH 027/189] haproxy: add ipa03 into the mix as a backup behind ipa02 Signed-off-by: Kevin Fenzi --- roles/haproxy/templates/haproxy.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/haproxy/templates/haproxy.cfg b/roles/haproxy/templates/haproxy.cfg index 4f7882868d..c86d0bdfe1 100644 --- a/roles/haproxy/templates/haproxy.cfg +++ b/roles/haproxy/templates/haproxy.cfg @@ -285,6 +285,7 @@ backend ipa-backend server ipa01 ipa01:443 check inter 10s rise 1 fall 2 ssl verify required ca-file /etc/haproxy/ipa.pem {% if env != "staging" %} server ipa02 ipa02:443 check inter 10s rise 1 fall 2 ssl verify required ca-file /etc/haproxy/ipa.pem backup + server ipa03 ipa03:443 check inter 10s rise 1 fall 2 ssl verify required ca-file /etc/haproxy/ipa.pem backup {% endif %} option httpchk GET /ipa/ui/ @@ -326,6 +327,7 @@ backend krb5-backend server ipa01 ipa01:88 weight 1 maxconn 16384 {% if env == "production" %} server ipa02 ipa02:88 weight 1 maxconn 16384 + server ipa03 ipa03:88 weight 1 maxconn 16384 {% endif %} frontend oci-candidate-registry-frontend From 421fbeff1239ebb547cf0094488bf6d962616325 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 17 May 2021 10:08:53 -0700 Subject: [PATCH 028/189] koji_hub: add fwupd-efi to secure boot channel. See https://pagure.io/fedora-infrastructure/issue/9912 Signed-off-by: Kevin Fenzi --- roles/koji_hub/templates/hub.conf.j2 | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/roles/koji_hub/templates/hub.conf.j2 b/roles/koji_hub/templates/hub.conf.j2 index eeb37e2c7c..f1612e280c 100644 --- a/roles/koji_hub/templates/hub.conf.j2 +++ b/roles/koji_hub/templates/hub.conf.j2 @@ -93,12 +93,12 @@ Plugins = fedmsg-koji-plugin runroot_hub hub_containerbuild tag2distrepo sidetag [policy] tag = - user mbs/mbs.fedoraproject.org && tag module-* && package kernel shim grub2 pesign fwupd :: allow - user mbs/mbs.fedoraproject.org && fromtag module-* && package kernel shim grub2 pesign fwupd :: allow - user bodhi && tag *-override && package kernel shim grub2 pesign fwupd :: allow - has_perm autosign && fromtag *-pending && package kernel shim grub2 pesign fwupd :: allow - has_perm autosign && fromtag *-candidate && package kernel shim grub2 pesign fwupd :: allow - has_perm secure-boot && package kernel shim grub2 pesign fwupd :: allow + user mbs/mbs.fedoraproject.org && tag module-* && package kernel shim grub2 pesign fwupd fwupd-efi :: allow + user mbs/mbs.fedoraproject.org && fromtag module-* && package kernel shim grub2 pesign fwupd fwupd-efi:: allow + user bodhi && tag *-override && package kernel shim grub2 pesign fwupd fwupd-efi :: allow + has_perm autosign && fromtag *-pending && package kernel shim grub2 pesign fwupd fwupd-efi :: allow + has_perm autosign && fromtag *-candidate && package kernel shim grub2 pesign fwupd fwupd-efi :: allow + has_perm secure-boot && package kernel shim grub2 pesign fwupd fwupd-efi :: allow # CoreOS continuous builds, https://pagure.io/releng/issue/8165 operation tag && tag f*-coreos-continuous && has_perm coreos-continuous :: allow operation untag && fromtag f*-coreos-continuous && has_perm coreos-continuous :: allow @@ -110,7 +110,7 @@ tag = operation tag && tag eln* && has_perm eln :: allow operation untag && fromtag eln* && has_perm eln :: allow # deny tagging secureboot packages that are not related to coreos-continuous and eln - package kernel shim grub2 pesign fwupd :: deny + package kernel shim grub2 pesign fwupd fwupd-efi :: deny # Allow people to tag stuff into infra-candidate if they're infra tag *-infra-candidate && has_perm infra :: allow tag *-infra-candidate :: deny @@ -138,6 +138,7 @@ channel = source */pesign* && has_perm secure-boot :: use secure-boot source */fwupdate* && has_perm secure-boot :: use secure-boot source */fwupd* && has_perm secure-boot :: use secure-boot + source */fwupd-efi* && has_perm secure-boot :: use secure-boot # set this package to use the 'heavybuilder' channel. Note that this is NOT good for most anything. # It just happens to be for this particular package. Please check before adding anything here, you could From 1bbfd7f02f80f35b429ba458bbe8b9833f327bf0 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 17 May 2021 10:17:43 -0700 Subject: [PATCH 029/189] koji_hub: increase max_delete_processes from 4 to 20 This is the threads that are doing deletes of repos. Since we have so many repos, lets increase the delete threads as well so it can hopefully keep up. Signed-off-by: Kevin Fenzi --- roles/koji_hub/templates/kojira.conf.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/koji_hub/templates/kojira.conf.j2 b/roles/koji_hub/templates/kojira.conf.j2 index e883860824..3a1f59c5eb 100644 --- a/roles/koji_hub/templates/kojira.conf.j2 +++ b/roles/koji_hub/templates/kojira.conf.j2 @@ -27,6 +27,7 @@ separate_source_tags=f{{ FedoraRawhideNumber }}-build eln-build ; prevent repo cleanup from stalling repo regen ;prune_batch_size=1 ;delete_batch_size=1 +max_delete_processes=20 ; prevent kojira from flooding the build system with newRepo tasks max_repo_tasks=30 From 1ee14dc2f454e6cfce7c87c90be348ebd57c55c1 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 17 May 2021 10:37:25 -0700 Subject: [PATCH 030/189] batcave: install postgresql12 psql for client querys to db-datanommer See ticket https://pagure.io/fedora-infrastructure/issue/9913 Signed-off-by: Kevin Fenzi --- roles/batcave/tasks/main.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/roles/batcave/tasks/main.yml b/roles/batcave/tasks/main.yml index 68e3ddc8c0..1b7be9d600 100644 --- a/roles/batcave/tasks/main.yml +++ b/roles/batcave/tasks/main.yml @@ -623,3 +623,24 @@ tags: - batcave - tmux + +- name: enable the postgresql 12 module for psql on batcave + copy: + dest: /etc/dnf/modules.d/postgresql.module + content: | + [postgresql] + name=postgresql + stream=12 + profiles= + state=enabled + tags: + - batcave + - postgres + +- name: install psql client + package: name=postgresql state=present + tags: + - batcave + - postgres + - config + - packages From b3d2dc7d90e6c5c83f5240f16d7d7692a21b7719 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 17 May 2021 10:42:18 -0700 Subject: [PATCH 031/189] batcave: clean up old tasks Signed-off-by: Kevin Fenzi --- roles/batcave/tasks/main.yml | 53 +++--------------------------------- 1 file changed, 4 insertions(+), 49 deletions(-) diff --git a/roles/batcave/tasks/main.yml b/roles/batcave/tasks/main.yml index 1b7be9d600..7547ef3fd0 100644 --- a/roles/batcave/tasks/main.yml +++ b/roles/batcave/tasks/main.yml @@ -21,44 +21,6 @@ - packages - yumrepos -- name: install packages needed (rhel7) - package: name={{ item }} state=present - with_items: - - srm # secure rm to delete sensitive files. - - ansible # This is our ansible master, needs ansible installed. - - ansible-openstack-modules # Needed to manage cloud with ansible - - ansible-collection-community-general # various useful community modules for ansible - - yum-metadata-parser # Needed for rhn sync - - yum-rhn-plugin # Needed for rhn sync - - createrepo_c # Needed for rhn sync - - ostree # Needed for rhn sync - - python-sqlalchemy # Needed for repo2json - - pyliblzma # Needed for repo2json - - ansible_utils # Needed for rbac-playbook - - python-GeoIP # Needed for geoip scripts - - python-fedmsg-fasclient # fasClient fedmsg job - - bind # named-checkzone for dns repo - - emacs-nox - - rpm-sign # for the sign-and-import playbook - - createrepo # for the sign-and-import playbook - - unzip # general useful util - - fpaste # general useful util - - mtr # useful for network debugging - - lftp # needed to easily pull in builds from koji for internal repos - - git-email # needed to send patches for review to the mailing list - - python-dns # needed to have ansible remove ip-based known_host entries - - libvirt-client # needed to allow migrations to be run from here. - - ansible-lint # needed to check ansible playbooks for issues. - - atomic-openshift-clients # For convenient client access to os.fp.o - - easy-rsa # For easy copying into ansible-private for certs. - - dnf # To get dnf reposync - - dnf-plugins-core # To get dnf reposync - - fedora-messaging # To send/receive messages on the amqp bus - tags: - - batcave - - config - when: ansible_distribution_major_version|int == 7 - - name: install packages needed (rhel8) package: name={{ item }} state=present with_items: @@ -88,7 +50,6 @@ tags: - batcave - config - when: ansible_distribution_major_version|int == 8 - name: setup ssh_known_hosts file copy: src=ssh_known_hosts dest=/etc/ssh/ssh_known_hosts mode=0644 @@ -455,16 +416,6 @@ - config when: inventory_hostname.startswith('batcave01.phx2') -# -# fasClient fedmsg job -# - -#- name: Install config for fedmsg tigger for fasClient runs -# copy: src=fedmsg-fasclient.py dest=/etc/fedmsg.d/fasclient.py mode=0644 -# tags: -# - batcave -# - config - # # set selinux context for /srv/web/infra # @@ -624,6 +575,10 @@ - batcave - tmux +# +# install psql to allow for some ro queries against db-datanommer01 +# + - name: enable the postgresql 12 module for psql on batcave copy: dest: /etc/dnf/modules.d/postgresql.module From 6ac5a89b14f43125bec0df83a41adab4645f9079 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 17 May 2021 12:30:42 -0700 Subject: [PATCH 032/189] playbooks / fedmsg: drop fedmsg/base from pkgs and releng-compose Signed-off-by: Kevin Fenzi --- playbooks/groups/pkgs.yml | 18 ------------------ playbooks/groups/releng-compose.yml | 2 -- 2 files changed, 20 deletions(-) diff --git a/playbooks/groups/pkgs.yml b/playbooks/groups/pkgs.yml index d49b4145fe..47eb586eb4 100644 --- a/playbooks/groups/pkgs.yml +++ b/playbooks/groups/pkgs.yml @@ -85,21 +85,3 @@ handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml" - -#- name: setup fedmsg on pkgs -# hosts: pkgs01.iad2.fedoraproject.org -# user: root -# gather_facts: True -# -# vars_files: -# - /srv/web/infra/ansible/vars/global.yml -# - "/srv/private/ansible/vars.yml" -# - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml -# -# roles: -# - { role: collectd/fedmsg-service, process: fedmsg-hub } -# - fedmsg/base -# - fedmsg/hub -# -# handlers: -# - import_tasks: "{{ handlers_path }}/restart_services.yml" diff --git a/playbooks/groups/releng-compose.yml b/playbooks/groups/releng-compose.yml index 9b7a070a0e..660f2fdc61 100644 --- a/playbooks/groups/releng-compose.yml +++ b/playbooks/groups/releng-compose.yml @@ -117,8 +117,6 @@ nfs_src_dir: "{{ koji_hub_nfs }}" when: "'releng_compose_stg' in group_names" - - fedmsg/base - - role: rabbit/user username: "pungi{{ env_suffix }}" From f23fd1b7a1dcab2accaafea61131600e8808fc9a Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 17 May 2021 13:00:56 -0700 Subject: [PATCH 033/189] totpcgi / 2fa: remove old totpci and files and roles. Note: there are still some calls here in old fas in openshift, but we will remove those when we remove old fas (likely as soon as zodbot is ported over to noggin). Signed-off-by: Kevin Fenzi --- files/2fa/pam_url.conf.cloud | 21 -- files/2fa/pam_url.conf.j2 | 27 -- files/2fa/pam_url.conf.stg.fedoraproject.org | 21 -- files/2fa/pam_url.conf.vpn.fedoraproject.org | 21 -- files/2fa/sudo.pam | 9 - files/2fa/sudo.pam.dev.fedoraproject.org | 6 - roles/haproxy/templates/haproxy.cfg | 32 --- roles/totpcgi/files/html/error.html | 26 -- roles/totpcgi/files/html/login.html | 49 ---- roles/totpcgi/files/html/totp.html | 53 ---- roles/totpcgi/files/index.cgi | 200 ------------- roles/totpcgi/files/pam_url.conf | 21 -- roles/totpcgi/files/pam_url.conf.fakefas01 | 21 -- roles/totpcgi/files/pam_url.conf.stg | 21 -- roles/totpcgi/files/pam_url.conf.vpn | 21 -- roles/totpcgi/files/provisioning.cgi | 265 ------------------ roles/totpcgi/files/sudo.pam | 11 - .../files/sudo.pam.dev.fedoraproject.org | 6 - .../files/sudo.pam.qa.fedoraproject.org | 6 - roles/totpcgi/tasks/main.yml | 236 ---------------- roles/totpcgi/templates/configmap.yml | 138 --------- .../templates/provisioning-httpd.conf.j2 | 44 --- roles/totpcgi/templates/provisioning.conf.j2 | 88 ------ roles/totpcgi/templates/totpcgi-httpd.conf.j2 | 37 --- .../templates/totpcgi-httpd.conf.stg.j2 | 37 --- .../templates/totpcgi-httpd.conf.vpn.j2 | 36 --- roles/totpcgi/templates/totpcgi.conf.j2 | 31 -- tasks/2fa_client.yml | 40 --- 28 files changed, 1524 deletions(-) delete mode 100644 files/2fa/pam_url.conf.cloud delete mode 100644 files/2fa/pam_url.conf.j2 delete mode 100644 files/2fa/pam_url.conf.stg.fedoraproject.org delete mode 100644 files/2fa/pam_url.conf.vpn.fedoraproject.org delete mode 100644 files/2fa/sudo.pam delete mode 100644 files/2fa/sudo.pam.dev.fedoraproject.org delete mode 100644 roles/totpcgi/files/html/error.html delete mode 100644 roles/totpcgi/files/html/login.html delete mode 100644 roles/totpcgi/files/html/totp.html delete mode 100755 roles/totpcgi/files/index.cgi delete mode 100644 roles/totpcgi/files/pam_url.conf delete mode 100644 roles/totpcgi/files/pam_url.conf.fakefas01 delete mode 100644 roles/totpcgi/files/pam_url.conf.stg delete mode 100644 roles/totpcgi/files/pam_url.conf.vpn delete mode 100644 roles/totpcgi/files/provisioning.cgi delete mode 100644 roles/totpcgi/files/sudo.pam delete mode 100644 roles/totpcgi/files/sudo.pam.dev.fedoraproject.org delete mode 100644 roles/totpcgi/files/sudo.pam.qa.fedoraproject.org delete mode 100644 roles/totpcgi/tasks/main.yml delete mode 100644 roles/totpcgi/templates/configmap.yml delete mode 100644 roles/totpcgi/templates/provisioning-httpd.conf.j2 delete mode 100644 roles/totpcgi/templates/provisioning.conf.j2 delete mode 100644 roles/totpcgi/templates/totpcgi-httpd.conf.j2 delete mode 100644 roles/totpcgi/templates/totpcgi-httpd.conf.stg.j2 delete mode 100644 roles/totpcgi/templates/totpcgi-httpd.conf.vpn.j2 delete mode 100644 roles/totpcgi/templates/totpcgi.conf.j2 delete mode 100644 tasks/2fa_client.yml diff --git a/files/2fa/pam_url.conf.cloud b/files/2fa/pam_url.conf.cloud deleted file mode 100644 index 9ce7690b81..0000000000 --- a/files/2fa/pam_url.conf.cloud +++ /dev/null @@ -1,21 +0,0 @@ -pam_url: -{ - settings: - { - url = "https://fas-all.phx2.fedoraproject.org:8443/"; # URI to fetch - returncode = "OK"; # The remote script/cgi should return a 200 http code and this string as its only results - userfield = "user"; # userfield name to send - passwdfield = "token"; # passwdfield name to send - extradata = "&do=login"; # extradata to send - prompt = "Password+Token: "; # password prompt - }; - - ssl: - { - verify_peer = true; # Should we verify SSL ? - verify_host = true; # Should we verify the CN in the SSL cert? - client_cert = "/etc/pki/tls/private/totpcgi.pem"; # file to use as client-side certificate - client_key = "/etc/pki/tls/private/totpcgi.pem"; # file to use as client-side key (can be same file as above if a single cert) - ca_cert = "/etc/pki/tls/private/totpcgi-ca.cert"; - }; -}; diff --git a/files/2fa/pam_url.conf.j2 b/files/2fa/pam_url.conf.j2 deleted file mode 100644 index 53de133ec2..0000000000 --- a/files/2fa/pam_url.conf.j2 +++ /dev/null @@ -1,27 +0,0 @@ -pam_url: -{ - settings: - { - {% if env == 'staging' %} - url = "https://fas-all.stg.phx2.fedoraproject.org:8443/"; # URI to fetch - {% elif datacenter == 'iad2' %} - url = "https://fas-all.iad2.fedoraproject.org:8443/"; # URI to fetch - {% else %} - url = "https://fas-all.vpn.fedoraproject.org:8443/"; # URI to fetch - {% endif %} - returncode = "OK"; # The remote script/cgi should return a 200 http code and this string as its only results - userfield = "user"; # userfield name to send - passwdfield = "token"; # passwdfield name to send - extradata = "&do=login"; # extradata to send - prompt = "Password+Token: "; # password prompt - }; - - ssl: - { - verify_peer = true; # Should we verify SSL ? - verify_host = true; # Should we verify the CN in the SSL cert? - client_cert = "/etc/pki/tls/private/totpcgi.pem"; # file to use as client-side certificate - client_key = "/etc/pki/tls/private/totpcgi.pem"; # file to use as client-side key (can be same file as above if a single cert) - ca_cert = "/etc/pki/tls/private/totpcgi-ca.cert"; - }; -}; diff --git a/files/2fa/pam_url.conf.stg.fedoraproject.org b/files/2fa/pam_url.conf.stg.fedoraproject.org deleted file mode 100644 index 508c5d9cf7..0000000000 --- a/files/2fa/pam_url.conf.stg.fedoraproject.org +++ /dev/null @@ -1,21 +0,0 @@ -pam_url: -{ - settings: - { - url = "https://fas-all.stg.phx2.fedoraproject.org:8443/"; # URI to fetch - returncode = "OK"; # The remote script/cgi should return a 200 http code and this string as its only results - userfield = "user"; # userfield name to send - passwdfield = "token"; # passwdfield name to send - extradata = "&do=login"; # extradata to send - prompt = "Password+Token: "; # password prompt - }; - - ssl: - { - verify_peer = true; # Should we verify SSL ? - verify_host = true; # Should we verify the CN in the SSL cert? - client_cert = "/etc/pki/tls/private/totpcgi.pem"; # file to use as client-side certificate - client_key = "/etc/pki/tls/private/totpcgi.pem"; # file to use as client-side key (can be same file as above if a single cert) - ca_cert = "/etc/pki/tls/private/totpcgi-ca.cert"; - }; -}; diff --git a/files/2fa/pam_url.conf.vpn.fedoraproject.org b/files/2fa/pam_url.conf.vpn.fedoraproject.org deleted file mode 100644 index 6e102e12f7..0000000000 --- a/files/2fa/pam_url.conf.vpn.fedoraproject.org +++ /dev/null @@ -1,21 +0,0 @@ -pam_url: -{ - settings: - { - url = "https://fas-all.vpn.fedoraproject.org:8443/"; # URI to fetch - returncode = "OK"; # The remote script/cgi should return a 200 http code and this string as its only results - userfield = "user"; # userfield name to send - passwdfield = "token"; # passwdfield name to send - extradata = "&do=login"; # extradata to send - prompt = "Password+Token: "; # password prompt - }; - - ssl: - { - verify_peer = true; # Should we verify SSL ? - verify_host = true; # Should we verify the CN in the SSL cert? - client_cert = "/etc/pki/tls/private/totpcgi.pem"; # file to use as client-side certificate - client_key = "/etc/pki/tls/private/totpcgi.pem"; # file to use as client-side key (can be same file as above if a single cert) - ca_cert = "/etc/pki/tls/private/totpcgi-ca.cert"; - }; -}; diff --git a/files/2fa/sudo.pam b/files/2fa/sudo.pam deleted file mode 100644 index 9912cdef59..0000000000 --- a/files/2fa/sudo.pam +++ /dev/null @@ -1,9 +0,0 @@ -#%PAM-1.0 -auth required pam_env.so -auth sufficient pam_url.so config=/etc/pam_url.conf -auth requisite pam_succeed_if.so uid >= 500 quiet -auth required pam_deny.so -account include system-auth -password include system-auth -session optional pam_keyinit.so revoke -session required pam_limits.so diff --git a/files/2fa/sudo.pam.dev.fedoraproject.org b/files/2fa/sudo.pam.dev.fedoraproject.org deleted file mode 100644 index 030bb26463..0000000000 --- a/files/2fa/sudo.pam.dev.fedoraproject.org +++ /dev/null @@ -1,6 +0,0 @@ -#%PAM-1.0 -auth include system-auth -account include system-auth -password include system-auth -session optional pam_keyinit.so revoke -session required pam_limits.so diff --git a/roles/haproxy/templates/haproxy.cfg b/roles/haproxy/templates/haproxy.cfg index c86d0bdfe1..a59e0f9c03 100644 --- a/roles/haproxy/templates/haproxy.cfg +++ b/roles/haproxy/templates/haproxy.cfg @@ -94,16 +94,6 @@ backend freemedia-backend #{% endif %} # option httpchk GET /packages/_heartbeat -frontend totpcgiprovision-frontend - bind 0.0.0.0:10019 - default_backend totpcgiprovision-backend - -backend totpcgiprovision-backend - balance hdr(appserver) - http-check expect status 401 - server fas01 fas01:8444 check inter 5s rise 1 fall 2 - option httpchk GET /index.cgi - frontend blockerbugs-frontend bind 0.0.0.0:10022 default_backend blockerbugs-backend @@ -289,28 +279,6 @@ backend ipa-backend {% endif %} option httpchk GET /ipa/ui/ -# This is for TOTPCGI (legacy 2fa). It goes to the Openshift routers, which then passthrough TLS to the totpcgi pods -frontend totp-frontend - mode tcp - bind 0.0.0.0:8443 - default_backend totp-backend - -backend totp-backend - mode tcp - option tcplog - balance roundrobin - maxconn 16384 - timeout queue 5000 - timeout server 86400000 - timeout connect 86400000 - server os-node01 os-node01:443 weight 1 maxconn 16384 - server os-node02 os-node02:443 weight 1 maxconn 16384 - server os-node03 os-node03:443 weight 1 maxconn 16384 - server os-node04 os-node04:443 weight 1 maxconn 16384 -{% if env == "production" %} - server os-node05 os-node05:443 weight 1 maxconn 16384 -{% endif %} - frontend krb5-frontend mode tcp bind 0.0.0.0:1088 diff --git a/roles/totpcgi/files/html/error.html b/roles/totpcgi/files/html/error.html deleted file mode 100644 index 7229ce49d8..0000000000 --- a/roles/totpcgi/files/html/error.html +++ /dev/null @@ -1,26 +0,0 @@ - - - - - Fedora Project Google Authenticator provisioning - Error - - - -
-
Fedora Project - Error
-
-

- $errormsg -

-

- You can try again or you can contact - the Fedora Infrastructure team at admin@fedoraproject.org. -

-
- -
- - diff --git a/roles/totpcgi/files/html/login.html b/roles/totpcgi/files/html/login.html deleted file mode 100644 index 024a8ac84d..0000000000 --- a/roles/totpcgi/files/html/login.html +++ /dev/null @@ -1,49 +0,0 @@ - - - - - Fedora Project Google Authenticator provisioning - - - - - - diff --git a/roles/totpcgi/files/html/totp.html b/roles/totpcgi/files/html/totp.html deleted file mode 100644 index 9e968e2ebd..0000000000 --- a/roles/totpcgi/files/html/totp.html +++ /dev/null @@ -1,53 +0,0 @@ - - - - - Fedora Project Google Authenticator Provisioning - - - - -
-
Fedora Project Google Authenticator Provisioning
-
-
- $qrcode_embed -
-
-

- Your new Google Authenticator token has been issued. - To import this token into your device, simply go to your - Google Authenticator app, select the option to add an - account, and then select "Scan Barcode". Point the camera - at the QR Barcode displayed next to this message. Google - Authenticator will then import your new token into the - device. It should be ready to use immediately. -

-
-
-

- If the administrator permitted the use of scratch tokens, - you should see them listed below. If you lose access to - your Google Authenticator device, you should be able to - use one of these tokens to gain emergency access to your - account. Please write them down. -

-
- $scratch_tokens -
-
-
-

- If you require any help with your Google Authenticator - token or experience any difficulty importing it into - your mobile device, please email - admin@fedoraproject.org. -

-
- -
- - diff --git a/roles/totpcgi/files/index.cgi b/roles/totpcgi/files/index.cgi deleted file mode 100755 index de3d066eac..0000000000 --- a/roles/totpcgi/files/index.cgi +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/python -ttW ignore::DeprecationWarning -## -# Copyright (C) 2012 by Konstantin Ryabitsev and contributors -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA -# 02111-1307, USA. -# -import os -import re -import sys -import cgi -import logging -logging.basicConfig(level=logging.INFO) -import urllib2 - -import totpcgi -import totpcgi.backends - -if len(sys.argv) > 1: - # blindly assume it's the config file - config_file = sys.argv[1] -else: - config_file = '/etc/totpcgi/totpcgi.conf' - -import ConfigParser - -from fedora.client import AuthError -from fedora.client.fasproxy import FasProxyClient - -config = ConfigParser.RawConfigParser() -config.read(config_file) - -require_pincode = config.getboolean('main', 'require_pincode') -success_string = config.get('main', 'success_string') - -fas_url = config.get('main', 'fas_url') -try: - fas = FasProxyClient(fas_url) -except Exception, e: - logging.exception("Problem connecting to FAS") - sys.exit(1) - -backends = totpcgi.backends.Backends() - -try: - backends.load_from_config(config) -except totpcgi.backends.BackendNotSupported, ex: - logging.exception("Backend engine not supported") - sys.exit(1) - -### Begin custom Fedora Functions - -def google_auth_fas_pincode_verify(user, pincode): - if not fas.verify_password(user, pincode): - raise totpcgi.UserPincodeError('User Password Error') - -backends.pincode_backend.verify_user_pincode = google_auth_fas_pincode_verify - -client_id = '1' - -def parse_token(token): - if token > 44: - otp = token[-44:] - if otp.startswith('ccccc'): - return token[:-44], otp - - # Not a password + yubikey - return False - -class YubikeyAuthenticator(object): - auth_regex = re.compile('^status=(?P\w{2})') - def __init__(self, require_pincode=False): - self.require_pincode = require_pincode - - def verify_user_token(self, user, token): - # Parse the token apart into a password and token - password, otp = parse_token(token) - - # Verify token against yubikey server - server_prefix = 'http://yubikey:8080/yk-val/verify?id=' - server_url = server_prefix + client_id + "&otp=" + otp - - fh = urllib2.urlopen(server_url) - - for line in fh: - match = self.auth_regex.search(line.strip('\n')) - if match: - if match.group('rc') == 'OK': - # Yubikey token is valid - break - raise totpcgi.VerifyFailed(line.split('=')[1]) - else: - raise totpcgi.VerifyFailed('yk-val returned malformed response') - - - # Verify that the yubikey token belongs to the user - # As a side effect, verify the password is good as well - # if the user+password are wrong, this will raise a fedora.client.AuthError - try: - response = fas.send_request('/config/list/%s/yubikey' % user, - auth_params={'username': user, 'password': password}) - except AuthError, e: - raise totpcgi.VerifyFailed('User Password Error: %s' % e) - if not response[1].configs.prefix or not response[1].configs.enabled: - raise totpcgi.VerifyFailed('Yubikey OTP unconfigured') - elif len(response[1].configs.prefix) != 12: - raise totpcgi.VerifyFailed('Invalid Yubikey OTP prefix') - if not otp.startswith(response[1].configs.prefix): - raise totpcgi.VerifyFailed('Unauthorized/Invalid OTP') - - # Okay, everything passed - return 'Valid yubikey returned' - - -### End of custom Fedora Functions - -def bad_request(why): - output = 'ERR\n' + why + '\n' - sys.stdout.write('Status: 400 BAD REQUEST\n') - sys.stdout.write('Content-type: text/plain\n') - sys.stdout.write('Content-Length: %s\n' % len(output)) - sys.stdout.write('\n') - - sys.stdout.write(output) - sys.exit(0) - -def cgimain(): - form = cgi.FieldStorage() - - must_keys = ('user', 'token', 'mode') - - for must_key in must_keys: - if must_key not in form: - bad_request("Missing field: %s" % must_key) - - user = form.getfirst('user') - token = form.getfirst('token') - mode = form.getfirst('mode') - - remote_host = os.environ['REMOTE_ADDR'] - - if mode != 'PAM_SM_AUTH': - bad_request('We only support PAM_SM_AUTH') - - if parse_token(token): - ga = YubikeyAuthenticator(require_pincode) - else: - # totp/googleauth - ga = totpcgi.GoogleAuthenticator(backends, require_pincode) - - try: - status = ga.verify_user_token(user, token) - except Exception, ex: - logging.warning( - "TOKEN FAILURE! user=%s, mode=%s, host=%s, message=%s", - user, - mode, - remote_host, - str(ex)) - bad_request(str(ex)) - - logging.info( - "Token success! user=%s, mode=%s, host=%s, message=%s", - user, - mode, - remote_host, - status) - - sys.stdout.write('Status: 200 OK\n') - sys.stdout.write('Content-type: text/plain\n') - sys.stdout.write('Content-Length: %s\n' % len(success_string)) - sys.stdout.write('\n') - - sys.stdout.write(success_string) - -if __name__ == '__main__': - try: - cgimain() - except Exception: - logging.exception("Server error during processing") - output = 'ERR\nInternal server error\n' - sys.stdout.write('Status: 500 SERVER ERROR\n') - sys.stdout.write('Content-type: text/plain\n') - sys.stdout.write('Content-Length: %s\n' % len(output)) - sys.stdout.write('\n') - - sys.stdout.write(output) - sys.exit(0) diff --git a/roles/totpcgi/files/pam_url.conf b/roles/totpcgi/files/pam_url.conf deleted file mode 100644 index 4ba6808355..0000000000 --- a/roles/totpcgi/files/pam_url.conf +++ /dev/null @@ -1,21 +0,0 @@ -pam_url: -{ - settings: - { - url = "https://fas-all.iad2.fedoraproject.org:8443/"; # URI to fetch - returncode = "OK"; # The remote script/cgi should return a 200 http code and this string as its only results - userfield = "user"; # userfield name to send - passwdfield = "token"; # passwdfield name to send - extradata = "&do=login"; # extradata to send - prompt = "Password+Token: "; # password prompt - }; - - ssl: - { - verify_peer = true; # Should we verify SSL ? - verify_host = true; # Should we verify the CN in the SSL cert? - client_cert = "/etc/pki/tls/private/totpcgi.pem"; # file to use as client-side certificate - client_key = "/etc/pki/tls/private/totpcgi.pem"; # file to use as client-side key (can be same file as above if a single cert) - ca_cert = "/etc/pki/tls/private/totpcgi-ca.cert"; - }; -}; diff --git a/roles/totpcgi/files/pam_url.conf.fakefas01 b/roles/totpcgi/files/pam_url.conf.fakefas01 deleted file mode 100644 index 4ba6808355..0000000000 --- a/roles/totpcgi/files/pam_url.conf.fakefas01 +++ /dev/null @@ -1,21 +0,0 @@ -pam_url: -{ - settings: - { - url = "https://fas-all.iad2.fedoraproject.org:8443/"; # URI to fetch - returncode = "OK"; # The remote script/cgi should return a 200 http code and this string as its only results - userfield = "user"; # userfield name to send - passwdfield = "token"; # passwdfield name to send - extradata = "&do=login"; # extradata to send - prompt = "Password+Token: "; # password prompt - }; - - ssl: - { - verify_peer = true; # Should we verify SSL ? - verify_host = true; # Should we verify the CN in the SSL cert? - client_cert = "/etc/pki/tls/private/totpcgi.pem"; # file to use as client-side certificate - client_key = "/etc/pki/tls/private/totpcgi.pem"; # file to use as client-side key (can be same file as above if a single cert) - ca_cert = "/etc/pki/tls/private/totpcgi-ca.cert"; - }; -}; diff --git a/roles/totpcgi/files/pam_url.conf.stg b/roles/totpcgi/files/pam_url.conf.stg deleted file mode 100644 index 047ff5a90b..0000000000 --- a/roles/totpcgi/files/pam_url.conf.stg +++ /dev/null @@ -1,21 +0,0 @@ -pam_url: -{ - settings: - { - url = "https://fas-all.stg.iad2.fedoraproject.org/"; # URI to fetch - returncode = "OK"; # The remote script/cgi should return a 200 http code and this string as its only results - userfield = "user"; # userfield name to send - passwdfield = "token"; # passwdfield name to send - extradata = "&do=login"; # extradata to send - prompt = "Password+Token: "; # password prompt - }; - - ssl: - { - verify_peer = true; # Should we verify SSL ? - verify_host = true; # Should we verify the CN in the SSL cert? - client_cert = "/etc/pki/tls/private/totpcgi.pem"; # file to use as client-side certificate - client_key = "/etc/pki/tls/private/totpcgi.pem"; # file to use as client-side key (can be same file as above if a single cert) - ca_cert = "/etc/pki/tls/private/totpcgi-ca.cert"; - }; -}; diff --git a/roles/totpcgi/files/pam_url.conf.vpn b/roles/totpcgi/files/pam_url.conf.vpn deleted file mode 100644 index 6e102e12f7..0000000000 --- a/roles/totpcgi/files/pam_url.conf.vpn +++ /dev/null @@ -1,21 +0,0 @@ -pam_url: -{ - settings: - { - url = "https://fas-all.vpn.fedoraproject.org:8443/"; # URI to fetch - returncode = "OK"; # The remote script/cgi should return a 200 http code and this string as its only results - userfield = "user"; # userfield name to send - passwdfield = "token"; # passwdfield name to send - extradata = "&do=login"; # extradata to send - prompt = "Password+Token: "; # password prompt - }; - - ssl: - { - verify_peer = true; # Should we verify SSL ? - verify_host = true; # Should we verify the CN in the SSL cert? - client_cert = "/etc/pki/tls/private/totpcgi.pem"; # file to use as client-side certificate - client_key = "/etc/pki/tls/private/totpcgi.pem"; # file to use as client-side key (can be same file as above if a single cert) - ca_cert = "/etc/pki/tls/private/totpcgi-ca.cert"; - }; -}; diff --git a/roles/totpcgi/files/provisioning.cgi b/roles/totpcgi/files/provisioning.cgi deleted file mode 100644 index 7e5dda5518..0000000000 --- a/roles/totpcgi/files/provisioning.cgi +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/python -tt -## -# Copyright (C) 2012 by Konstantin Ryabitsev and contributors -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA -# 02111-1307, USA. -# -import os -import sys -import cgi -import syslog -import logging - -import cgitb -cgitb.enable(display=0, logdir="/tmp") - -import totpcgi -import totpcgi.backends -import totpcgi.utils - -import qrcode -from qrcode.image import svg -from StringIO import StringIO - -from string import Template - -if len(sys.argv) > 1: - # blindly assume it's the config file - config_file = sys.argv[1] -else: - config_file = '/etc/totpcgi/provisioning.conf' - -import ConfigParser - -config = ConfigParser.RawConfigParser() -config.read(config_file) - -backends = totpcgi.backends.Backends() - -try: - backends.load_from_config(config) -except totpcgi.backends.BackendNotSupported, ex: - syslog.syslog(syslog.LOG_CRIT, - 'Backend engine not supported: %s' % ex) - sys.exit(1) - -syslog.openlog('provisioning.cgi', syslog.LOG_PID, syslog.LOG_AUTH) - -def bad_request(config, why): - templates_dir = config.get('secret', 'templates_dir') - fh = open(os.path.join(templates_dir, 'error.html')) - tpt = Template(fh.read()) - fh.close() - - vals = { - 'action_url': config.get('secret', 'action_url'), - 'css_root': config.get('secret', 'css_root'), - 'errormsg': cgi.escape(why) - } - - out = tpt.safe_substitute(vals) - - sys.stdout.write('Status: 400 BAD REQUEST\n') - sys.stdout.write('Content-type: text/html\n') - sys.stdout.write('Content-Length: %s\n' % len(out)) - sys.stdout.write('\n') - - sys.stdout.write(out) - sys.exit(0) - -def show_qr_code(data): - qr = qrcode.QRCode( - version=1, - error_correction=qrcode.constants.ERROR_CORRECT_L, - box_size=5, - border=4) - - qr.add_data(data) - qr.make(fit=True) - - img = qr.make_image() - - fh = StringIO() - img.save(fh) - out = fh.getvalue() - fh.close() - - sys.stdout.write('Status: 200 OK\n') - sys.stdout.write('Content-type: image/png\n') - sys.stdout.write('Content-Length: %s\n' % len(out)) - sys.stdout.write('\n') - - sys.stdout.write(out) - sys.exit(0) - -def show_login_form(config): - templates_dir = config.get('secret', 'templates_dir') - fh = open(os.path.join(templates_dir, 'login.html')) - tpt = Template(fh.read()) - fh.close() - - vals = { - 'action_url': config.get('secret', 'action_url'), - 'css_root': config.get('secret', 'css_root') - } - - out = tpt.safe_substitute(vals) - - sys.stdout.write('Status: 200 OK\n') - sys.stdout.write('Content-type: text/html\n') - sys.stdout.write('Content-Length: %s\n' % len(out)) - sys.stdout.write('\n') - - sys.stdout.write(out) - sys.exit(0) - -def show_totp_page(config, user, gaus): - # generate provisioning URI - tpt = Template(config.get('secret', 'totp_user_mask')) - totp_user = tpt.safe_substitute(username=user) - totp_qr_uri = gaus.totp.provisioning_uri(totp_user) - - action_url = config.get('secret', 'action_url') - - qrcode_embed = '' % (action_url, totp_qr_uri) - - templates_dir = config.get('secret', 'templates_dir') - fh = open(os.path.join(templates_dir, 'totp.html')) - tpt = Template(fh.read()) - fh.close() - - if gaus.scratch_tokens: - scratch_tokens = '
'.join(gaus.scratch_tokens) - else: - scratch_tokens = ' ' - - vals = { - 'action_url': action_url, - 'css_root': config.get('secret', 'css_root'), - 'qrcode_embed': qrcode_embed, - 'scratch_tokens': scratch_tokens - } - - out = tpt.safe_substitute(vals) - - sys.stdout.write('Status: 200 OK\n') - sys.stdout.write('Content-type: text/html\n') - sys.stdout.write('Content-Length: %s\n' % len(out)) - sys.stdout.write('\n') - - sys.stdout.write(out) - sys.exit(0) - -def generate_secret(config): - encrypt_secret = config.getboolean('secret', 'encrypt_secret') - window_size = config.getint('secret', 'window_size') - rate_limit = config.get('secret', 'rate_limit') - - # scratch tokens don't make any sense with encrypted secret - if not encrypt_secret: - scratch_tokens_n = config.getint('secret', 'scratch_tokens_n') - else: - scratch_tokens_n = 0 - - (times, secs) = rate_limit.split(',') - rate_limit = (int(times), int(secs)) - - gaus = totpcgi.utils.generate_secret(rate_limit, window_size, - scratch_tokens_n) - - return gaus - - -def cgimain(): - form = cgi.FieldStorage() - - if 'qrcode' in form: - #if os.environ['HTTP_REFERER'].find(os.environ['SERVER_NAME']) == -1: - # bad_request(config, 'Sorry, you failed the HTTP_REFERER check') - - qrcode = form.getfirst('qrcode') - show_qr_code(qrcode) - - remote_host = os.environ['REMOTE_ADDR'] - - try: - trust_http_auth = config.getboolean('secret', 'trust_http_auth') - except ConfigParser.NoOptionError: - trust_http_auth = False - - if trust_http_auth and os.environ.has_key('REMOTE_USER'): - user = os.environ['REMOTE_USER'] - pincode = None - - syslog.syslog(syslog.LOG_NOTICE, - 'Success (http-auth): user=%s, host=%s' % (user, remote_host)) - - else: - must_keys = ('username', 'pincode') - - for must_key in must_keys: - if must_key not in form: - show_login_form(config) - - user = form.getfirst('username') - pincode = form.getfirst('pincode') - - # start by verifying the pincode - try: - backends.pincode_backend.verify_user_pincode(user, pincode) - except Exception, ex: - syslog.syslog(syslog.LOG_NOTICE, - 'Failure: user=%s, host=%s, message=%s' % (user, remote_host, - str(ex))) - bad_request(config, str(ex)) - - # pincode verified - syslog.syslog(syslog.LOG_NOTICE, - 'Success: user=%s, host=%s' % (user, remote_host)) - - # is there an existing secret for this user? - exists = True - - try: - backends.secret_backend.get_user_secret(user, pincode) - except totpcgi.UserNotFound: - # if we got it, then there isn't an existing secret in place - exists = False - - if exists: - syslog.syslog(syslog.LOG_NOTICE, - 'Secret exists: user=%s, host=%s' % (user, remote_host)) - bad_request(config, 'Existing secret found. It must be removed first.') - - # now generate the secret and store it - - gaus = generate_secret(config) - - # if we don't need to encrypt the secret, set pincode to None - encrypt_secret = config.getboolean('secret', 'encrypt_secret') - if not encrypt_secret: - pincode = None - - backends.secret_backend.save_user_secret(user, gaus, pincode) - # purge all old state, as it's now obsolete - - backends.state_backend.delete_user_state(user) - - show_totp_page(config, user, gaus) - - -if __name__ == '__main__': - cgimain() diff --git a/roles/totpcgi/files/sudo.pam b/roles/totpcgi/files/sudo.pam deleted file mode 100644 index aa59ebf7a7..0000000000 --- a/roles/totpcgi/files/sudo.pam +++ /dev/null @@ -1,11 +0,0 @@ -#%PAM-1.0 -auth required pam_env.so -auth sufficient pam_url.so config=/etc/pam_url.conf -auth requisite pam_succeed_if.so uid >= 500 quiet -auth required pam_deny.so - -auth include system-auth -account include system-auth -password include system-auth -session optional pam_keyinit.so revoke -session required pam_limits.so diff --git a/roles/totpcgi/files/sudo.pam.dev.fedoraproject.org b/roles/totpcgi/files/sudo.pam.dev.fedoraproject.org deleted file mode 100644 index 030bb26463..0000000000 --- a/roles/totpcgi/files/sudo.pam.dev.fedoraproject.org +++ /dev/null @@ -1,6 +0,0 @@ -#%PAM-1.0 -auth include system-auth -account include system-auth -password include system-auth -session optional pam_keyinit.so revoke -session required pam_limits.so diff --git a/roles/totpcgi/files/sudo.pam.qa.fedoraproject.org b/roles/totpcgi/files/sudo.pam.qa.fedoraproject.org deleted file mode 100644 index 030bb26463..0000000000 --- a/roles/totpcgi/files/sudo.pam.qa.fedoraproject.org +++ /dev/null @@ -1,6 +0,0 @@ -#%PAM-1.0 -auth include system-auth -account include system-auth -password include system-auth -session optional pam_keyinit.so revoke -session required pam_limits.so diff --git a/roles/totpcgi/tasks/main.yml b/roles/totpcgi/tasks/main.yml deleted file mode 100644 index 6169dac7e5..0000000000 --- a/roles/totpcgi/tasks/main.yml +++ /dev/null @@ -1,236 +0,0 @@ -- name: add totpcgi user - user: name=totpcgi uid=501 state=present home=/var/lib/totpcgi createhome=yes system=yes - tags: - - config - -- name: install needed packages - package: name={{ item }} state=present - with_items: - - mod_auth_pgsql - - totpcgi - - totpcgi-selinux - - totpcgi-provisioning - - python-qrcode - - httpd - - mod_ssl - tags: - - packages - -- name: Install the cgi apache configuration files - template: > - src={{ item.file }}.j2 dest=/etc/httpd/conf.d/{{ item.dest }} - owner=root group=root mode=0444 - with_items: - - {file: provisioning-httpd.conf, dest: totpcgi-provisioning.conf } - tags: - - files - - config - notify: - - restart apache - -- name: create directories - file: path=/etc/{{ item }} state=directory owner=root group=totpcgi mode=750 - with_items: - - pki/totpcgi - - totpcgi/totp - -- name: create template directory for totpcgiprov - file: path=/etc/{{ item }} state=directory owner=root group=totpcgiprov mode=750 - with_items: - - totpcgi/templates - -- name: create /etc/totpcgi with the proper rights and owners - file: path=/etc/{{ item }} state=directory owner=totpcgiprov group=totpcgi mode=750 - with_items: - - totpcgi - -- name: copy html files over - copy: > - src=html - dest=/etc/totpcgi/templates - owner=root - group=totpcgiprov - mode=0750 - tags: - - files - - config - -- name: copy provisioning index file over - copy: > - src=provisioning.cgi - dest=/var/www/totpcgi-provisioning/index.cgi - owner=totpcgiprov - group=totpcgiprov - mode=0550 - tags: - - files - - config - -- name: copy index file over - copy: > - src=index.cgi - dest=/var/www/totpcgi/index.cgi - owner=totpcgi - group=totpcgi - mode=0550 - tags: - - files - - config - -- name: copy totpcgi.conf file over - template: > - src=totpcgi.conf.j2 - dest=/etc/totpcgi/totpcgi.conf - owner=root - group=totpcgi - mode=0640 - tags: - - files - - config - -# staging certs - -- name: copy staging server cert file over - copy: > - src={{ private }}/files/2fa-certs/keys/fas-all.stg.iad2.fedoraproject.org.crt - dest=/etc/pki/tls/certs/totpcgi-server.crt - owner=root - group=totpcgi - mode=0640 - tags: - - files - - config - when: env == "staging" - -- name: copy staging server key file over - copy: > - src={{ private }}/files/2fa-certs/keys/fas-all.stg.iad2.fedoraproject.org.key - dest=/etc/pki/totpcgi/totpcgi-server.key - owner=root - group=totpcgi - mode=0640 - tags: - - files - - config - when: env == "staging" - -- name: copy staging server conf file over - template: > - src=totpcgi-httpd.conf.stg.j2 - dest=/etc/httpd/conf.d/totpcgi.conf - owner=root - group=root - mode=0444 - tags: - - files - - config - - sslciphers - when: env == "staging" - -# prod certs - -- name: copy server cert file over - copy: > - src={{ private }}/files/2fa-certs/keys/fas-all.iad2.fedoraproject.org.crt - dest=/etc/pki/totpcgi/totpcgi-server.crt - owner=root - group=totpcgi - mode=0640 - tags: - - files - - config - notify: - - reload httpd - when: env == "production" - -- name: copy server cert file over - copy: > - src={{ private }}/files/2fa-certs/keys/fas-all.iad2.fedoraproject.org.key - dest=/etc/pki/totpcgi/totpcgi-server.key - owner=root - group=totpcgi - mode=0640 - tags: - - files - - config - notify: - - reload httpd - when: env == "production" - -- name: copy totpcgi httpd config - template: > - src=totpcgi-httpd.conf.j2 - dest=/etc/httpd/conf.d/totpcgi.conf - owner=root - group=root - mode=0444 - tags: - - files - - config - - sslciphers - notify: - - reload httpd - when: env == "production" - -# vpn certs - -- name: copy VPN server cert file over - copy: > - src={{ private }}/files/2fa-certs/keys/fas-all.vpn.fedoraproject.org.crt - dest=/etc/pki/totpcgi/totpcgi-server-vpn.crt - owner=root - group=totpcgi - mode=0640 - tags: - - files - - config - notify: - - reload httpd - when: env == "production" - -- name: copy VPN server cert file over - copy: > - src={{ private }}/files/2fa-certs/keys/fas-all.vpn.fedoraproject.org.key - dest=/etc/pki/totpcgi/totpcgi-server-vpn.key - owner=root - group=totpcgi - mode=0640 - tags: - - files - - config - when: env == "production" - -- name: copy VPN server cert file over - template: > - src=totpcgi-httpd.conf.vpn.j2 - dest=/etc/httpd/conf.d/totpcgi-vpn.conf - owner=root - group=root - mode=0444 - tags: - - files - - config - - sslciphers - when: env == "production" - -- name: copy ca cert over - copy: > - src={{ private }}/files/2fa-certs/keys/ca.crt - dest=/etc/pki/totpcgi/totpcgi-ca.crt - owner=root - group=totpcgi - mode=0640 - tags: - - files - - config - -- name: copy provisioning.conf over - template: > - src=provisioning.conf.j2 - dest=/etc/totpcgi/provisioning.conf - owner=root - group=totpcgiprov - mode=0640 - tags: - - files - - config diff --git a/roles/totpcgi/templates/configmap.yml b/roles/totpcgi/templates/configmap.yml deleted file mode 100644 index fc059a8d76..0000000000 --- a/roles/totpcgi/templates/configmap.yml +++ /dev/null @@ -1,138 +0,0 @@ -{% macro load_file(filename) %}{% include filename %}{%- endmacro -%} ---- -apiVersion: v1 -kind: ConfigMap -metadata: -{% if objectname == "configmap-totpcgi-vpn.yml" %} - name: totpcgi-vpn -{% else %} - name: totpcgi -{% endif %} - labels: - app: fas -data: - totpcgi-ca.crt: |- - {{ lookup('file', '{{ private }}/files/2fa-certs/keys/ca.crt') | indent() }} - provisioning.conf: |- - {{ load_file('provisioning.conf.j2') | indent() }} - totpcgi.conf: |- - {{ load_file('totpcgi.conf.j2') | indent() }} - start.sh: |- - set -xe - rm -rf /httpdir/* - mkdir /httpdir/run - ln -s /etc/httpd/modules /httpdir/modules - truncate --size=0 /httpdir/accesslog /httpdir/errorlog - tail -qf /httpdir/accesslog /httpdir/errorlog & - exec /usr/sbin/httpd.worker -f /etc/totpcgi/httpd.conf -DFOREGROUND -DNO_DETACH - httpd.conf: |- - ServerRoot "/httpdir" - PidFile "/httpdir/httpd.pid" - LoadModule authn_file_module modules/mod_authn_file.so - LoadModule authn_anon_module modules/mod_authn_anon.so - LoadModule authz_user_module modules/mod_authz_user.so - LoadModule authz_host_module modules/mod_authz_host.so - LoadModule include_module modules/mod_include.so - LoadModule log_config_module modules/mod_log_config.so - LoadModule env_module modules/mod_env.so - LoadModule ext_filter_module modules/mod_ext_filter.so - LoadModule expires_module modules/mod_expires.so - LoadModule headers_module modules/mod_headers.so - LoadModule mime_module modules/mod_mime.so - LoadModule dir_module modules/mod_dir.so - LoadModule alias_module modules/mod_alias.so - LoadModule version_module modules/mod_version.so - LoadModule ssl_module modules/mod_ssl.so - LoadModule auth_pgsql_module modules/mod_auth_pgsql.so - LoadModule cgi_module modules/mod_cgi.so - - # There's so much in python(-fedora) that wants a valid homedir.... - SetEnv HOME /httpdir - - Listen 0.0.0.0:8080 - Listen 0.0.0.0:8443 https - - StartServers 4 - MaxClients 300 - MinSpareThreads 25 - MaxSpareThreads 75 - ThreadsPerChild 25 - MaxRequestsPerChild 0 - LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined - CustomLog /httpdir/accesslog combined - ErrorLog /httpdir/errorlog - LogLevel info - TypesConfig /etc/mime.types - AddDefaultCharset UTF-8 - CoreDumpDirectory /tmp - - DirectoryIndex index.cgi - - - Header set Cache-Control no-cache - Header set Expires 0 - Alias /totpcgiprovision /var/www/totpcgi-provisioning -{% if env == "staging" %} - Alias /totpcgiprovision/ /var/www/totpcgi-provisioning -{% endif %} - AddHandler cgi-script .cgi - - DocumentRoot /var/www/totpcgi-provisioning - - Options ExecCGI - - - - AuthType Basic - AuthName "Fedora totpcgi" - - Auth_PG_host db-fas{{ env_suffix }} - Auth_PG_port 5432 - Auth_PG_user fasreadonly - Auth_PG_pwd {{ fasReadOnlyPassword }} - Auth_PG_database fas2 - Auth_PG_pwd_table people - Auth_PG_uid_field username - Auth_PG_pwd_field password - Auth_PG_pwd_whereclause " and status='active'" - - Require valid-user - - - - -{% if env == "staging" %} - ServerName fas-all.stg.iad2.fedoraproject.org:8443 -{% elif objectname == "configmap-totpcgi-vpn.yml" %} - ServerName fas-all.vpn.fedoraproject.org:8443 -{% elif datacenter == 'iad2' %} - ServerName fas-all.iad2.fedoraproject.org:8443 -{% endif %} - SSLEngine on -{% if objectname == "configmap-totpcgi-vpn.yml" %} - SSLCertificateFile /etc/pki/totp/tls.crt - SSLCertificateKeyFile /etc/pki/totp/tls.key -{% else %} - SSLCertificateFile /etc/pki/totp/tls.crt - SSLCertificateKeyFile /etc/pki/totp/tls.key -{% endif %} - SSLCACertificateFile /etc/totpcgi/totpcgi-ca.crt - SSLHonorCipherOrder On - SSLCipherSuite {{ ssl_ciphers }} - SSLProtocol {{ ssl_protocols }} - AddHandler cgi-script .cgi - - SSLVerifyClient require - SSLVerifyDepth 10 - - DocumentRoot /var/www/totpcgi - - Options ExecCGI - - - error.html: |- - {{ lookup('file', '{{ roles_path }}/totpcgi/files/html/error.html') | indent() }} - login.html: |- - {{ lookup('file', '{{ roles_path }}/totpcgi/files/html/login.html') | indent() }} - totp.html: |- - {{ lookup('file', '{{ roles_path }}/totpcgi/files/html/totp.html') | indent() }} diff --git a/roles/totpcgi/templates/provisioning-httpd.conf.j2 b/roles/totpcgi/templates/provisioning-httpd.conf.j2 deleted file mode 100644 index 6b776fbb27..0000000000 --- a/roles/totpcgi/templates/provisioning-httpd.conf.j2 +++ /dev/null @@ -1,44 +0,0 @@ -Listen 8444 - - LoadModule suexec_module modules/mod_suexec.so - - DocumentRoot /var/www/totpcgi-provisioning - ServerName fas01.stg.iad2.fedoraproject.org:8444 - ErrorLog /var/log/httpd/totpcgi-provisioning-error.log - SuexecUserGroup totpcgiprov totpcgiprov - - AddHandler cgi-script .cgi - DirectoryIndex index.cgi - - Header set Cache-Control no-cache - Header set Expires 0 - - #SSLEngine on - #SSLCertificateFile /etc/pki/totpcgi/totpcgi-server.crt - #SSLCertificateKeyFile /etc/pki/totpcgi/totpcgi-server.key - #SSLCACertificateFile /etc/pki/totpcgi/totpcgi-ca.crt - - #CustomLog /var/log/httpd/totpcgi-provisioning-ssl-request-log \ - # "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b" - - - Options ExecCGI - - - - AuthType Basic - AuthName "Fedora totpcgi" - - Auth_PG_host db-fas - Auth_PG_port 5432 - Auth_PG_user fasreadonly - Auth_PG_pwd {{ fasReadOnlyPassword }} - Auth_PG_database fas2 - Auth_PG_pwd_table people - Auth_PG_uid_field username - Auth_PG_pwd_field password - Auth_PG_pwd_whereclause " and status='active'" - - Require valid-user - - diff --git a/roles/totpcgi/templates/provisioning.conf.j2 b/roles/totpcgi/templates/provisioning.conf.j2 deleted file mode 100644 index b15c918d45..0000000000 --- a/roles/totpcgi/templates/provisioning.conf.j2 +++ /dev/null @@ -1,88 +0,0 @@ -[secret] -# Whether to encrypt the secret when we generate it. Encrypting the secret -# with the user's pincode means that even if the .totp file is leaked, an -# attacker will not be able to get the secret without knowing the user's -# pincode. The downside is that if a user forgets their pincode, both the -# pincode and the secret will need to be fully re-provisioned. -# Setting to "True" will also turn off scratch-token support. -encrypt_secret = False - -# You can allow for some clock drift between the client and server by setting -# the permitted window size. Window size is calculated in 10-second intervals, -# so a window size of 6 allows clock drift of 60 seconds in either direction. -window_size = 6 - -# First value is the number of times. Second value is the number of seconds. -# So, "3, 30" means "3 falures within 30 seconds" -rate_limit = 3, 30 - -# How many scratch tokens to generate. Note, that this setting is ignored -# if encrypt_secret is set to True. -scratch_tokens_n = 5 - -# This identifies the token in the Google Authenticator application. It comes -# very handy when users have more than one token, so set this to something -# descriptive of your environment. -{% if env == "staging" %} -totp_user_mask = $username@stg.fedoraproject.org -{% else %} -totp_user_mask = $username@fedoraproject.org -{% endif %} - -# Used by provisioning.cgi -# Where the provisioning CGI is located, with regards to the web root. -action_url = /totpcgiprovision/index.cgi - -# Used by provisioning.cgi -# Where provisioning.css and provisioning-print.css are located with regards -# to the web root. -css_root = /totpcgiprovision/ - -# Used by provisioning.cgi -# Where to find the templates files. -templates_dir = /etc/totpcgi - -# Used by provisioning.cgi -# Whether to rely on HTTP auth to handle authentication. -# As we don't get the password, only the username, turning this on -# will automatically set encrypt_secret to false. -# -# Be careful turning this on. -trust_http_auth = True - - -[pincode] -# Which hashing mechanism to use. Valid entries: md5, bcrypt, sha256, sha512 -usehash = sha256 - -# Whether to compile the DBM database (only meaningful with the file backend) -makedb = True - -# The backends are pretty much the same as in totpcgi.conf, except if you -# are using the postgresql secret backend, you need to connect as a user -# that is allowed to modify user records (e.g. totpcgi_admin). -[secret_backend] -;engine = file -;secrets_dir = /etc/totpcgi/totp - -; For PostgreSQL backend: -engine = pgsql -pg_connect_string = user={{ totpcgiadminDBUser }} password={{ totpcgiadminDBPassword }} host=db-fas01{{ env_suffix }}.{{ datacenter }}.fedoraproject.org dbname=totpcgi - -[pincode_backend] -engine = pgsql -pg_connect_string = user={{ totpcgiadminDBUser }} password={{ totpcgiadminDBPassword }} host=db-fas01{{ env_suffix }}.{{ datacenter }}.fedoraproject.org dbname=totpcgi - -; For LDAP backend (simple bind auth): -;engine = ldap -;ldap_url = ldaps://ipa.example.com:636/ -;ldap_dn = uid=$username,cn=users,cn=accounts,dc=example,dc=com -;ldap_cacert = /etc/pki/tls/certs/ipa-ca.crt - -[state_backend] -;engine = file -;state_dir = /var/lib/totpcgi - -; For PostgreSQL backend: -engine = pgsql -pg_connect_string = user={{ totpcgiadminDBUser }} password={{ totpcgiadminDBPassword }} host=db-fas01{{ env_suffix }}.{{ datacenter }}.fedoraproject.org dbname=totpcgi diff --git a/roles/totpcgi/templates/totpcgi-httpd.conf.j2 b/roles/totpcgi/templates/totpcgi-httpd.conf.j2 deleted file mode 100644 index 29cd380418..0000000000 --- a/roles/totpcgi/templates/totpcgi-httpd.conf.j2 +++ /dev/null @@ -1,37 +0,0 @@ -Listen 8443 - - # Load this module locally here. - LoadModule suexec_module modules/mod_suexec.so - - ServerAdmin admin@fedoraproject.org - DocumentRoot /var/www/totpcgi - ServerName fas-all.iad2.fedoraproject.org:8443 - ErrorLog /var/log/httpd/totpcgi-error.log - SuexecUserGroup totpcgi totpcgi - - # Use this for totp.cgi - AddHandler cgi-script .cgi - DirectoryIndex index.cgi - - # Or use this for totp.fcgi: - #AddHandler fcgid-script .fcgi - #DirectoryIndex index.fcgi - - SSLEngine on - SSLCertificateFile /etc/pki/totpcgi/totpcgi-server.crt - SSLCertificateKeyFile /etc/pki/totpcgi/totpcgi-server.key - SSLCACertificateFile /etc/pki/totpcgi/totpcgi-ca.crt - SSLHonorCipherOrder On - SSLCipherSuite {{ ssl_ciphers }} - SSLProtocol {{ ssl_protocols }} - - SSLVerifyClient require - SSLVerifyDepth 10 - - CustomLog /var/log/httpd/totpcgi-ssl-request-log \ - "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b" - - - Options ExecCGI - - diff --git a/roles/totpcgi/templates/totpcgi-httpd.conf.stg.j2 b/roles/totpcgi/templates/totpcgi-httpd.conf.stg.j2 deleted file mode 100644 index 5424c8e3d0..0000000000 --- a/roles/totpcgi/templates/totpcgi-httpd.conf.stg.j2 +++ /dev/null @@ -1,37 +0,0 @@ -Listen 8443 - - # Load this module locally here. - LoadModule suexec_module modules/mod_suexec.so - - ServerAdmin admin@fedoraproject.org - DocumentRoot /var/www/totpcgi - ServerName fas-all.stg.iad2.fedoraproject.org:8443 - ErrorLog /var/log/httpd/totpcgi-error.log - SuexecUserGroup totpcgi totpcgi - - # Use this for totp.cgi - AddHandler cgi-script .cgi - DirectoryIndex index.cgi - - # Or use this for totp.fcgi: - #AddHandler fcgid-script .fcgi - #DirectoryIndex index.fcgi - - SSLEngine on - SSLCertificateFile /etc/pki/totpcgi/totpcgi-server.crt - SSLCertificateKeyFile /etc/pki/totpcgi/totpcgi-server.key - SSLCACertificateFile /etc/pki/totpcgi/totpcgi-ca.crt - SSLHonorCipherOrder On - SSLCipherSuite {{ ssl_ciphers }} - SSLProtocol {{ ssl_protocols }} - - SSLVerifyClient require - SSLVerifyDepth 10 - - CustomLog /var/log/httpd/totpcgi-ssl-request-log \ - "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b" - - - Options ExecCGI - - diff --git a/roles/totpcgi/templates/totpcgi-httpd.conf.vpn.j2 b/roles/totpcgi/templates/totpcgi-httpd.conf.vpn.j2 deleted file mode 100644 index eb88528e3f..0000000000 --- a/roles/totpcgi/templates/totpcgi-httpd.conf.vpn.j2 +++ /dev/null @@ -1,36 +0,0 @@ - - # Load this module locally here. - LoadModule suexec_module modules/mod_suexec.so - - ServerAdmin admin@fedoraproject.org - DocumentRoot /var/www/totpcgi - ServerName fas-all.vpn.fedoraproject.org:8443 - ErrorLog /var/log/httpd/totpcgi-error.log - SuexecUserGroup totpcgi totpcgi - - # Use this for totp.cgi - AddHandler cgi-script .cgi - DirectoryIndex index.cgi - - # Or use this for totp.fcgi: - #AddHandler fcgid-script .fcgi - #DirectoryIndex index.fcgi - - SSLEngine on - SSLCertificateFile /etc/pki/totpcgi/totpcgi-server-vpn.crt - SSLCertificateKeyFile /etc/pki/totpcgi/totpcgi-server-vpn.key - SSLCACertificateFile /etc/pki/totpcgi/totpcgi-ca.crt - SSLHonorCipherOrder On - SSLCipherSuite {{ ssl_ciphers }} - SSLProtocol {{ ssl_protocols }} - - SSLVerifyClient require - SSLVerifyDepth 10 - - CustomLog /var/log/httpd/totpcgi-ssl-request-log \ - "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b" - - - Options ExecCGI - - diff --git a/roles/totpcgi/templates/totpcgi.conf.j2 b/roles/totpcgi/templates/totpcgi.conf.j2 deleted file mode 100644 index bd6630dfad..0000000000 --- a/roles/totpcgi/templates/totpcgi.conf.j2 +++ /dev/null @@ -1,31 +0,0 @@ -[main] -require_pincode = True -success_string = OK -{% if env == "staging" %} -fas_url = https://admin.stg.fedoraproject.org/accounts/ -{% else %} -fas_url = https://admin.fedoraproject.org/accounts/ -{% endif %} - -[secret_backend] -; For PostgreSQL backend: -engine = pgsql -pg_connect_string = user={{ totpcgiDBUser }} password={{ totpcgiDBPassword }} host=db-fas01{{ env_suffix }}.{{ datacenter }}.fedoraproject.org dbname=totpcgi - -[pincode_backend] -engine = pgsql -pg_connect_string = user={{ totpcgiDBUser }} password={{ totpcgiDBPassword }} host=db-fas01{{ env_suffix }}.{{ datacenter }}.fedoraproject.org dbname=totpcgi - -; For LDAP backend (simple bind auth): -;engine = ldap -;ldap_url = ldaps://ipa.example.com:636/ -;ldap_dn = uid=$username,cn=users,cn=accounts,dc=example,dc=com -;ldap_cacert = /etc/pki/tls/certs/ipa-ca.crt - -[state_backend] -;engine = file -;state_dir = /var/lib/totpcgi - -; For PostgreSQL backend: -engine = pgsql -pg_connect_string = user={{ totpcgiDBUser }} password={{ totpcgiDBPassword }} host=db-fas01{{ env_suffix }}.{{ datacenter }}.fedoraproject.org dbname=totpcgi diff --git a/tasks/2fa_client.yml b/tasks/2fa_client.yml deleted file mode 100644 index 8106193021..0000000000 --- a/tasks/2fa_client.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -- name: install pam_url - package: name=pam_url state=present - tags: - - packages - - 2fa - -- name: /etc/pki/tls/private/totpcgi.pem - copy: src="{{ private }}/files/2fa-certs/keys/{{ inventory_hostname }}.pem" dest=/etc/pki/tls/private/totpcgi.pem mode=0400 - tags: - - config - - 2fa - -- name: /etc/pki/tls/private/totpcgi-ca.cert - copy: src="{{ private }}/files/2fa-certs/keys/ca.crt" dest=/etc/pki/tls/private/totpcgi-ca.cert mode=0400 - tags: - - config - - 2fa - -- name: /etc/pam_url.conf - split for staging/phx2/everyone else - template: src={{ item }} dest=/etc/pam_url.conf mode=0644 - with_first_found: - - "{{ files }}/2fa/pam_url.conf.{{ inventory_hostname }}" - - "{{ files }}/2fa/pam_url.conf.{{ ansible_domain }}" - - "{{ files }}/2fa/pam_url.conf.{{ datacenter }}" - - "{{ files }}/2fa/pam_url.conf.j2" - tags: - - config - - pam_url - - 2fa - -- name: /etc/pam.d/sudo - copy: src={{ item }} dest=/etc/pam.d/sudo mode=0644 - with_first_found: - - "{{ files }}/2fa/sudo.pam.{{ inventory_hostname }}" - - "{{ files }}/2fa/sudo.pam.{{ ansible_domain }}" - - "{{ files }}/2fa/sudo.pam" - tags: - - config - - 2fa From e0662c5f2dea6a2329f34ff0840683672236e8b0 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 17 May 2021 13:46:36 -0700 Subject: [PATCH 034/189] rsyncd / download: adjust odcs to allow package rsyncs odcs just links to the packages in a compose on the /mnt/koji volume with relative symlinks. ie ( ../../../../../../../../mnt/koji/packages/zip/3.0/28.eln109/data/signed/9867c58f/x86_64/zip-3.0-28.eln109.x86_64.rpm ) Change odcs on download servers to mount on /mnt like koji and then tell rsync to chroot at /mnt, so it can follow those links and copy the file from koji volume. Signed-off-by: Kevin Fenzi --- playbooks/groups/download.yml | 2 +- .../download/files/httpd/dl.fedoraproject.org/secondary.conf | 4 ++-- roles/rsyncd/templates/rsyncd.conf.download.j2 | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/playbooks/groups/download.yml b/playbooks/groups/download.yml index a5238e8ec9..051fe40b6e 100644 --- a/playbooks/groups/download.yml +++ b/playbooks/groups/download.yml @@ -40,7 +40,7 @@ - { role: nfs/client, when: datacenter == "iad2" or datacenter == "rdu", mnt_dir: '/srv/pub', nfs_src_dir: 'fedora_ftp/fedora.redhat.com/pub' } - { role: nfs/client, when: datacenter == "iad2" or datacenter == "rdu", mnt_dir: '/srv/pub/archive', nfs_src_dir: 'fedora_ftp_archive' } - { role: nfs/client, when: datacenter == "iad2", mnt_dir: '/mnt/koji', nfs_src_dir: 'fedora_koji/koji/' } # needed for internal sync and odcs - - { role: nfs/client, when: datacenter == "iad2", mnt_dir: '/srv/odcs', nfs_src_dir: 'fedora_odcs' } # needed for internal sync + - { role: nfs/client, when: datacenter == "iad2", mnt_dir: '/mnt/odcs', nfs_src_dir: 'fedora_odcs' } # needed for internal sync - sudo pre_tasks: diff --git a/roles/download/files/httpd/dl.fedoraproject.org/secondary.conf b/roles/download/files/httpd/dl.fedoraproject.org/secondary.conf index be85ad32d8..7f0f3695ae 100644 --- a/roles/download/files/httpd/dl.fedoraproject.org/secondary.conf +++ b/roles/download/files/httpd/dl.fedoraproject.org/secondary.conf @@ -1,5 +1,5 @@ Alias /pub /srv/pub -Alias /odcs /srv/odcs +Alias /odcs /mnt/odcs DocumentRoot /srv/web @@ -18,7 +18,7 @@ DocumentRoot /srv/web - + HeaderName /HEADER.html ReadmeName /FOOTER.html Options Indexes FollowSymLinks diff --git a/roles/rsyncd/templates/rsyncd.conf.download.j2 b/roles/rsyncd/templates/rsyncd.conf.download.j2 index 47e212a3af..4af35b9cfb 100644 --- a/roles/rsyncd/templates/rsyncd.conf.download.j2 +++ b/roles/rsyncd/templates/rsyncd.conf.download.j2 @@ -96,7 +96,7 @@ refuse options = checksum [ fedora-eln ] comment = Fedora ELN composes - path = /srv/odcs + path = /./mnt/odcs {% if inventory_hostname in groups['download_tier1'] %} hosts allow = {% for host in vars['dl_tier1'] %}{{host}},{% endfor %} {% endif %} From 488bb78ccdd79a601d20693474c9b23e64d769d5 Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Tue, 18 May 2021 12:07:49 -0600 Subject: [PATCH 035/189] blockerbugs: upgrade prod to F33 --- inventory/host_vars/blockerbugs01.iad2.fedoraproject.org | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/inventory/host_vars/blockerbugs01.iad2.fedoraproject.org b/inventory/host_vars/blockerbugs01.iad2.fedoraproject.org index ebe61448f7..4a6035ef9e 100644 --- a/inventory/host_vars/blockerbugs01.iad2.fedoraproject.org +++ b/inventory/host_vars/blockerbugs01.iad2.fedoraproject.org @@ -2,8 +2,10 @@ nm: 255.255.255.0 gw: 10.3.163.254 dns: 10.3.163.33 -ks_url: http://10.3.163.35/repo/rhel/ks/kvm-fedora-32-iad2 -ks_repo: http://10.3.163.35/pub/fedora/linux/releases/32/Server/x86_64/os/ + +ks_url: http://10.3.163.35/repo/rhel/ks/kvm-fedora +ks_repo: http://10.3.163.35/pub/fedora/linux/releases/33/Server/x86_64/os/ + volgroup: /dev/vg_guests eth0_ip: 10.3.163.88 vmhost: vmhost-x86-07.iad2.fedoraproject.org From 9c1297a026ef589a2fcff632e6a60a5b7d5e2920 Mon Sep 17 00:00:00 2001 From: Brendan Early Date: Tue, 18 May 2021 13:13:23 -0500 Subject: [PATCH 036/189] packages-static: make sure that solrconfig is applied properly --- .../fedora-packages-static/templates/configmap.yml | 4 ++-- .../templates/packages/{ => conf}/solrconfig.xml | 0 .../fedora-packages-static/templates/solr-start.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename roles/openshift-apps/fedora-packages-static/templates/packages/{ => conf}/solrconfig.xml (100%) diff --git a/roles/openshift-apps/fedora-packages-static/templates/configmap.yml b/roles/openshift-apps/fedora-packages-static/templates/configmap.yml index 0cd8c85dfc..830901e2ce 100644 --- a/roles/openshift-apps/fedora-packages-static/templates/configmap.yml +++ b/roles/openshift-apps/fedora-packages-static/templates/configmap.yml @@ -17,8 +17,8 @@ metadata: labels: app: fedora-packages-static data: - solrconfig.xml: |- - {{ load_file('packages/solrconfig.xml') | indent }} + conf/solrconfig.xml: |- + {{ load_file('packages/conf/solrconfig.xml') | indent }} schema.xml: |- {{ load_file('packages/schema.xml') | indent }} --- diff --git a/roles/openshift-apps/fedora-packages-static/templates/packages/solrconfig.xml b/roles/openshift-apps/fedora-packages-static/templates/packages/conf/solrconfig.xml similarity index 100% rename from roles/openshift-apps/fedora-packages-static/templates/packages/solrconfig.xml rename to roles/openshift-apps/fedora-packages-static/templates/packages/conf/solrconfig.xml diff --git a/roles/openshift-apps/fedora-packages-static/templates/solr-start.sh b/roles/openshift-apps/fedora-packages-static/templates/solr-start.sh index 9c341f5edf..d5428c13f7 100644 --- a/roles/openshift-apps/fedora-packages-static/templates/solr-start.sh +++ b/roles/openshift-apps/fedora-packages-static/templates/solr-start.sh @@ -13,7 +13,7 @@ fi . /opt/docker-solr/scripts/run-initdb -/opt/docker-solr/scripts/precreate-core "packages" +/opt/docker-solr/scripts/precreate-core "packages" /opt/solr/server/solr/configsets/packages #!/bin/bash # @@ -70,4 +70,4 @@ elif [[ "$TINI" == no ]]; then else echo "invalid value TINI=$TINI" exit 1 -fi \ No newline at end of file +fi From 81b2cd2e9c735a568528d598d510c81a94149aca Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 18 May 2021 13:33:37 -0700 Subject: [PATCH 037/189] download: adjust koji mount to allow odcs/eln images to work right Right now odcs uses /mnt/koji for packages links, but uses /mnt/fedora_koji/koji for images. Adjusting this and adding a link in should fix it for both. Signed-off-by: Kevin Fenzi --- playbooks/groups/download.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/playbooks/groups/download.yml b/playbooks/groups/download.yml index 051fe40b6e..b096276db7 100644 --- a/playbooks/groups/download.yml +++ b/playbooks/groups/download.yml @@ -39,7 +39,7 @@ - rsyncd - { role: nfs/client, when: datacenter == "iad2" or datacenter == "rdu", mnt_dir: '/srv/pub', nfs_src_dir: 'fedora_ftp/fedora.redhat.com/pub' } - { role: nfs/client, when: datacenter == "iad2" or datacenter == "rdu", mnt_dir: '/srv/pub/archive', nfs_src_dir: 'fedora_ftp_archive' } - - { role: nfs/client, when: datacenter == "iad2", mnt_dir: '/mnt/koji', nfs_src_dir: 'fedora_koji/koji/' } # needed for internal sync and odcs + - { role: nfs/client, when: datacenter == "iad2", mnt_dir: '/mnt/fedora_koji', nfs_src_dir: 'fedora_koji' } # needed for internal sync and odcs - { role: nfs/client, when: datacenter == "iad2", mnt_dir: '/mnt/odcs', nfs_src_dir: 'fedora_odcs' } # needed for internal sync - sudo @@ -73,6 +73,9 @@ copy: src="{{ files }}/download/download-sync.cron.cc-rdu01" dest=/etc/cron.d/download-sync owner=root group=root mode=644 when: inventory_hostname == 'download-cc-rdu01.fedoraproject.org' + - name: make a mnt/koji link + file: state=link src=/mnt/fedora_koji/koji dest=/mnt/koji + handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml" From decd358666cf60524734f6069350bfcf7241ac9d Mon Sep 17 00:00:00 2001 From: Brendan Early Date: Tue, 18 May 2021 15:51:38 -0500 Subject: [PATCH 038/189] packages-static: have the init script mange solr config openshift does not allow subdirectories in configmap volumes. --- .../fedora-packages-static/templates/configmap.yml | 4 ++-- .../templates/packages/{conf => }/solrconfig.xml | 0 .../fedora-packages-static/templates/solr-start.sh | 10 +++++++++- 3 files changed, 11 insertions(+), 3 deletions(-) rename roles/openshift-apps/fedora-packages-static/templates/packages/{conf => }/solrconfig.xml (100%) diff --git a/roles/openshift-apps/fedora-packages-static/templates/configmap.yml b/roles/openshift-apps/fedora-packages-static/templates/configmap.yml index 830901e2ce..0cd8c85dfc 100644 --- a/roles/openshift-apps/fedora-packages-static/templates/configmap.yml +++ b/roles/openshift-apps/fedora-packages-static/templates/configmap.yml @@ -17,8 +17,8 @@ metadata: labels: app: fedora-packages-static data: - conf/solrconfig.xml: |- - {{ load_file('packages/conf/solrconfig.xml') | indent }} + solrconfig.xml: |- + {{ load_file('packages/solrconfig.xml') | indent }} schema.xml: |- {{ load_file('packages/schema.xml') | indent }} --- diff --git a/roles/openshift-apps/fedora-packages-static/templates/packages/conf/solrconfig.xml b/roles/openshift-apps/fedora-packages-static/templates/packages/solrconfig.xml similarity index 100% rename from roles/openshift-apps/fedora-packages-static/templates/packages/conf/solrconfig.xml rename to roles/openshift-apps/fedora-packages-static/templates/packages/solrconfig.xml diff --git a/roles/openshift-apps/fedora-packages-static/templates/solr-start.sh b/roles/openshift-apps/fedora-packages-static/templates/solr-start.sh index d5428c13f7..45a272560b 100644 --- a/roles/openshift-apps/fedora-packages-static/templates/solr-start.sh +++ b/roles/openshift-apps/fedora-packages-static/templates/solr-start.sh @@ -13,7 +13,15 @@ fi . /opt/docker-solr/scripts/run-initdb -/opt/docker-solr/scripts/precreate-core "packages" /opt/solr/server/solr/configsets/packages +# Manually arrange config files into directory structure solr needs because openshift just won't let this be simple +mkdir -p /var/solr/openshift/packages +cp /opt/solr/server/solr/configsets/packages/schema.xml /var/solr/openshift/packages + +mkdir -p /var/solr/openshift/packages/conf +cp /opt/solr/server/solr/configsets/packages/solrconfig.xml /var/solr/openshift/packages/conf + +# Create the core +/opt/docker-solr/scripts/precreate-core "packages" /var/solr/openshift/packages #!/bin/bash # From a6558b4b35f061e9f8616eaa3b0e0408ca73ca56 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 18 May 2021 13:56:38 -0700 Subject: [PATCH 039/189] download: do not make a /mnt/koji/compose as it is handled by symlink later Signed-off-by: Kevin Fenzi --- roles/download/tasks/main.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml index bb6925cd24..fa3686906d 100644 --- a/roles/download/tasks/main.yml +++ b/roles/download/tasks/main.yml @@ -14,10 +14,6 @@ - name: Create /srv/web directory file: path=/srv/web state=directory -## This is used by internal systems to get beta content -- name: Create /mnt/koji/compose directory - file: path=/mnt/koji/compose state=directory - - name: Set httpd_use_nfs seboolean seboolean: name=httpd_use_nfs state=yes persistent=yes From 8b203e5d856277a6123d8250f0812aa86d39aaf9 Mon Sep 17 00:00:00 2001 From: Brendan Early Date: Tue, 18 May 2021 16:17:28 -0500 Subject: [PATCH 040/189] packages-static: correct solr deploymentconfig --- roles/openshift-apps/fedora-packages-static/files/service.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/openshift-apps/fedora-packages-static/files/service.yml b/roles/openshift-apps/fedora-packages-static/files/service.yml index a4dd6b1fb3..3be295f31f 100644 --- a/roles/openshift-apps/fedora-packages-static/files/service.yml +++ b/roles/openshift-apps/fedora-packages-static/files/service.yml @@ -29,4 +29,4 @@ spec: targetPort: 8983 selector: app: fedora-packages-static - deploymentconfig: fedora-packages-static + deploymentconfig: solr From e9b1c3c53d36cc0bd4d421bd8d0cf99562b1ad7a Mon Sep 17 00:00:00 2001 From: Brendan Early Date: Tue, 18 May 2021 16:52:53 -0500 Subject: [PATCH 041/189] packages-static: use unprivileged port on container --- roles/openshift-apps/fedora-packages-static/files/service.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/openshift-apps/fedora-packages-static/files/service.yml b/roles/openshift-apps/fedora-packages-static/files/service.yml index 3be295f31f..659e531014 100644 --- a/roles/openshift-apps/fedora-packages-static/files/service.yml +++ b/roles/openshift-apps/fedora-packages-static/files/service.yml @@ -10,7 +10,7 @@ spec: - name: 80-tcp port: 80 protocol: TCP - targetPort: 80 + targetPort: 8080 selector: app: fedora-packages-static deploymentconfig: fedora-packages-static From 52e2a301cc4d07eb90c408ebde4dd5350bc9ad53 Mon Sep 17 00:00:00 2001 From: Brendan Early Date: Tue, 18 May 2021 17:25:18 -0500 Subject: [PATCH 042/189] packages-static: update container port --- .../fedora-packages-static/templates/deploymentconfig.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/openshift-apps/fedora-packages-static/templates/deploymentconfig.yml b/roles/openshift-apps/fedora-packages-static/templates/deploymentconfig.yml index 89a0723aeb..4104c7babf 100644 --- a/roles/openshift-apps/fedora-packages-static/templates/deploymentconfig.yml +++ b/roles/openshift-apps/fedora-packages-static/templates/deploymentconfig.yml @@ -38,7 +38,7 @@ spec: name: fedora-packages-static-configmap key: sitemap-url ports: - - containerPort: 80 + - containerPort: 8080 resources: {} volumeMounts: - name: data-volume @@ -51,13 +51,13 @@ spec: initialDelaySeconds: 600 httpGet: path: / - port: 80 + port: 8080 livenessProbe: timeoutSeconds: 1 initialDelaySeconds: 630 httpGet: path: / - port: 80 + port: 8080 volumes: - name: data-volume persistentVolumeClaim: From 30b8e21945408741452f9408b1f62969f5a3d760 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Wed, 19 May 2021 13:28:02 +0200 Subject: [PATCH 043/189] copr-be: spot instance spawning, #1 --- .../provision/builderpb-aws-spot-aarch64.yml | 52 +++++++++++++++++++ .../provision/builderpb-aws-spot-x86_64.yml | 47 +++++++++++++++++ .../files/provision/spinup_aws_spot_task.yml | 47 +++++++++++++++++ .../backend/templates/resalloc/vm-aws-new | 18 ++++++- 4 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 roles/copr/backend/files/provision/builderpb-aws-spot-aarch64.yml create mode 100644 roles/copr/backend/files/provision/builderpb-aws-spot-x86_64.yml create mode 100644 roles/copr/backend/files/provision/spinup_aws_spot_task.yml diff --git a/roles/copr/backend/files/provision/builderpb-aws-spot-aarch64.yml b/roles/copr/backend/files/provision/builderpb-aws-spot-aarch64.yml new file mode 100644 index 0000000000..7d948c3e38 --- /dev/null +++ b/roles/copr/backend/files/provision/builderpb-aws-spot-aarch64.yml @@ -0,0 +1,52 @@ +--- +- name: create an aarch64 spot instance in aws + hosts: 127.0.0.1 + gather_facts: false + + roles: + - spawner + + vars_files: + - vars.yml + - aws_cloud_vars.yml + + vars: + keypair: copr-builder + instance_type: a1.xlarge + # We keep this around the on-demand price, as we don't want unnecessary + # interrupts. + spot_price: 0.102 + security_group: ssh-only + max_spawn_time: 1100 + spawning_vm_user: "fedora" + arch: aarch64 + image_name: "{{ builder_images[arch] }}" + instance_volumes: + - device_name: sdb + delete_on_termination: True + volume_type: gp2 + volume_size: 160 + + tasks: + - include: "spinup_aws_spot_task.yml" + + +- name: provision builder + hosts: builder_temp_group + gather_facts: false + become: true + user: fedora + + vars_files: + - vars.yml + - aws_cloud_vars.yml + + vars: + ansible_python_interpreter: /usr/bin/python3 + + tasks: + - include: "create_swap_file.yml" + when: + - prepare_base_image is defined + + - include: "provision_builder_tasks.yml" diff --git a/roles/copr/backend/files/provision/builderpb-aws-spot-x86_64.yml b/roles/copr/backend/files/provision/builderpb-aws-spot-x86_64.yml new file mode 100644 index 0000000000..d08aaa6fcd --- /dev/null +++ b/roles/copr/backend/files/provision/builderpb-aws-spot-x86_64.yml @@ -0,0 +1,47 @@ +--- +- name: create an x86_64 spot instance in aws + hosts: 127.0.0.1 + gather_facts: false + + roles: + - spawner + + vars_files: + - vars.yml + - aws_cloud_vars.yml + + vars: + keypair: copr-builder + instance_type: i3.large + # We keep this around the on-demand price, as we don't want unnecessary + # interrupts. + spot_price: 0.156 + security_group: ssh-only + max_spawn_time: 1100 + spawning_vm_user: "fedora" + arch: x86_64 + image_name: "{{ builder_images[arch] }}" + + tasks: + - include: "spinup_aws_spot_task.yml" + + +- name: provision builder + hosts: builder_temp_group + gather_facts: false + become: true + user: fedora + + vars_files: + - vars.yml + - aws_cloud_vars.yml + + vars: + ansible_python_interpreter: /usr/bin/python3 + + tasks: + - include: "create_swap_file.yml" + when: + - prepare_base_image is defined + + - include: "provision_builder_tasks.yml" diff --git a/roles/copr/backend/files/provision/spinup_aws_spot_task.yml b/roles/copr/backend/files/provision/spinup_aws_spot_task.yml new file mode 100644 index 0000000000..c1b94c63bb --- /dev/null +++ b/roles/copr/backend/files/provision/spinup_aws_spot_task.yml @@ -0,0 +1,47 @@ +--- +- debug: msg="vm_name={{ vm_name }}" + +- name: random subnet to overcome datacenter failures + set_fact: subnet_id={{ item }} + with_random_choice: "{{ aws_arch_subnets[arch] }}" + +- name: Launch instance + ec2: + key_name: "{{ keypair }}" + group: "{{ security_group }}" + instance_type: "{{ instance_type }}" + image: "{{ image_name }}" + wait: true + region: "{{ aws_region }}" + # both x86_64 and aarch64 arches can be allocated in us-east-1c + vpc_subnet_id: "{{ subnet_id }}" + assign_public_ip: yes + instance_tags: + FedoraGroup: copr + CoprPurpose: builder + CoprInstance: "{% if devel %}devel{% else %}production{% endif %}" + Name: "{{ vm_name }}" + arch: "{{ arch }}" + aws_access_key: "{{ aws_access_key }}" + aws_secret_key: "{{ aws_secret_key }}" + volumes: "{% if instance_volumes is defined %}{{ instance_volumes }}{% else %}[]{% endif %}" + spot_price: "{{ spot_price }}" + register: ec2 + +- name: Add new instance to host group + add_host: + hostname: "{{ item.public_ip }}" + groupname: builder_temp_group + loop: "{{ ec2.instances }}" + +- set_fact: builder_ip={{ ec2.instances[0].public_ip }} + +- debug: msg="VM_IP={{ builder_ip }}" + +- name: wait for he host to be hot + local_action: wait_for host={{ builder_ip }} port=22 delay=1 timeout={{ max_spawn_time }} + +- name: wait until ssh is available + local_action: shell false; until [ "$?" -eq "0" ]; do sleep 2; ssh -o PasswordAuthentication=no {{ spawning_vm_user|default('fedora') }}@{{ builder_ip }} 'echo foobar' 2>/dev/null; done + async: 600 + poll: 2 diff --git a/roles/copr/backend/templates/resalloc/vm-aws-new b/roles/copr/backend/templates/resalloc/vm-aws-new index 3777a1a4f0..d8a1e88be5 100755 --- a/roles/copr/backend/templates/resalloc/vm-aws-new +++ b/roles/copr/backend/templates/resalloc/vm-aws-new @@ -15,6 +15,7 @@ Allocate AWS VM instance for Copr build system. Options: --arch native architecture (required) -h, --help show this help + --spot start a spot instance Environment variables: \$RESALLOC_NAME the name given to the allocated instance, will be @@ -31,11 +32,12 @@ test $# -eq 0 && show_help 1 test -n "$RESALLOC_NAME" || show_help 1 -ARGS=$(getopt -o "h" -l "arch:,help" -n "getopt" -- "$@") \ +ARGS=$(getopt -o "h" -l "arch:,spot,help" -n "getopt" -- "$@") \ || show_help 1 eval set -- "$ARGS" opt_arch= +opt_spot=false while true; do # now the name is in $1 and argument in $2 @@ -47,6 +49,13 @@ while true; do eval "opt_$opt=\$2" shift 2 ;; + --spot) + opt=${1##--} + opt=${opt##-} + opt=${opt//-/_} + eval "opt_$opt=:" + shift + ;; -h|--help) show_help 0;; --) # end! shift; break;; @@ -72,10 +81,15 @@ cleanup_actions() } trap cleanup_actions EXIT +spot_pb_part= +if $opt_spot; then + spot_pb_part=-spot +fi + # TODO: We should call aws-cli directly here, instead of parsing output of # ansible playbook. But at the time of writing this script we had the playbook # available, so parsing the output is the easiest way to start. -playbook=/var/lib/resallocserver/provision/builderpb-aws-"$opt_arch".yml +playbook=/var/lib/resallocserver/provision/builderpb-aws"$spot_pb_part"-"$opt_arch".yml { vm_ip=$(ansible-playbook "$playbook" \ From ee0a88df3321515bde72859944f3cae00323b911 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Wed, 19 May 2021 13:59:41 +0200 Subject: [PATCH 044/189] copr-be: terminate spot instances on shutdown Per the following error: fatal: [127.0.0.1]: FAILED! => {"changed": false, "msg": "instance_initiated_shutdown_behavior=stop is not supported for spot instances."} --- roles/copr/backend/files/provision/spinup_aws_spot_task.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/copr/backend/files/provision/spinup_aws_spot_task.yml b/roles/copr/backend/files/provision/spinup_aws_spot_task.yml index c1b94c63bb..18b0fdb993 100644 --- a/roles/copr/backend/files/provision/spinup_aws_spot_task.yml +++ b/roles/copr/backend/files/provision/spinup_aws_spot_task.yml @@ -26,6 +26,7 @@ aws_secret_key: "{{ aws_secret_key }}" volumes: "{% if instance_volumes is defined %}{{ instance_volumes }}{% else %}[]{% endif %}" spot_price: "{{ spot_price }}" + instance_initiated_shutdown_behavior: terminate register: ec2 - name: Add new instance to host group From 0295b35a21295d5f739d7e97714e596d0c60e9b6 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Wed, 19 May 2021 14:19:07 +0200 Subject: [PATCH 045/189] copr-dev: configure spot instances And also add a configuration options for the s390x arch quota. --- inventory/group_vars/copr_aws | 6 ++++ inventory/group_vars/copr_dev_aws | 10 +++++-- .../backend/templates/resalloc/pools.yaml | 29 +++++++++++++------ 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/inventory/group_vars/copr_aws b/inventory/group_vars/copr_aws index fed831617c..206d284752 100644 --- a/inventory/group_vars/copr_aws +++ b/inventory/group_vars/copr_aws @@ -30,6 +30,12 @@ builders: aws: x86_64: [100,15,15] armhfp: [20,3,3] + s390x: [5,1,1] aarch64: [20,6,6] + aws_spot: + x86_64: [0,1,1] + armhfp: [0,0,0] + s390x: [0,0,0] + aarch64: [0,2,2] rpm_vendor_copr_name: Fedora Copr diff --git a/inventory/group_vars/copr_dev_aws b/inventory/group_vars/copr_dev_aws index 156a7fdc4c..a5b2ebd7bf 100644 --- a/inventory/group_vars/copr_dev_aws +++ b/inventory/group_vars/copr_dev_aws @@ -29,8 +29,14 @@ nm_controlled_resolv: True builders: # max|max_spawn|max_prealloc aws: - x86_64: [10,1,1] - armhfp: [3,1,1] + x86_64: [4,1,1] + armhfp: [2,0,0] + s390x: [2,0,0] + aarch64: [2,1,1] + aws_spot: + x86_64: [5,2,3] + armhfp: [2,0,0] + s390x: [1,0,0] aarch64: [5,2,2] rpm_vendor_copr_name: Fedora Copr (devel) diff --git a/roles/copr/backend/templates/resalloc/pools.yaml b/roles/copr/backend/templates/resalloc/pools.yaml index c3e7b0624b..3ad1b4bd0c 100644 --- a/roles/copr/backend/templates/resalloc/pools.yaml +++ b/roles/copr/backend/templates/resalloc/pools.yaml @@ -1,5 +1,5 @@ -{% macro aws_x86_64(max, max_starting, max_prealloc) %} -aws_x86_64_normal_{% if devel %}dev{% else %}prod{% endif %}: +{% macro aws_x86_64(max, max_starting, max_prealloc, spot=False) %} +aws_x86_64_{% if spot %}spot{% else %}normal{% endif %}_{% if devel %}dev{% else %}prod{% endif %}: max: {{ max }} max_starting: {{ max_starting }} max_prealloc: {{ max_prealloc }} @@ -18,7 +18,7 @@ aws_x86_64_normal_{% if devel %}dev{% else %}prod{% endif %}: - arch_s390x - arch_s390x_emulated - aws - cmd_new: "/var/lib/resallocserver/resalloc_provision/vm-aws-new --arch=x86_64" + cmd_new: "/var/lib/resallocserver/resalloc_provision/vm-aws-new --arch=x86_64 {% if spot %}--spot{% endif %}" cmd_delete: "/var/lib/resallocserver/resalloc_provision/vm-aws-delete" cmd_livecheck: "/var/lib/resallocserver/resalloc_provision/vm-aws-check" livecheck_period: 180 @@ -27,8 +27,8 @@ aws_x86_64_normal_{% if devel %}dev{% else %}prod{% endif %}: reuse_max_time: 1800 {% endmacro %} -{% macro aws_aarch64(max, max_starting, max_prealloc) %} -aws_aarch64_normal_{% if devel %}dev{% else %}prod{% endif %}: +{% macro aws_aarch64(max, max_starting, max_prealloc, spot=False) %} +aws_aarch64_{% if spot %}spot{% else %}normal{% endif %}_{% if devel %}dev{% else %}prod{% endif %}: max: {{ max }} max_starting: {{ max_starting }} max_prealloc: {{ max_prealloc }} @@ -37,7 +37,7 @@ aws_aarch64_normal_{% if devel %}dev{% else %}prod{% endif %}: - arch_aarch64 - arch_aarch64_native - aws - cmd_new: "/var/lib/resallocserver/resalloc_provision/vm-aws-new --arch=aarch64" + cmd_new: "/var/lib/resallocserver/resalloc_provision/vm-aws-new --arch=aarch64 {% if spot %}--spot{% endif %}" cmd_delete: "/var/lib/resallocserver/resalloc_provision/vm-aws-delete" cmd_livecheck: "/var/lib/resallocserver/resalloc_provision/vm-aws-check" livecheck_period: 180 @@ -109,15 +109,26 @@ aarch64_{{ id }}_{{ inst }}: - arch_aarch64_native {% endmacro %} -{{ aws_x86_64(builders.aws.x86_64[0] + builders.aws.armhfp[0], - builders.aws.x86_64[1] + builders.aws.armhfp[1], - builders.aws.x86_64[2] + builders.aws.armhfp[2]) +{{ aws_x86_64(builders.aws.x86_64[0] + builders.aws.armhfp[0] + builders.aws.s390x[0], + builders.aws.x86_64[1] + builders.aws.armhfp[1] + builders.aws.s390x[1], + builders.aws.x86_64[2] + builders.aws.armhfp[2] + builders.aws.s390x[2]) }} {{ aws_aarch64(builders.aws.aarch64[0], builders.aws.aarch64[1], builders.aws.aarch64[2]) }} +{{ aws_x86_64(builders.aws_spot.x86_64[0] + builders.aws_spot.armhfp[0] + builders.aws_spot.s390x[0], + builders.aws_spot.x86_64[1] + builders.aws_spot.armhfp[1] + builders.aws_spot.s390x[1], + builders.aws_spot.x86_64[2] + builders.aws_spot.armhfp[2] + builders.aws_spot.s390x[2], + True) +}} +{{ aws_aarch64(builders.aws_spot.aarch64[0], + builders.aws_spot.aarch64[1], + builders.aws_spot.aarch64[2], + True) +}} + {% if env == "production" %} {{ hw_aarch64("01", "prod", 4, 2, 4) }} {{ hw_aarch64("02", "prod", 4, 2, 4) }} From cc5436f6b04c1d888022a9ae424091b26b8ffb56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Wed, 19 May 2021 15:50:43 +0200 Subject: [PATCH 046/189] Deploy the new bugzilla2fedmsg to openshift in prod MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Aurélien Bompard --- playbooks/openshift-apps/bugzilla2fedmsg.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/openshift-apps/bugzilla2fedmsg.yml b/playbooks/openshift-apps/bugzilla2fedmsg.yml index 2ce98ebcee..34161512fd 100644 --- a/playbooks/openshift-apps/bugzilla2fedmsg.yml +++ b/playbooks/openshift-apps/bugzilla2fedmsg.yml @@ -1,5 +1,5 @@ - name: make the app be real - hosts: os_masters_stg[0] + hosts: os_masters[0]:os_masters_stg[0] user: root gather_facts: False From 0b6408df272fabc0582d39d7ede13d6dc963f909 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Wed, 19 May 2021 16:37:55 +0200 Subject: [PATCH 047/189] copr-be: implement livecheck for the hypervisor VMs, too --- roles/copr/backend/tasks/resalloc.yml | 2 +- roles/copr/backend/templates/resalloc/pools.yaml | 6 +++--- .../backend/templates/resalloc/{vm-aws-check => vm-check} | 0 3 files changed, 4 insertions(+), 4 deletions(-) rename roles/copr/backend/templates/resalloc/{vm-aws-check => vm-check} (100%) diff --git a/roles/copr/backend/tasks/resalloc.yml b/roles/copr/backend/tasks/resalloc.yml index d5e2fa63cc..1a830ea0fa 100644 --- a/roles/copr/backend/tasks/resalloc.yml +++ b/roles/copr/backend/tasks/resalloc.yml @@ -61,7 +61,7 @@ - vm-aarch64-new - vm-aws-new - vm-aws-delete - - vm-aws-check + - vm-check tags: - provision_config diff --git a/roles/copr/backend/templates/resalloc/pools.yaml b/roles/copr/backend/templates/resalloc/pools.yaml index 3ad1b4bd0c..f45f966655 100644 --- a/roles/copr/backend/templates/resalloc/pools.yaml +++ b/roles/copr/backend/templates/resalloc/pools.yaml @@ -20,7 +20,7 @@ aws_x86_64_{% if spot %}spot{% else %}normal{% endif %}_{% if devel %}dev{% else - aws cmd_new: "/var/lib/resallocserver/resalloc_provision/vm-aws-new --arch=x86_64 {% if spot %}--spot{% endif %}" cmd_delete: "/var/lib/resallocserver/resalloc_provision/vm-aws-delete" - cmd_livecheck: "/var/lib/resallocserver/resalloc_provision/vm-aws-check" + cmd_livecheck: "/var/lib/resallocserver/resalloc_provision/vm-check" livecheck_period: 180 reuse_opportunity_time: 180 reuse_max_count: 8 @@ -39,7 +39,7 @@ aws_aarch64_{% if spot %}spot{% else %}normal{% endif %}_{% if devel %}dev{% els - aws cmd_new: "/var/lib/resallocserver/resalloc_provision/vm-aws-new --arch=aarch64 {% if spot %}--spot{% endif %}" cmd_delete: "/var/lib/resallocserver/resalloc_provision/vm-aws-delete" - cmd_livecheck: "/var/lib/resallocserver/resalloc_provision/vm-aws-check" + cmd_livecheck: "/var/lib/resallocserver/resalloc_provision/vm-check" livecheck_period: 180 reuse_opportunity_time: 180 reuse_max_count: 8 @@ -77,7 +77,7 @@ copr_hv_x86_64_{{ hv }}_{% if devel %}dev{% else %}prod{% endif %}: cmd_new: "/var/lib/resallocserver/provision/libvirt-new --swap-vol-size 168" cmd_delete: "/var/lib/resallocserver/provision/libvirt-new --swap-vol-size 168" cmd_delete: "/var/lib/resallocserver/provision/libvirt-delete" - cmd_livecheck: "echo TODO" + cmd_livecheck: "/var/lib/resallocserver/resalloc_provision/vm-check" livecheck_period: 180 reuse_opportunity_time: 180 reuse_max_count: 8 diff --git a/roles/copr/backend/templates/resalloc/vm-aws-check b/roles/copr/backend/templates/resalloc/vm-check similarity index 100% rename from roles/copr/backend/templates/resalloc/vm-aws-check rename to roles/copr/backend/templates/resalloc/vm-check From c95c641f1ea4545595f26809c8792f4adb2423c2 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Wed, 19 May 2021 16:46:17 +0200 Subject: [PATCH 048/189] copr-builder: run cleanup script after each builder release Under root, so we can clean-up everything needed: Fixes: https://pagure.io/copr/copr/issue/1258 --- roles/copr/backend/tasks/resalloc.yml | 1 + roles/copr/backend/templates/resalloc/pools.yaml | 3 +++ roles/copr/backend/templates/resalloc/vm-release | 13 +++++++++++++ 3 files changed, 17 insertions(+) create mode 100755 roles/copr/backend/templates/resalloc/vm-release diff --git a/roles/copr/backend/tasks/resalloc.yml b/roles/copr/backend/tasks/resalloc.yml index 1a830ea0fa..6cc11910ab 100644 --- a/roles/copr/backend/tasks/resalloc.yml +++ b/roles/copr/backend/tasks/resalloc.yml @@ -62,6 +62,7 @@ - vm-aws-new - vm-aws-delete - vm-check + - vm-release tags: - provision_config diff --git a/roles/copr/backend/templates/resalloc/pools.yaml b/roles/copr/backend/templates/resalloc/pools.yaml index f45f966655..ea0d4986cd 100644 --- a/roles/copr/backend/templates/resalloc/pools.yaml +++ b/roles/copr/backend/templates/resalloc/pools.yaml @@ -21,6 +21,7 @@ aws_x86_64_{% if spot %}spot{% else %}normal{% endif %}_{% if devel %}dev{% else cmd_new: "/var/lib/resallocserver/resalloc_provision/vm-aws-new --arch=x86_64 {% if spot %}--spot{% endif %}" cmd_delete: "/var/lib/resallocserver/resalloc_provision/vm-aws-delete" cmd_livecheck: "/var/lib/resallocserver/resalloc_provision/vm-check" + cmd_release: "/var/lib/resallocserver/resalloc_provision/vm-release" livecheck_period: 180 reuse_opportunity_time: 180 reuse_max_count: 8 @@ -40,6 +41,7 @@ aws_aarch64_{% if spot %}spot{% else %}normal{% endif %}_{% if devel %}dev{% els cmd_new: "/var/lib/resallocserver/resalloc_provision/vm-aws-new --arch=aarch64 {% if spot %}--spot{% endif %}" cmd_delete: "/var/lib/resallocserver/resalloc_provision/vm-aws-delete" cmd_livecheck: "/var/lib/resallocserver/resalloc_provision/vm-check" + cmd_release: "/var/lib/resallocserver/resalloc_provision/vm-release" livecheck_period: 180 reuse_opportunity_time: 180 reuse_max_count: 8 @@ -78,6 +80,7 @@ copr_hv_x86_64_{{ hv }}_{% if devel %}dev{% else %}prod{% endif %}: cmd_delete: "/var/lib/resallocserver/provision/libvirt-new --swap-vol-size 168" cmd_delete: "/var/lib/resallocserver/provision/libvirt-delete" cmd_livecheck: "/var/lib/resallocserver/resalloc_provision/vm-check" + cmd_release: "/var/lib/resallocserver/resalloc_provision/vm-release" livecheck_period: 180 reuse_opportunity_time: 180 reuse_max_count: 8 diff --git a/roles/copr/backend/templates/resalloc/vm-release b/roles/copr/backend/templates/resalloc/vm-release new file mode 100755 index 0000000000..9772ec1bf1 --- /dev/null +++ b/roles/copr/backend/templates/resalloc/vm-release @@ -0,0 +1,13 @@ +#! /bin/sh + +die() { echo "$*" >&2 ; exit 1; } + +set -x +set -e +test -n "$RESALLOC_NAME" +test -n "$RESALLOC_RESOURCE_DATA" + +# we only put IP out in spawning script, nothing else +set -- $(echo "$RESALLOC_RESOURCE_DATA" | base64 --decode) +IP=$1 +ssh -o ConnectTimeout=10 "root@$IP" copr-builder-cleanup From 33813598766481f91ae282ef87c74ae956c46aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Wed, 19 May 2021 17:33:10 +0200 Subject: [PATCH 049/189] Disable the VM version of bugzilla2fedmsg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Aurélien Bompard --- roles/bugzilla2fedmsg/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/bugzilla2fedmsg/tasks/main.yml b/roles/bugzilla2fedmsg/tasks/main.yml index 49afc994b2..bd9b3da9b8 100644 --- a/roles/bugzilla2fedmsg/tasks/main.yml +++ b/roles/bugzilla2fedmsg/tasks/main.yml @@ -60,6 +60,6 @@ - bugzilla2fedmsg - name: moksha-hub service - service: name=moksha-hub state=started enabled=yes + service: name=moksha-hub state=stopped enabled=no tags: - bugzilla2fedmsg From 45cf6f92d1805fda6fcb9223c39642ae556d74cc Mon Sep 17 00:00:00 2001 From: Adam Saleh Date: Wed, 19 May 2021 18:21:59 +0200 Subject: [PATCH 050/189] Toddlers - add asaleh as owner and remove ref to staging, as it was never deployed there. --- playbooks/openshift-apps/toddlers.yml | 1 + roles/openshift-apps/toddlers/templates/buildconfig.yml | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/playbooks/openshift-apps/toddlers.yml b/playbooks/openshift-apps/toddlers.yml index 2e22f2e642..06d51a66ef 100644 --- a/playbooks/openshift-apps/toddlers.yml +++ b/playbooks/openshift-apps/toddlers.yml @@ -18,6 +18,7 @@ - mohanboddu - zlopez - scoady + - asaleh tags: - appowners diff --git a/roles/openshift-apps/toddlers/templates/buildconfig.yml b/roles/openshift-apps/toddlers/templates/buildconfig.yml index be9d5bf1c4..5465386a18 100644 --- a/roles/openshift-apps/toddlers/templates/buildconfig.yml +++ b/roles/openshift-apps/toddlers/templates/buildconfig.yml @@ -13,11 +13,7 @@ spec: type: Git git: uri: https://pagure.io/fedora-infra/toddlers.git -{% if env == 'staging' %} - ref: "staging" -{% else %} ref: "production" -{% endif %} strategy: type: Docker triggers: From d890a9fbf4662fac4b26da3522484cae49d8869a Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 19 May 2021 12:00:49 -0700 Subject: [PATCH 051/189] bugzilla2fedmsg: drop checks against vm as it has moved to openshift Signed-off-by: Kevin Fenzi --- .../templates/check_fedmsg_consumers.cfg.j2 | 2 -- .../nagios/services/iad2_internal/fedmsg.cfg | 36 +++++++++---------- .../nagios_server/templates/nrpe/nrpe.cfg.j2 | 3 -- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/roles/nagios_client/templates/check_fedmsg_consumers.cfg.j2 b/roles/nagios_client/templates/check_fedmsg_consumers.cfg.j2 index cc709c26a8..27984103d2 100644 --- a/roles/nagios_client/templates/check_fedmsg_consumers.cfg.j2 +++ b/roles/nagios_client/templates/check_fedmsg_consumers.cfg.j2 @@ -11,7 +11,6 @@ command[check_fedmsg_cp_app]={{libdir}}/nagios/plugins/check_fedmsg_producers_co command[check_fedmsg_cp_value]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-irc IRCBotConsumer MonitoringProducer command[check_fedmsg_cp_badges_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FedoraBadgesConsumer MonitoringProducer command[check_fedmsg_cp_notifs_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FMNConsumer DigestProducer ConfirmationProducer MonitoringProducer -command[check_fedmsg_cp_bugzilla2fedmsg]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py moksha-hub BugzillaConsumer MonitoringProducer command[check_fedmsg_cp_fedimg_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FedimgConsumer MonitoringProducer command[check_fedmsg_cp_hotness_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub BugzillaTicketFiler MonitoringProducer command[check_fedmsg_cp_packages_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub CacheInvalidator MonitoringProducer @@ -29,7 +28,6 @@ command[check_fedmsg_cexceptions_app]={{libdir}}/nagios/plugins/check_fedmsg_con command[check_fedmsg_cexceptions_value]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-irc IRCBotConsumer 1 10 command[check_fedmsg_cexceptions_badges_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub FedoraBadgesConsumer 1 10 command[check_fedmsg_cexceptions_notifs_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub FMNConsumer 1 10 -command[check_fedmsg_cexceptions_bugzilla2fedmsg]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py moksha-hub BugzillaConsumer 1 10 command[check_fedmsg_cexceptions_fedimg_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub FedimgConsumer 1 10 command[check_fedmsg_cexceptions_hotness_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub BugzillaTicketFiler 1 10 command[check_fedmsg_cexceptions_packages_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub CacheInvalidator 1 10 diff --git a/roles/nagios_server/files/nagios/services/iad2_internal/fedmsg.cfg b/roles/nagios_server/files/nagios/services/iad2_internal/fedmsg.cfg index 6f8a2b2844..b557fa90dc 100644 --- a/roles/nagios_server/files/nagios/services/iad2_internal/fedmsg.cfg +++ b/roles/nagios_server/files/nagios/services/iad2_internal/fedmsg.cfg @@ -278,12 +278,12 @@ define service { use defaulttemplate } -define service { - host_name bugzilla2fedmsg01.iad2.fedoraproject.org - service_description Check fedmsg consumers and producers hub - check_command check_by_nrpe!check_fedmsg_cp_bugzilla2fedmsg - use defaulttemplate -} +#define service { +# host_name bugzilla2fedmsg01.iad2.fedoraproject.org +# service_description Check fedmsg consumers and producers hub +# check_command check_by_nrpe!check_fedmsg_cp_bugzilla2fedmsg +# use defaulttemplate +#} define service { host_name fedimg01.iad2.fedoraproject.org @@ -351,12 +351,12 @@ define service { use defaulttemplate } -define service { - host_name bugzilla2fedmsg01.iad2.fedoraproject.org - service_description Check fedmsg-hub consumers exceptions - check_command check_by_nrpe!check_fedmsg_cexceptions_bugzilla2fedmsg - use defaulttemplate -} +#define service { +# host_name bugzilla2fedmsg01.iad2.fedoraproject.org +# service_description Check fedmsg-hub consumers exceptions +# check_command check_by_nrpe!check_fedmsg_cexceptions_bugzilla2fedmsg +# use defaulttemplate +#} define service { host_name fedimg01.iad2.fedoraproject.org @@ -425,12 +425,12 @@ define service { use defaulttemplate } -define service { - host_name bugzilla2fedmsg01.iad2.fedoraproject.org - service_description Check fedmsg-hub consumers backlog - check_command check_by_nrpe!check_fedmsg_cbacklog_bugzilla2fedmsg - use defaulttemplate -} +#define service { +# host_name bugzilla2fedmsg01.iad2.fedoraproject.org +# service_description Check fedmsg-hub consumers backlog +# check_command check_by_nrpe!check_fedmsg_cbacklog_bugzilla2fedmsg +# use defaulttemplate +#} define service { host_name fedimg01.iad2.fedoraproject.org diff --git a/roles/nagios_server/templates/nrpe/nrpe.cfg.j2 b/roles/nagios_server/templates/nrpe/nrpe.cfg.j2 index e31505bf33..884707d8ac 100644 --- a/roles/nagios_server/templates/nrpe/nrpe.cfg.j2 +++ b/roles/nagios_server/templates/nrpe/nrpe.cfg.j2 @@ -387,7 +387,6 @@ command[check_fedmsg_cp_app]=/usr/lib64/nagios/plugins/check_fedmsg_producers_co command[check_fedmsg_cp_value]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-irc IRCBotConsumer MonitoringProducer command[check_fedmsg_cp_badges_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FedoraBadgesConsumer MonitoringProducer command[check_fedmsg_cp_notifs_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FMNConsumer DigestProducer ConfirmationProducer MonitoringProducer -command[check_fedmsg_cp_bugzilla2fedmsg]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py moksha-hub BugzillaConsumer MonitoringProducer command[check_fedmsg_cp_fedimg_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FedimgConsumer MonitoringProducer command[check_fedmsg_cp_hotness_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub BugzillaTicketFiler MonitoringProducer command[check_fedmsg_cp_packages_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub CacheInvalidator MonitoringProducer @@ -400,7 +399,6 @@ command[check_fedmsg_cexceptions_app]=/usr/lib64/nagios/plugins/check_fedmsg_con command[check_fedmsg_cexceptions_value]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-irc IRCBotConsumer 1 10 command[check_fedmsg_cexceptions_badges_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub FedoraBadgesConsumer 1 10 command[check_fedmsg_cexceptions_notifs_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub FMNConsumer 1 10 -command[check_fedmsg_cexceptions_bugzilla2fedmsg]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py moksha-hub BugzillaConsumer 1 10 command[check_fedmsg_cexceptions_fedimg_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub FedimgConsumer 1 10 command[check_fedmsg_cexceptions_hotness_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub BugzillaTicketFiler 1 10 command[check_fedmsg_cexceptions_packages_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub CacheInvalidator 1 10 @@ -413,7 +411,6 @@ command[check_fedmsg_cbacklog_app]=/usr/lib64/nagios/plugins/check_fedmsg_consum command[check_fedmsg_cbacklog_value]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-irc IRCBotConsumer 10 50 command[check_fedmsg_cbacklog_badges_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub FedoraBadgesConsumer 25000 35000 command[check_fedmsg_cbacklog_notifs_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub FMNConsumer 10 50 -command[check_fedmsg_cbacklog_bugzilla2fedmsg]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py moksha-hub BugzillaConsumer 10 100 command[check_fedmsg_cbacklog_fedimg_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub FedimgConsumer 2000 5000 command[check_fedmsg_cbacklog_hotness_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub BugzillaTicketFiler 100 500 command[check_fedmsg_cbacklog_packages_backend_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub CacheInvalidator 30000 40000 From 535041fcddd253757d923650f07e541beb63cda3 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 19 May 2021 12:06:51 -0700 Subject: [PATCH 052/189] robosignatory: add kinoite signing for rawhide Signed-off-by: Kevin Fenzi --- roles/robosignatory/templates/robosignatory.toml.j2 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/roles/robosignatory/templates/robosignatory.toml.j2 b/roles/robosignatory/templates/robosignatory.toml.j2 index e0c2090d11..90c85ce057 100644 --- a/roles/robosignatory/templates/robosignatory.toml.j2 +++ b/roles/robosignatory/templates/robosignatory.toml.j2 @@ -524,6 +524,15 @@ handlers = ["console"] directory = "/mnt/fedora_koji/koji/compose/ostree/repo/" key = "{{ (env == 'production')|ternary('fedora-35', 'testkey') }}" + [consumer_config.ostree_refs."fedora/rawhide/aarch64/kinoite"] + directory = "/mnt/fedora_koji/koji/compose/ostree/repo/" + key = "{{ (env == 'production')|ternary('fedora-35', 'testkey') }}" + [consumer_config.ostree_refs."fedora/rawhide/ppc64le/kinoite"] + directory = "/mnt/fedora_koji/koji/compose/ostree/repo/" + key = "{{ (env == 'production')|ternary('fedora-35', 'testkey') }}" + [consumer_config.ostree_refs."fedora/rawhide/x86_64/kinoite"] + directory = "/mnt/fedora_koji/koji/compose/ostree/repo/" + key = "{{ (env == 'production')|ternary('fedora-35', 'testkey') }}" [consumer_config.coreos] bucket = "fcos-builds" From c9ee450c6a56cb0fa900ae5980d1d3d37dc960e5 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Wed, 19 May 2021 15:31:39 -0700 Subject: [PATCH 053/189] greenwave: specify exact versions for openQA gating policy I was assuming Bodhi queried on "fedora-rawhide" for Rawhide updates, but it seems it does not, it queries on "fedora-35" (or whatever the release number is). We do not want to apply the gating to Rawhide updates ATM, as openQA does not test them. So unfortunately we'll need to hand-tend this list unless I can think of a better approach. Signed-off-by: Adam Williamson --- .../greenwave/templates/fedora.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/roles/openshift-apps/greenwave/templates/fedora.yaml b/roles/openshift-apps/greenwave/templates/fedora.yaml index 6cf0f9a8bc..1677551890 100644 --- a/roles/openshift-apps/greenwave/templates/fedora.yaml +++ b/roles/openshift-apps/greenwave/templates/fedora.yaml @@ -79,18 +79,18 @@ subject_type: koji_build rules: - !RemoteRule {} -# For critical path updates, we require passes for all openQA update tests +# For critical path updates in stable and Branched, we require passes +# for all openQA update tests --- !Policy id: "bodhiupdate_bodhipush_openqa" product_versions: - # this should cover us for a while... - - fedora-3* - - fedora-4* - - fedora-5* - - fedora-6* - - fedora-7* - - fedora-8* - - fedora-9* + # FIXME: we would use some kind of matching here, but we don't want + # to include whatever version Rawhide is, as openQA tests don't run + # on Rawhide. So for now this list needs to be updated each time a + # release is branched (or goes EOL). + - fedora-32 + - fedora-33 + - fedora-34 decision_contexts: - bodhi_update_push_testing_critpath - bodhi_update_push_stable_critpath From 7185d7774e4d740b3a284810b23ea87d5c15e793 Mon Sep 17 00:00:00 2001 From: "Frank Ch. Eigler" Date: Tue, 18 May 2021 11:27:45 -0400 Subject: [PATCH 054/189] debuginfod redirect+proxy: combine into handcrafted httpd conf file --- playbooks/include/proxies-redirects.yml | 10 ---------- .../templates/reversepassproxy.debuginfod.conf | 14 ++++++++------ 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/playbooks/include/proxies-redirects.yml b/playbooks/include/proxies-redirects.yml index bb2433ccfc..3cb32aa888 100644 --- a/playbooks/include/proxies-redirects.yml +++ b/playbooks/include/proxies-redirects.yml @@ -872,13 +872,3 @@ website: qa.fedoraproject.org path: / target: https://fedoraproject.org/wiki/QA - - # redirects for debuginfod main page; subdirs are reverse-proxied - - role: httpd/redirect - shortname: debuginfod - website: debuginfod.fedoraproject.org - regex: ^/$ - target: https://fedoraproject.org/wiki/Debuginfod - tags: - - debuginfod - diff --git a/roles/httpd/reverseproxy/templates/reversepassproxy.debuginfod.conf b/roles/httpd/reverseproxy/templates/reversepassproxy.debuginfod.conf index 8093cd40bc..0300d06b29 100644 --- a/roles/httpd/reverseproxy/templates/reversepassproxy.debuginfod.conf +++ b/roles/httpd/reverseproxy/templates/reversepassproxy.debuginfod.conf @@ -1,15 +1,17 @@ -# Broadly reverse-proxy -ProxyPassMatch "^/(.*)$" "{{proxyurl}}/$1" {{proxyopts}} +# Reverse-proxy /buildid/* queries +ProxyPassMatch "^/buildid/(.*)$" "{{proxyurl}}/buildid/$1" {{proxyopts}} + +# Reverse-proxy /metrics - without (...) grouping op, URL just gets copied +ProxyPassMatch "^/metrics$" "{{proxyurl}}" {{proxyopts}} + ProxyPassReverse / {{proxyurl}}/ # ... but not the top level, which is instead redirected ProxyPass / ! -# ... configure this redirect in playbooks/include/proxies-redirects.yml -# Redirect permanent / https://fedoraproject.org/wiki/Debuginfod -# (or e.g.,) -# Redirect permanent / https://sourceware.org/elfutils/Debuginfod.html +# Redirect / URL over to wiki +Redirect permanent / https://fedoraproject.org/wiki/Debuginfod # test it with: # From 61be8367c8a201886fc14e3099736d21766525b6 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 19 May 2021 18:29:52 -0700 Subject: [PATCH 055/189] remove last vestiges of 2fa_client Signed-off-by: Kevin Fenzi --- playbooks/groups/copr-db.yml | 1 - playbooks/hosts/cloud-noc-os01.rdu-cc.fedoraproject.org.yml | 1 - playbooks/hosts/cloud-noc01.fedorainfracloud.org.yml | 1 - playbooks/hosts/data-analysis01.phx2.fedoraproject.org.yml | 1 - 4 files changed, 4 deletions(-) diff --git a/playbooks/groups/copr-db.yml b/playbooks/groups/copr-db.yml index 1f6c5a9ce9..a46b3bf161 100644 --- a/playbooks/groups/copr-db.yml +++ b/playbooks/groups/copr-db.yml @@ -35,7 +35,6 @@ - sudo tasks: - - import_tasks: "{{ tasks_path }}/2fa_client.yml" when: env != "staging" - import_tasks: "{{ tasks_path }}/motd.yml" diff --git a/playbooks/hosts/cloud-noc-os01.rdu-cc.fedoraproject.org.yml b/playbooks/hosts/cloud-noc-os01.rdu-cc.fedoraproject.org.yml index 47e23b5264..8dad7675d4 100644 --- a/playbooks/hosts/cloud-noc-os01.rdu-cc.fedoraproject.org.yml +++ b/playbooks/hosts/cloud-noc-os01.rdu-cc.fedoraproject.org.yml @@ -27,7 +27,6 @@ - import_tasks: "{{ tasks_path }}/yumrepos.yml" tasks: - - import_tasks: "{{ tasks_path }}/2fa_client.yml" - import_tasks: "{{ tasks_path }}/motd.yml" - name: install some packages which arent in playbooks diff --git a/playbooks/hosts/cloud-noc01.fedorainfracloud.org.yml b/playbooks/hosts/cloud-noc01.fedorainfracloud.org.yml index 40aa8cf3c6..96b0c4fd0f 100644 --- a/playbooks/hosts/cloud-noc01.fedorainfracloud.org.yml +++ b/playbooks/hosts/cloud-noc01.fedorainfracloud.org.yml @@ -27,7 +27,6 @@ - import_tasks: "{{ tasks_path }}/yumrepos.yml" tasks: - - import_tasks: "{{ tasks_path }}/2fa_client.yml" - import_tasks: "{{ tasks_path }}/motd.yml" - name: check if ntpd port is already known by selinux diff --git a/playbooks/hosts/data-analysis01.phx2.fedoraproject.org.yml b/playbooks/hosts/data-analysis01.phx2.fedoraproject.org.yml index 62e8e32781..70a4285c44 100644 --- a/playbooks/hosts/data-analysis01.phx2.fedoraproject.org.yml +++ b/playbooks/hosts/data-analysis01.phx2.fedoraproject.org.yml @@ -31,7 +31,6 @@ - web-data-analysis tasks: - - import_tasks: "{{ tasks_path }}/2fa_client.yml" - import_tasks: "{{ tasks_path }}/motd.yml" handlers: From a2f14e982e30754d3de744ee7e4dda728267595d Mon Sep 17 00:00:00 2001 From: Carl George Date: Tue, 4 May 2021 15:38:37 -0500 Subject: [PATCH 056/189] bodhi2/backend: Add epel8-next to releases in new-updates-sync Signed-off-by: Carl George --- roles/bodhi2/backend/files/new-updates-sync | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/roles/bodhi2/backend/files/new-updates-sync b/roles/bodhi2/backend/files/new-updates-sync index 472b2bb7ba..df3f570fac 100755 --- a/roles/bodhi2/backend/files/new-updates-sync +++ b/roles/bodhi2/backend/files/new-updates-sync @@ -172,6 +172,20 @@ RELEASES = {'f34': {'topic': 'fedora', 'dest': os.path.join(EPELDEST, '8', 'Everything')} ]}} }, + 'epel8n': {'topic': 'epel', + 'version': '8', + 'modules': ['epel'], + 'repos': {'epel-testing': { + 'from': 'epel8-next-testing', + 'to': [{'arches': ['x86_64', 'aarch64', 'ppc64le', 'source'], + 'dest': os.path.join(EPELDEST, 'testing', 'next', '8', 'Everything')} + ]}, + 'epel': { + 'from': 'epel8-next', + 'to': [{'arches': ['x86_64', 'aarch64', 'ppc64le', 'source'], + 'dest': os.path.join(EPELDEST, 'next', '8', 'Everything')} + ]}} + }, 'epel8m': {'topic': 'epel', 'version': '8m', 'modules': ['epel'], From c9f5c11ba61495edb85e1da3706364cd0efa6478 Mon Sep 17 00:00:00 2001 From: Brendan Early Date: Tue, 18 May 2021 19:17:40 -0500 Subject: [PATCH 057/189] packages-static: correct port --- playbooks/openshift-apps/fedora-packages-static.yml | 2 +- roles/openshift-apps/fedora-packages-static/files/service.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/playbooks/openshift-apps/fedora-packages-static.yml b/playbooks/openshift-apps/fedora-packages-static.yml index 227e5e2011..9646ca616c 100644 --- a/playbooks/openshift-apps/fedora-packages-static.yml +++ b/playbooks/openshift-apps/fedora-packages-static.yml @@ -49,7 +49,7 @@ app: fedora-packages-static routename: fedora-packages-static host: "packages{{ env_suffix }}.fedoraproject.org" - serviceport: 80-tcp + serviceport: 8080-tcp servicename: fedora-packages-static - role: openshift/object diff --git a/roles/openshift-apps/fedora-packages-static/files/service.yml b/roles/openshift-apps/fedora-packages-static/files/service.yml index 659e531014..ad30ccf27f 100644 --- a/roles/openshift-apps/fedora-packages-static/files/service.yml +++ b/roles/openshift-apps/fedora-packages-static/files/service.yml @@ -7,8 +7,8 @@ metadata: name: fedora-packages-static spec: ports: - - name: 80-tcp - port: 80 + - name: 8080-tcp + port: 8080 protocol: TCP targetPort: 8080 selector: From 6580fb89c06adc7b21fbc6b1de2005d55b72f436 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Thu, 20 May 2021 13:20:33 -0700 Subject: [PATCH 058/189] openQA staging: use new openqa scratch build too Signed-off-by: Adam Williamson --- inventory/group_vars/openqa_lab | 3 +++ inventory/group_vars/openqa_lab_workers | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/inventory/group_vars/openqa_lab b/inventory/group_vars/openqa_lab index 47f62a6c67..5c740b87e1 100644 --- a/inventory/group_vars/openqa_lab +++ b/inventory/group_vars/openqa_lab @@ -36,6 +36,9 @@ openqa_env: staging # ON THE EDGE (radical guitar riff) openqa_repo: updates-testing +# 2021-05 git bump +openqa_scratch: ["68350730"] + wikitcms_token: "{{ private }}/files/openidc/staging/wikitcms.json" openqa_wikitcms_hostname: stg.fedoraproject.org openqa_resultsdb_url: http://resultsdb01.stg.iad2.fedoraproject.org/resultsdb_api/api/v2.0/ diff --git a/inventory/group_vars/openqa_lab_workers b/inventory/group_vars/openqa_lab_workers index 23bf6a88ae..e6c0299de0 100644 --- a/inventory/group_vars/openqa_lab_workers +++ b/inventory/group_vars/openqa_lab_workers @@ -19,8 +19,8 @@ openqa_repo: updates-testing # we are all NFS workers for now at least openqa_nfs_worker: true -# 2021-05 git bump -openqa_scratch: ["67449532"] +# 2021-05 git bumps +openqa_scratch: ["67449532", "68350730"] deployment_type: stg freezes: false From 176f1fa3cf051d58e15c6f79ac7ea012a11676e3 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Thu, 20 May 2021 13:53:13 -0700 Subject: [PATCH 059/189] openQA lab: bump os-autoinst scratch build Signed-off-by: Adam Williamson --- inventory/group_vars/openqa_lab_workers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/openqa_lab_workers b/inventory/group_vars/openqa_lab_workers index e6c0299de0..1beea1ff19 100644 --- a/inventory/group_vars/openqa_lab_workers +++ b/inventory/group_vars/openqa_lab_workers @@ -20,7 +20,7 @@ openqa_repo: updates-testing openqa_nfs_worker: true # 2021-05 git bumps -openqa_scratch: ["67449532", "68350730"] +openqa_scratch: ["68353556", "68350730"] deployment_type: stg freezes: false From fafa47de963bfa17d39430572273359a7d9c9e75 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Fri, 21 May 2021 09:40:19 -0700 Subject: [PATCH 060/189] bodhi / backend: add url for repomd sync checking for epel8-next and epel8-next-testing Signed-off-by: Kevin Fenzi --- roles/bodhi2/base/templates/production.ini.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/bodhi2/base/templates/production.ini.j2 b/roles/bodhi2/base/templates/production.ini.j2 index 47535f00d1..76eca61a69 100644 --- a/roles/bodhi2/base/templates/production.ini.j2 +++ b/roles/bodhi2/base/templates/production.ini.j2 @@ -245,6 +245,8 @@ fedora_epel_modular_8_stable_master_repomd = http://dl-iad05.fedoraproject.org/p fedora_epel_modular_8_testing_master_repomd = http://dl-iad05.fedoraproject.org/pub/epel/testing/%%s/Modular/%%s/repodata/repomd.xml fedora_epel_stable_master_repomd = http://dl-iad05.fedoraproject.org/pub/epel/%%s/%%s/repodata/repomd.xml fedora_epel_testing_master_repomd = http://dl-iad05.fedoraproject.org/pub/epel/testing/%%s/%%s/repodata/repomd.xml +fedora_epel_next_8_master_repomd = http://dl-iad05.fedoraproject.org/pub/epel/next/%%s/Everything/%%s/repodata/repomd.xml +fedora_epel_next_8_testing_master_repomd = http://dl-iad05.fedoraproject.org/pub/epel/testing/next/%%s/Everything/%%s/repodata/repomd.xml fedora_modular_stable_master_repomd = http://dl-iad05.fedoraproject.org/pub/fedora/linux/updates/%%s/Modular/%%s/repodata/repomd.xml fedora_modular_testing_master_repomd = http://dl-iad05.fedoraproject.org/pub/fedora/linux/updates/testing/%%s/Modular/%%s/repodata/repomd.xml {% elif env == 'staging' %} From e3484c5f3d0a924ca0a600a4546b33959b6fa97c Mon Sep 17 00:00:00 2001 From: Brendan Early Date: Thu, 20 May 2021 19:00:48 -0500 Subject: [PATCH 061/189] packages-static: add proxy --- playbooks/include/proxies-reverseproxy.yml | 7 +++++++ playbooks/include/proxies-websites.yml | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/playbooks/include/proxies-reverseproxy.yml b/playbooks/include/proxies-reverseproxy.yml index 6f456a1465..18d70b0f4b 100644 --- a/playbooks/include/proxies-reverseproxy.yml +++ b/playbooks/include/proxies-reverseproxy.yml @@ -757,3 +757,10 @@ proxyopts: "connectiontimeout=600 timeout=600 keepalive=on" tags: - debuginfod + + - role: httpd/reverseproxy + website: packages{{ env_suffix }}.fedoraproject.org + destname: fedora-packages-static + balancer_name: app-os + targettype: openshift + keephost: true diff --git a/playbooks/include/proxies-websites.yml b/playbooks/include/proxies-websites.yml index 69b84346d9..40771b6f43 100644 --- a/playbooks/include/proxies-websites.yml +++ b/playbooks/include/proxies-websites.yml @@ -1055,3 +1055,9 @@ site_name: ols.fedoraproject.org sslonly: true cert_name: "{{wildcard_cert_name}}" + + - role: httpd/website + site_name: packages.fedoraproject.org + server_aliases: [packages.stg.fedoraproject.org] + sslonly: true + cert_name: "{{wildcard_cert_name}}" From 1dc01d036eea003fea59ebd8bbfb70e16e55f8ae Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Fri, 21 May 2021 11:11:35 -0700 Subject: [PATCH 062/189] proxies: packages site is the same in prod and stg with an alias Signed-off-by: Kevin Fenzi --- playbooks/include/proxies-reverseproxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/include/proxies-reverseproxy.yml b/playbooks/include/proxies-reverseproxy.yml index 18d70b0f4b..e0c4b60ab4 100644 --- a/playbooks/include/proxies-reverseproxy.yml +++ b/playbooks/include/proxies-reverseproxy.yml @@ -759,7 +759,7 @@ - debuginfod - role: httpd/reverseproxy - website: packages{{ env_suffix }}.fedoraproject.org + website: packages.fedoraproject.org destname: fedora-packages-static balancer_name: app-os targettype: openshift From 9006cf784e2dfc4724ab5e4885c2503ca7c468d6 Mon Sep 17 00:00:00 2001 From: Francois Andrieu Date: Fri, 21 May 2021 00:16:09 +0200 Subject: [PATCH 063/189] nagios: remove unused check_datanommer_faf --- .../templates/check_datanommer_history.cfg.j2 | 1 - .../files/nagios/services/iad2_internal/fedmsg.cfg | 7 ------- roles/nagios_server/templates/nrpe/nrpe.cfg.j2 | 1 - 3 files changed, 9 deletions(-) diff --git a/roles/nagios_client/templates/check_datanommer_history.cfg.j2 b/roles/nagios_client/templates/check_datanommer_history.cfg.j2 index aa58c83043..09077675ca 100644 --- a/roles/nagios_client/templates/check_datanommer_history.cfg.j2 +++ b/roles/nagios_client/templates/check_datanommer_history.cfg.j2 @@ -20,7 +20,6 @@ command[check_datanommer_bodhi_composes]={{libdir}}/nagios/plugins/check_datanom command[check_datanommer_buildsys]={{libdir}}/nagios/plugins/check_datanommer_timesince.py buildsys 14400 86400 command[check_datanommer_compose]={{libdir}}/nagios/plugins/check_datanommer_timesince.py compose 259200 1814400 command[check_datanommer_copr]={{libdir}}/nagios/plugins/check_datanommer_timesince.py copr 21600 86400 -command[check_datanommer_faf]={{libdir}}/nagios/plugins/check_datanommer_timesince.py faf 86400 259200 command[check_datanommer_fas]={{libdir}}/nagios/plugins/check_datanommer_timesince.py fas 1814400 2628000 command[check_datanommer_fedbadges]={{libdir}}/nagios/plugins/check_datanommer_timesince.py fedbadges 86400 259200 command[check_datanommer_fedimg]={{libdir}}/nagios/plugins/check_datanommer_timesince.py fedimg 259200 604800 diff --git a/roles/nagios_server/files/nagios/services/iad2_internal/fedmsg.cfg b/roles/nagios_server/files/nagios/services/iad2_internal/fedmsg.cfg index b557fa90dc..e548d18b47 100644 --- a/roles/nagios_server/files/nagios/services/iad2_internal/fedmsg.cfg +++ b/roles/nagios_server/files/nagios/services/iad2_internal/fedmsg.cfg @@ -177,13 +177,6 @@ define service { use defaulttemplate } -define service { - host_name busgateway01.iad2.fedoraproject.org - service_description Check datanommer for recent faf messages - check_command check_by_nrpe!check_datanommer_faf - use defaulttemplate -} - define service { host_name busgateway01.iad2.fedoraproject.org service_description Check datanommer for recent mailman messages diff --git a/roles/nagios_server/templates/nrpe/nrpe.cfg.j2 b/roles/nagios_server/templates/nrpe/nrpe.cfg.j2 index 884707d8ac..353f52c3d0 100644 --- a/roles/nagios_server/templates/nrpe/nrpe.cfg.j2 +++ b/roles/nagios_server/templates/nrpe/nrpe.cfg.j2 @@ -358,7 +358,6 @@ command[check_datanommer_bodhi_composes]=/usr/lib64/nagios/plugins/check_datanom command[check_datanommer_buildsys]=/usr/lib64/nagios/plugins/check_datanommer_timesince.py buildsys 14400 86400 command[check_datanommer_compose]=/usr/lib64/nagios/plugins/check_datanommer_timesince.py compose 259200 1814400 command[check_datanommer_copr]=/usr/lib64/nagios/plugins/check_datanommer_timesince.py copr 21600 86400 -command[check_datanommer_faf]=/usr/lib64/nagios/plugins/check_datanommer_timesince.py faf 86400 259200 command[check_datanommer_fas]=/usr/lib64/nagios/plugins/check_datanommer_timesince.py fas 1814400 2628000 command[check_datanommer_fedbadges]=/usr/lib64/nagios/plugins/check_datanommer_timesince.py fedbadges 86400 259200 command[check_datanommer_fedimg]=/usr/lib64/nagios/plugins/check_datanommer_timesince.py fedimg 259200 604800 From d9fc78b0e41ff815830ac224702f18dee00c7e77 Mon Sep 17 00:00:00 2001 From: Francois Andrieu Date: Fri, 21 May 2021 00:36:49 +0200 Subject: [PATCH 064/189] nagios: remove MBSProducer check from mbs-backend --- roles/nagios_client/templates/check_fedmsg_consumers.cfg.j2 | 2 +- roles/nagios_server/templates/nrpe/nrpe.cfg.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/nagios_client/templates/check_fedmsg_consumers.cfg.j2 b/roles/nagios_client/templates/check_fedmsg_consumers.cfg.j2 index 27984103d2..7c680e6f5f 100644 --- a/roles/nagios_client/templates/check_fedmsg_consumers.cfg.j2 +++ b/roles/nagios_client/templates/check_fedmsg_consumers.cfg.j2 @@ -14,7 +14,7 @@ command[check_fedmsg_cp_notifs_backend]={{libdir}}/nagios/plugins/check_fedmsg_p command[check_fedmsg_cp_fedimg_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FedimgConsumer MonitoringProducer command[check_fedmsg_cp_hotness_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub BugzillaTicketFiler MonitoringProducer command[check_fedmsg_cp_packages_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub CacheInvalidator MonitoringProducer -command[check_fedmsg_cp_mbs_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub MBSConsumer MBSProducer MonitoringProducer +command[check_fedmsg_cp_mbs_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub MBSConsumer MonitoringProducer command[check_fedmsg_cexceptions_busgateway_hub]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub Nommer 1 10 command[check_fedmsg_cexceptions_busgateway_relay]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-relay RelayConsumer 1 10 diff --git a/roles/nagios_server/templates/nrpe/nrpe.cfg.j2 b/roles/nagios_server/templates/nrpe/nrpe.cfg.j2 index 353f52c3d0..6ccdbb21d0 100644 --- a/roles/nagios_server/templates/nrpe/nrpe.cfg.j2 +++ b/roles/nagios_server/templates/nrpe/nrpe.cfg.j2 @@ -389,7 +389,7 @@ command[check_fedmsg_cp_notifs_backend]=/usr/lib64/nagios/plugins/check_fedmsg_p command[check_fedmsg_cp_fedimg_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FedimgConsumer MonitoringProducer command[check_fedmsg_cp_hotness_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub BugzillaTicketFiler MonitoringProducer command[check_fedmsg_cp_packages_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub CacheInvalidator MonitoringProducer -command[check_fedmsg_cp_mbs_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub MBSConsumer MBSProducer MonitoringProducer +command[check_fedmsg_cp_mbs_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub MBSConsumer MonitoringProducer command[check_fedmsg_cexceptions_busgateway_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub Nommer 1 10 command[check_fedmsg_cexceptions_busgateway_relay]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-relay RelayConsumer 1 10 From 24ae7d3d16891f3e64e9e6f45d94440fd0a0e231 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Fri, 21 May 2021 12:51:32 -0700 Subject: [PATCH 065/189] ipa / client: rework the excluded local users from sssd There's a real user 'mock' who we want to allow on ipsilon (so they can login to anything) and people02 (so they can get to their people space), but no where else, since we ened the local mock user on places like builders, etc. Signed-off-by: Kevin Fenzi --- roles/ipa/client/tasks/main.yml | 23 +------------------ .../fedora-nss-ignore.conf | 0 .../templates/fedora-nss-ignore.conf.j2 | 20 ++++++++++++++++ .../fedora-nss-ignore.conf.people | 0 .../fedora-nss-ignore.conf.staging | 0 5 files changed, 21 insertions(+), 22 deletions(-) rename roles/ipa/client/{files => templates}/fedora-nss-ignore.conf (100%) create mode 100644 roles/ipa/client/templates/fedora-nss-ignore.conf.j2 rename roles/ipa/client/{files => templates}/fedora-nss-ignore.conf.people (100%) rename roles/ipa/client/{files => templates}/fedora-nss-ignore.conf.staging (100%) diff --git a/roles/ipa/client/tasks/main.yml b/roles/ipa/client/tasks/main.yml index 3b9af8cec0..caf45b0f8a 100644 --- a/roles/ipa/client/tasks/main.yml +++ b/roles/ipa/client/tasks/main.yml @@ -72,31 +72,10 @@ run_once: yes - name: Ensure that nss knows to skip certain users - copy: src=fedora-nss-ignore.conf dest=/etc/sssd/conf.d/ mode=600 owner=root group=root + template: src=fedora-nss-ignore.conf.j2 dest=/etc/sssd/conf.d/ mode=600 owner=root group=root tags: - ipa/client - config notify: - restart sssd - clean sss caches - when: env == "production" and inventory_hostname != 'people02.fedoraproject.org' - -- name: Ensure that nss knows to skip certain users (on people02) - copy: src=fedora-nss-ignore.conf.people dest=/etc/sssd/conf.d/ mode=600 owner=root group=root - tags: - - ipa/client - - config - notify: - - restart sssd - - clean sss caches - when: env == "production" and inventory_hostname == 'people02.fedoraproject.org' - -- name: Ensure that nss knows to skip certain users - copy: src=fedora-nss-ignore.conf.staging dest=/etc/sssd/conf.d/fedora-nss-ignore.conf mode=600 owner=root group=root - tags: - - ipa/client - - config - notify: - - restart sssd - - clean sss caches - when: env == "staging" diff --git a/roles/ipa/client/files/fedora-nss-ignore.conf b/roles/ipa/client/templates/fedora-nss-ignore.conf similarity index 100% rename from roles/ipa/client/files/fedora-nss-ignore.conf rename to roles/ipa/client/templates/fedora-nss-ignore.conf diff --git a/roles/ipa/client/templates/fedora-nss-ignore.conf.j2 b/roles/ipa/client/templates/fedora-nss-ignore.conf.j2 new file mode 100644 index 0000000000..fda576fb2a --- /dev/null +++ b/roles/ipa/client/templates/fedora-nss-ignore.conf.j2 @@ -0,0 +1,20 @@ +{% comment %} +# There are a bunch of users we want to prefer the local version instead of the ipa/ldap version. +# But in at least one case, there is a real user ('mock') that we want to exclude on most machines, +# but not on people and ipsilon to allow them access to their people space and be able to login to things. +{% endcomment %} +{% if inventory_hostname in groups['people','ipsilon','ipsilon_stg'] %} +## This file contains users who are in ipa to stop people from +## creating restricted accounts but we want to make sure the id in +## /etc/passwd and /etc/group are used. +[nss] +filter_users = root,bin,daemon,adm,lp,sync,shutdown,halt,mail,operator,games,ftp,nobody,avahi-autoipd,dbus,polkitd,rpc,tss,ntp,rpcuser,nfsnobody,postfix,sshd,nagios,nrpe,openvpn,,chrony,sssd,named,apache +filter_groups = root,bin,daemon,sys,adm,tty,disk,lp,mem,kmem,wheel,cdrom,mail,man,dialout,floppy,games,tape,video,ftp,lock,audio,nobody,users,utmp,utempter,avahi-autoipd,ssh_keys,systemd-journal,dbus,rpc,tss,ntp,dip,rpcuser,nfsnobody,postdrop,postfix,sshd,screen,nagios,nrpe,openvpn,input,systemd-bus-proxy,systemd-network,cgred,chrony,printadmin,sssd,named,apache +{% else %} +## This file contains users who are in ipa to stop people from +## creating restricted accounts but we want to make sure the id in +## /etc/passwd and /etc/group are used. +[nss] +filter_users = root,bin,daemon,adm,lp,sync,shutdown,halt,mail,operator,games,ftp,nobody,avahi-autoipd,dbus,polkitd,rpc,tss,ntp,rpcuser,nfsnobody,postfix,sshd,nagios,nrpe,openvpn,,chrony,sssd,named,mock,apache,bodhi,ftpsync +filter_groups = root,bin,daemon,sys,adm,tty,disk,lp,mem,kmem,wheel,cdrom,mail,man,dialout,floppy,games,tape,video,ftp,lock,audio,nobody,users,utmp,utempter,avahi-autoipd,ssh_keys,systemd-journal,dbus,rpc,tss,ntp,dip,rpcuser,nfsnobody,postdrop,postfix,sshd,screen,nagios,nrpe,openvpn,input,systemd-bus-proxy,systemd-network,cgred,chrony,printadmin,sssd,named,mock,apache +{% endif %} diff --git a/roles/ipa/client/files/fedora-nss-ignore.conf.people b/roles/ipa/client/templates/fedora-nss-ignore.conf.people similarity index 100% rename from roles/ipa/client/files/fedora-nss-ignore.conf.people rename to roles/ipa/client/templates/fedora-nss-ignore.conf.people diff --git a/roles/ipa/client/files/fedora-nss-ignore.conf.staging b/roles/ipa/client/templates/fedora-nss-ignore.conf.staging similarity index 100% rename from roles/ipa/client/files/fedora-nss-ignore.conf.staging rename to roles/ipa/client/templates/fedora-nss-ignore.conf.staging From 1c6dfc82fd1541e59fd83975753170468e3b7211 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Sat, 22 May 2021 10:26:50 -0700 Subject: [PATCH 066/189] ipa/client: no comment in this jinja2 sadly, just make this a normal comment Signed-off-by: Kevin Fenzi --- roles/ipa/client/templates/fedora-nss-ignore.conf | 6 ------ roles/ipa/client/templates/fedora-nss-ignore.conf.j2 | 2 -- roles/ipa/client/templates/fedora-nss-ignore.conf.people | 6 ------ roles/ipa/client/templates/fedora-nss-ignore.conf.staging | 6 ------ 4 files changed, 20 deletions(-) delete mode 100644 roles/ipa/client/templates/fedora-nss-ignore.conf delete mode 100644 roles/ipa/client/templates/fedora-nss-ignore.conf.people delete mode 100644 roles/ipa/client/templates/fedora-nss-ignore.conf.staging diff --git a/roles/ipa/client/templates/fedora-nss-ignore.conf b/roles/ipa/client/templates/fedora-nss-ignore.conf deleted file mode 100644 index a4214b99a9..0000000000 --- a/roles/ipa/client/templates/fedora-nss-ignore.conf +++ /dev/null @@ -1,6 +0,0 @@ -## This file contains users who are in ipa to stop people from -## creating restricted accounts but we want to make sure the id in -## /etc/passwd and /etc/group are used. -[nss] -filter_users = root,bin,daemon,adm,lp,sync,shutdown,halt,mail,operator,games,ftp,nobody,avahi-autoipd,dbus,polkitd,rpc,tss,ntp,rpcuser,nfsnobody,postfix,sshd,nagios,nrpe,openvpn,,chrony,sssd,named,mock,apache,bodhi,ftpsync -filter_groups = root,bin,daemon,sys,adm,tty,disk,lp,mem,kmem,wheel,cdrom,mail,man,dialout,floppy,games,tape,video,ftp,lock,audio,nobody,users,utmp,utempter,avahi-autoipd,ssh_keys,systemd-journal,dbus,rpc,tss,ntp,dip,rpcuser,nfsnobody,postdrop,postfix,sshd,screen,nagios,nrpe,openvpn,input,systemd-bus-proxy,systemd-network,cgred,chrony,printadmin,sssd,named,mock,apache diff --git a/roles/ipa/client/templates/fedora-nss-ignore.conf.j2 b/roles/ipa/client/templates/fedora-nss-ignore.conf.j2 index fda576fb2a..784e4033b3 100644 --- a/roles/ipa/client/templates/fedora-nss-ignore.conf.j2 +++ b/roles/ipa/client/templates/fedora-nss-ignore.conf.j2 @@ -1,8 +1,6 @@ -{% comment %} # There are a bunch of users we want to prefer the local version instead of the ipa/ldap version. # But in at least one case, there is a real user ('mock') that we want to exclude on most machines, # but not on people and ipsilon to allow them access to their people space and be able to login to things. -{% endcomment %} {% if inventory_hostname in groups['people','ipsilon','ipsilon_stg'] %} ## This file contains users who are in ipa to stop people from ## creating restricted accounts but we want to make sure the id in diff --git a/roles/ipa/client/templates/fedora-nss-ignore.conf.people b/roles/ipa/client/templates/fedora-nss-ignore.conf.people deleted file mode 100644 index d6ef610c3f..0000000000 --- a/roles/ipa/client/templates/fedora-nss-ignore.conf.people +++ /dev/null @@ -1,6 +0,0 @@ -## This file contains users who are in ipa to stop people from -## creating restricted accounts but we want to make sure the id in -## /etc/passwd and /etc/group are used. -[nss] -filter_users = root,bin,daemon,adm,lp,sync,shutdown,halt,mail,operator,games,ftp,nobody,avahi-autoipd,dbus,polkitd,rpc,tss,ntp,rpcuser,nfsnobody,postfix,sshd,nagios,nrpe,openvpn,,chrony,sssd,named,apache -filter_groups = root,bin,daemon,sys,adm,tty,disk,lp,mem,kmem,wheel,cdrom,mail,man,dialout,floppy,games,tape,video,ftp,lock,audio,nobody,users,utmp,utempter,avahi-autoipd,ssh_keys,systemd-journal,dbus,rpc,tss,ntp,dip,rpcuser,nfsnobody,postdrop,postfix,sshd,screen,nagios,nrpe,openvpn,input,systemd-bus-proxy,systemd-network,cgred,chrony,printadmin,sssd,named,apache diff --git a/roles/ipa/client/templates/fedora-nss-ignore.conf.staging b/roles/ipa/client/templates/fedora-nss-ignore.conf.staging deleted file mode 100644 index d0d0023615..0000000000 --- a/roles/ipa/client/templates/fedora-nss-ignore.conf.staging +++ /dev/null @@ -1,6 +0,0 @@ -## This file contains users who are in ipa to stop people from -## creating restricted accounts but we want to make sure the id in -## /etc/passwd and /etc/group are used. -[nss] -filter_users = root,bin,daemon,adm,lp,sync,shutdown,halt,mail,operator,games,ftp,nobody,avahi-autoipd,dbus,polkitd,rpc,tss,ntp,rpcuser,nfsnobody,postfix,sshd,nagios,nrpe,openvpn,,chrony,sssd,named,mock -filter_groups = root,bin,daemon,sys,adm,tty,disk,lp,mem,kmem,wheel,cdrom,mail,man,dialout,floppy,games,tape,video,ftp,lock,audio,nobody,users,utmp,utempter,avahi-autoipd,ssh_keys,systemd-journal,dbus,rpc,tss,ntp,dip,rpcuser,nfsnobody,postdrop,postfix,sshd,screen,nagios,nrpe,openvpn,input,systemd-bus-proxy,systemd-network,cgred,chrony,printadmin,sssd,named,mock From 52a197735b7c6ecaef93c503169f5d6c85240a9e Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Sat, 22 May 2021 10:36:20 -0700 Subject: [PATCH 067/189] ipa/client: split out these groups Signed-off-by: Kevin Fenzi --- roles/ipa/client/templates/fedora-nss-ignore.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/ipa/client/templates/fedora-nss-ignore.conf.j2 b/roles/ipa/client/templates/fedora-nss-ignore.conf.j2 index 784e4033b3..b83c765706 100644 --- a/roles/ipa/client/templates/fedora-nss-ignore.conf.j2 +++ b/roles/ipa/client/templates/fedora-nss-ignore.conf.j2 @@ -1,7 +1,7 @@ # There are a bunch of users we want to prefer the local version instead of the ipa/ldap version. # But in at least one case, there is a real user ('mock') that we want to exclude on most machines, # but not on people and ipsilon to allow them access to their people space and be able to login to things. -{% if inventory_hostname in groups['people','ipsilon','ipsilon_stg'] %} +{% if inventory_hostname in groups['people'] or inventory_hostname in groups['ipsilon'] or inventory_hostname in groups['ipsilon_stg'] %} ## This file contains users who are in ipa to stop people from ## creating restricted accounts but we want to make sure the id in ## /etc/passwd and /etc/group are used. From 24def8cd002feae09d40c5edcb45576f9c58a7f1 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Sun, 23 May 2021 10:51:01 -0700 Subject: [PATCH 068/189] staging-sync/koji: nfs is nfs-server now Signed-off-by: Kevin Fenzi --- playbooks/manual/staging-sync/koji.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/manual/staging-sync/koji.yml b/playbooks/manual/staging-sync/koji.yml index 693b37a957..d4af35f799 100644 --- a/playbooks/manual/staging-sync/koji.yml +++ b/playbooks/manual/staging-sync/koji.yml @@ -97,8 +97,8 @@ - wipe-fs tasks: - - name: stop nfs - service: name=nfs state=stopped + - name: stop nfs-server + service: name=nfs-server state=stopped - name: unmount /mnt/fedora_koji/koji/ostree mount: name=/mnt/fedora_koji/koji/ostree state=unmounted - name: unmount /mnt/fedora_koji/koji/compose/ostree From fd2a8ab38c1b0977834662ccd2ad67fc50248cb2 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Sun, 23 May 2021 10:52:37 -0700 Subject: [PATCH 069/189] staging-sync/koji: fix another nfs server case Signed-off-by: Kevin Fenzi --- playbooks/manual/staging-sync/koji.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/manual/staging-sync/koji.yml b/playbooks/manual/staging-sync/koji.yml index d4af35f799..a6e4c38528 100644 --- a/playbooks/manual/staging-sync/koji.yml +++ b/playbooks/manual/staging-sync/koji.yml @@ -124,8 +124,8 @@ # TODO recreate directories under /mnt/koji/compose (which ones?) that composer expects to exist - name: symlink production volume file: src=/mnt/fedora_koji_prod/koji dest=/mnt/koji/vol/prod state=link - - name: restart nfs - service: name=nfs state=started + - name: restart nfs-server + service: name=nfs-server state=started - name: restart httpd service: name=httpd state=started - name: restart kojira From 54a29eaaa107b0116ce2a69973c425d7d3aa7897 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Sun, 23 May 2021 11:07:35 -0700 Subject: [PATCH 070/189] repo files: drop failovermethod=priority as it's not been used in years Additionally, newer fedora complains about it being invalid. It was used back in the yum days and never ported to dnf. Signed-off-by: Kevin Fenzi --- files/common/fedora-modular.repo | 3 --- files/common/fedora-updates-modular.repo | 3 --- files/common/fedora-updates-testing-modular.repo | 3 --- files/common/fedora-updates-testing.repo | 3 --- files/common/fedora-updates.repo | 3 --- files/common/fedora.repo | 3 --- 6 files changed, 18 deletions(-) diff --git a/files/common/fedora-modular.repo b/files/common/fedora-modular.repo index cacbc82abd..7f69a45281 100644 --- a/files/common/fedora-modular.repo +++ b/files/common/fedora-modular.repo @@ -1,6 +1,5 @@ [fedora-modular] name=Fedora Modular $releasever - $basearch -failovermethod=priority {% if ansible_distribution_major_version|int < ( FedoraBranchedNumber|int if FedoraBranched == True else FedoraRawhideNumber|int ) %} baseurl=https://infrastructure.fedoraproject.org/pub{{ archive_if_archived }}/fedora/linux/releases/$releasever/Modular/$basearch/os/ {% else %} @@ -14,7 +13,6 @@ gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch [fedora-modular-debuginfo] name=Fedora Modular $releasever - $basearch - Debug -failovermethod=priority {% if ansible_distribution_major_version|int < ( FedoraBranchedNumber|int if FedoraBranched == True else FedoraRawhideNumber|int ) %} baseurl=https://infrastructure.fedoraproject.org/pub{{ archive_if_archived }}/fedora/linux/releases/$releasever/Modular/$basearch/debug/tree/ {% else %} @@ -28,7 +26,6 @@ gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch [fedora-modular-source] name=Fedora Modular $releasever - Source -failovermethod=priority {% if ansible_distribution_major_version|int < ( FedoraBranchedNumber|int if FedoraBranched == True else FedoraRawhideNumber|int ) %} baseurl=https://infrastructure.fedoraproject.org/pub{{ archive_if_archived }}/fedora/linux/releases/$releasever/Modular/source/tree/ {% else %} diff --git a/files/common/fedora-updates-modular.repo b/files/common/fedora-updates-modular.repo index e6633268f9..58d114804e 100644 --- a/files/common/fedora-updates-modular.repo +++ b/files/common/fedora-updates-modular.repo @@ -1,6 +1,5 @@ [updates-modular] name=Fedora Modular $releasever - $basearch - Updates -failovermethod=priority baseurl=https://infrastructure.fedoraproject.org/pub{{ archive_if_archived }}/fedora/linux/updates/$releasever/Modular/$basearch/ #metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-released-modular-f$releasever&arch=$basearch enabled=1 @@ -9,7 +8,6 @@ gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch [updates-modular-debuginfo] name=Fedora Modular $releasever - $basearch - Updates - Debug -failovermethod=priority baseurl=https://infrastructure.fedoraproject.org/pub{{ archive_if_archived }}/fedora/linux/updates/$releasever/Modular/$basearch/debug/ #metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-released-modular-debug-f$releasever&arch=$basearch enabled=0 @@ -18,7 +16,6 @@ gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch [updates-modular-source] name=Fedora Modular $releasever - Updates Source -failovermethod=priority baseurl=https://infrastructure.fedoraproject.org/pub{{ archive_if_archived }}/fedora/linux/updates/$releasever/Modular/SRPMS/ #metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-released-modular-source-f$releasever&arch=$basearch enabled=0 diff --git a/files/common/fedora-updates-testing-modular.repo b/files/common/fedora-updates-testing-modular.repo index 2047e50451..7b01cac09f 100644 --- a/files/common/fedora-updates-testing-modular.repo +++ b/files/common/fedora-updates-testing-modular.repo @@ -1,6 +1,5 @@ [updates-testing-modular] name=Fedora Modular $releasever - $basearch - Test Updates -failovermethod=priority baseurl=https://infrastructure.fedoraproject.org/pub{{ archive_if_archived }}/fedora/linux/updates/testing/$releasever/Modular/$basearch/ #metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-modular-f$releasever&arch=$basearch enabled=0 @@ -9,7 +8,6 @@ gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch [updates-testing-modular-debuginfo] name=Fedora Modular $releasever - $basearch - Test Updates Debug -failovermethod=priority baseurl=https://infrastructure.fedoraproject.org/pub{{ archive_if_archived }}/fedora/linux/updates/$releasever/Modular/$basearch/debug/ #metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-modular-debug-f$releasever&arch=$basearch enabled=0 @@ -18,7 +16,6 @@ gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch [updates-testing-modular-source] name=Fedora Modular $releasever - Test Updates Source -failovermethod=priority baseurl=https://infrastructure.fedoraproject.org/pub{{ archive_if_archived }}/fedora/linux/updates/$releasever/Modular/SRPMS/ #metalink=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-modular-source-f$releasever&arch=$basearch enabled=0 diff --git a/files/common/fedora-updates-testing.repo b/files/common/fedora-updates-testing.repo index 52f807b444..05e87ff1e3 100644 --- a/files/common/fedora-updates-testing.repo +++ b/files/common/fedora-updates-testing.repo @@ -1,6 +1,5 @@ [updates-testing] name=Fedora $releasever - $basearch - Test Updates -failovermethod=priority {% if ansible_distribution_major_version|int >27 %} baseurl=https://infrastructure.fedoraproject.org/pub{{ archive_if_archived }}/fedora/linux/updates/testing/$releasever/Everything/$basearch/ {% else %} @@ -13,7 +12,6 @@ gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch [updates-testing-debuginfo] name=Fedora $releasever - $basearch - Test Updates Debug -failovermethod=priority {% if ansible_distribution_major_version|int >27 %} baseurl=https://infrastructure.fedoraproject.org/pub{{ archive_if_archived }}/fedora/linux/updates/testing/$releasever/Everything/$basearch/debug/ {% else %} @@ -26,7 +24,6 @@ gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch [updates-testing-source] name=Fedora $releasever - Test Updates Source -failovermethod=priority {% if ansible_distribution_major_version|int >27 %} baseurl=https://infrastructure.fedoraproject.org/pub{{ archive_if_archived }}/fedora/linux/updates/testing/$releasever/Everything/SRPMS/ {% else %} diff --git a/files/common/fedora-updates.repo b/files/common/fedora-updates.repo index f01ae60fb7..04851e12c0 100644 --- a/files/common/fedora-updates.repo +++ b/files/common/fedora-updates.repo @@ -1,6 +1,5 @@ [updates] name=Fedora $releasever - $basearch - Updates -failovermethod=priority {% if ansible_distribution_major_version|int > 27 %} baseurl=https://infrastructure.fedoraproject.org/pub{{ archive_if_archived }}/fedora/linux/updates/$releasever/Everything/$basearch/ {% else %} @@ -13,7 +12,6 @@ gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch [updates-debuginfo] name=Fedora $releasever - $basearch - Updates - Debug -failovermethod=priority {% if ansible_distribution_major_version|int > 27 %} baseurl=http://infrastructure.fedoraproject.org/pub{{ archive_if_archived }}/fedora/linux/updates/$releasever/Everything/$basearch/debug/ {% else %} @@ -26,7 +24,6 @@ gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch [updates-source] name=Fedora $releasever - Updates Source -failovermethod=priority {% if ansible_distribution_major_version|int > 27 %} baseurl=http://infrastructure.fedoraproject.org/pub{{ archive_if_archived }}/fedora/linux/updates/$releasever/Everything/SRPMS/ {% else %} diff --git a/files/common/fedora.repo b/files/common/fedora.repo index b8dcf7662e..794ade7733 100644 --- a/files/common/fedora.repo +++ b/files/common/fedora.repo @@ -1,6 +1,5 @@ [fedora] name=Fedora $releasever - $basearch -failovermethod=priority {% if ansible_distribution_major_version|int < ( FedoraBranchedNumber|int if FedoraBranched == True else FedoraRawhideNumber|int ) %} baseurl=https://infrastructure.fedoraproject.org/pub{{ archive_if_archived }}/fedora/linux/releases/$releasever/Everything/$basearch/os/ #metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch @@ -14,7 +13,6 @@ gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch [fedora-debuginfo] name=Fedora $releasever - $basearch - Debug -failovermethod=priority {% if ansible_distribution_major_version|int < ( FedoraBranchedNumber|int if FedoraBranched == True else FedoraRawhideNumber|int ) %} baseurl=http://infrastructure.fedoraproject.org/pub{{ archive_if_archived }}/fedora/linux/releases/$releasever/Everything/$basearch/debug/tree/ #metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-debug-$releasever&arch=$basearch @@ -28,7 +26,6 @@ gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch [fedora-source] name=Fedora $releasever - Source -failovermethod=priority {% if ansible_distribution_major_version|int < ( FedoraBranchedNumber|int if FedoraBranched|int is defined else FedoraRawhideNumber|int ) %} baseurl=http://infrastructure.fedoraproject.org/pub{{ archive_if_archived }}/fedora/linux/releases/$releasever/Everything/source/tree/ #metalink=https://mirrors.fedoraproject.org/metalink?repo=fedora-source-$releasever&arch=$basearch From 9b00f4b02a9c986eccef1ce38017d2d61785fab3 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Sun, 23 May 2021 22:05:03 +0200 Subject: [PATCH 071/189] copr-be: centralize aws/spot/hypervisor config in inventory --- inventory/group_vars/copr_aws | 12 +++++++++ inventory/group_vars/copr_dev_aws | 12 +++++++++ roles/copr/backend/tasks/main.yml | 25 +++++++++++++++++++ roles/copr/backend/templates/copr-be.conf.j2 | 9 ++++--- .../backend/templates/resalloc/pools.yaml | 20 +++++++-------- 5 files changed, 64 insertions(+), 14 deletions(-) diff --git a/inventory/group_vars/copr_aws b/inventory/group_vars/copr_aws index 206d284752..6bc9e1f4a6 100644 --- a/inventory/group_vars/copr_aws +++ b/inventory/group_vars/copr_aws @@ -38,4 +38,16 @@ builders: s390x: [0,0,0] aarch64: [0,2,2] + #x86_hypervisor_01: + # x86_64: [2,1,1] + + x86_hypervisor_02: + x86_64: [20,3,20] + + x86_hypervisor_03: + x86_64: [20,3,20] + + x86_hypervisor_04: + x86_64: [20,3,20] + rpm_vendor_copr_name: Fedora Copr diff --git a/inventory/group_vars/copr_dev_aws b/inventory/group_vars/copr_dev_aws index a5b2ebd7bf..611759b6d3 100644 --- a/inventory/group_vars/copr_dev_aws +++ b/inventory/group_vars/copr_dev_aws @@ -39,4 +39,16 @@ builders: s390x: [1,0,0] aarch64: [5,2,2] + #x86_hypervisor_01: + # x86_64: [2,1,1] + + x86_hypervisor_02: + x86_64: [2,1,1] + + x86_hypervisor_03: + x86_64: [2,1,1] + + x86_hypervisor_04: + x86_64: [2,1,1] + rpm_vendor_copr_name: Fedora Copr (devel) diff --git a/roles/copr/backend/tasks/main.yml b/roles/copr/backend/tasks/main.yml index 83b2387030..7805c42b22 100644 --- a/roles/copr/backend/tasks/main.yml +++ b/roles/copr/backend/tasks/main.yml @@ -162,6 +162,31 @@ # - library # tags: # - provision_config +# +- name: calculate the maximum resalloc workers per architecture + set_fact: "max_{{ item }}_workers={{ builders | community.general.json_query('*.'+item+'[0]') | sum() }}" + with_items: + - x86_64 + - armhfp + - s390x + - aarch64 + tags: + - provision_config + +- name: calculate max number of workers + set_fact: "max_workers={{ max_x86_64_workers|int + max_armhfp_workers|int + max_s390x_workers|int + max_aarch64_workers|int }}" + tags: + - provision_config + +- name: print max arch workers + debug: "var=max_{{ item }}_workers" + with_items: + - x86_64 + - armhfp + - s390x + - aarch64 + tags: + - provision_config - name: resalloc import_tasks: resalloc.yml diff --git a/roles/copr/backend/templates/copr-be.conf.j2 b/roles/copr/backend/templates/copr-be.conf.j2 index ba67a34907..06986e33ac 100644 --- a/roles/copr/backend/templates/copr-be.conf.j2 +++ b/roles/copr/backend/templates/copr-be.conf.j2 @@ -38,13 +38,14 @@ sleeptime=20 # This option basically controls the amount of RAM allocated for # processing builds on copr backend, and how many resalloc tickets can # be taken at the same time. -builds_max_workers={{ builders.aws.x86_64[0] + builders.aws.aarch64[0] + builders.aws.armhfp[0] }} +builds_max_workers={{ max_workers }} # Maximum number of concurrently running tasks per architecture. builds_max_workers_arch= - x86_64={{ builders.aws.x86_64[0] }}, - aarch64={{ builders.aws.aarch64[0]}}, - armhfp={{ builders.aws.armhfp[0] }} + x86_64={{ max_x86_64_workers }}, + aarch64={{ max_aarch64_workers }}, + armhfp={{ max_armhfp_workers }}, + s390x={{ max_s390x_workers }} # Maximum number of concurrently running tasks per project owner. {% if env == 'production' %} diff --git a/roles/copr/backend/templates/resalloc/pools.yaml b/roles/copr/backend/templates/resalloc/pools.yaml index ea0d4986cd..f9b1e36ab1 100644 --- a/roles/copr/backend/templates/resalloc/pools.yaml +++ b/roles/copr/backend/templates/resalloc/pools.yaml @@ -49,17 +49,12 @@ aws_aarch64_{% if spot %}spot{% else %}normal{% endif %}_{% if devel %}dev{% els {% endmacro %} # x86_64 hypervisors -{% for hv in ["02", "03", "04"] %} +{% for hv in ["01", "02", "03", "04"] %} +{% if "x86_hypervisor_" + hv in builders %} copr_hv_x86_64_{{ hv }}_{% if devel %}dev{% else %}prod{% endif %}: -{% if devel %} - max: 3 - max_starting: 1 - max_prealloc: 1 -{% else %} - max: 20 - max_starting: 4 - max_prealloc: 20 -{% endif %} + max: {{ builders["x86_hypervisor_" + hv]["x86_64"][0] }} + max_starting: {{ builders["x86_hypervisor_" + hv]["x86_64"][1] }} + max_prealloc: {{ builders["x86_hypervisor_" + hv]["x86_64"][2] }} tags: - copr_builder - arch_x86_64 @@ -85,6 +80,11 @@ copr_hv_x86_64_{{ hv }}_{% if devel %}dev{% else %}prod{% endif %}: reuse_opportunity_time: 180 reuse_max_count: 8 reuse_max_time: 1800 +{% else %} +# +# x86_hypervisor_{{ hv }} not configured in group_vars +# +{% endif %} {% endfor %} {% macro hw_aarch64(id, inst, max, max_starting, max_prealloc) %} From 07ac44211622d3f4ae0f91995d76b4c925eebbf8 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Sun, 23 May 2021 22:08:23 +0200 Subject: [PATCH 072/189] copr-be: backend config tagged as provision_config The provision limits/quota is directly reflected in copr-be.conf. --- roles/copr/backend/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/copr/backend/tasks/main.yml b/roles/copr/backend/tasks/main.yml index 7805c42b22..22bc213483 100644 --- a/roles/copr/backend/tasks/main.yml +++ b/roles/copr/backend/tasks/main.yml @@ -232,6 +232,7 @@ tags: - config - copr_infrastructure_password + - provision_config - name: create messaging config directory file: dest=/etc/copr/msgbuses state=directory owner=copr group=copr mode=0700 From 3fdc635dd42c756a5668774260228c68689ffe4a Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Sun, 23 May 2021 22:40:48 +0200 Subject: [PATCH 073/189] copr-be: simplify VM quota --- inventory/group_vars/copr_dev_aws | 19 ++++++++---------- roles/copr/backend/tasks/main.yml | 6 +----- roles/copr/backend/templates/copr-be.conf.j2 | 7 ++++--- .../backend/templates/resalloc/pools.yaml | 20 +++++++------------ 4 files changed, 20 insertions(+), 32 deletions(-) diff --git a/inventory/group_vars/copr_dev_aws b/inventory/group_vars/copr_dev_aws index 611759b6d3..95ac07ecd9 100644 --- a/inventory/group_vars/copr_dev_aws +++ b/inventory/group_vars/copr_dev_aws @@ -29,26 +29,23 @@ nm_controlled_resolv: True builders: # max|max_spawn|max_prealloc aws: - x86_64: [4,1,1] - armhfp: [2,0,0] - s390x: [2,0,0] - aarch64: [2,1,1] + x86_64: [4, 1, 1] + aarch64: [2, 1, 1] + aws_spot: - x86_64: [5,2,3] - armhfp: [2,0,0] - s390x: [1,0,0] - aarch64: [5,2,2] + x86_64: [5, 2, 3] + aarch64: [5, 2, 2] #x86_hypervisor_01: # x86_64: [2,1,1] x86_hypervisor_02: - x86_64: [2,1,1] + x86_64: [2, 1, 1] x86_hypervisor_03: - x86_64: [2,1,1] + x86_64: [2, 1, 1] x86_hypervisor_04: - x86_64: [2,1,1] + x86_64: [2, 1, 1] rpm_vendor_copr_name: Fedora Copr (devel) diff --git a/roles/copr/backend/tasks/main.yml b/roles/copr/backend/tasks/main.yml index 22bc213483..5915005e9f 100644 --- a/roles/copr/backend/tasks/main.yml +++ b/roles/copr/backend/tasks/main.yml @@ -167,14 +167,12 @@ set_fact: "max_{{ item }}_workers={{ builders | community.general.json_query('*.'+item+'[0]') | sum() }}" with_items: - x86_64 - - armhfp - - s390x - aarch64 tags: - provision_config - name: calculate max number of workers - set_fact: "max_workers={{ max_x86_64_workers|int + max_armhfp_workers|int + max_s390x_workers|int + max_aarch64_workers|int }}" + set_fact: "max_workers={{ max_x86_64_workers|int + max_aarch64_workers|int }}" tags: - provision_config @@ -182,8 +180,6 @@ debug: "var=max_{{ item }}_workers" with_items: - x86_64 - - armhfp - - s390x - aarch64 tags: - provision_config diff --git a/roles/copr/backend/templates/copr-be.conf.j2 b/roles/copr/backend/templates/copr-be.conf.j2 index 06986e33ac..dab60b1baa 100644 --- a/roles/copr/backend/templates/copr-be.conf.j2 +++ b/roles/copr/backend/templates/copr-be.conf.j2 @@ -40,12 +40,13 @@ sleeptime=20 # be taken at the same time. builds_max_workers={{ max_workers }} -# Maximum number of concurrently running tasks per architecture. +# Maximum number of concurrently running tasks per architecture. Since we +# emulate s390x and armhfp on x86_64, we give it ~25% of the x86_64 quota. builds_max_workers_arch= x86_64={{ max_x86_64_workers }}, aarch64={{ max_aarch64_workers }}, - armhfp={{ max_armhfp_workers }}, - s390x={{ max_s390x_workers }} + armhfp={{ (max_x86_64_workers|int / 4)|int }}, + s390x={{ (max_x86_64_workers|int / 4)|int }}, # Maximum number of concurrently running tasks per project owner. {% if env == 'production' %} diff --git a/roles/copr/backend/templates/resalloc/pools.yaml b/roles/copr/backend/templates/resalloc/pools.yaml index f9b1e36ab1..af23fabee5 100644 --- a/roles/copr/backend/templates/resalloc/pools.yaml +++ b/roles/copr/backend/templates/resalloc/pools.yaml @@ -112,24 +112,18 @@ aarch64_{{ id }}_{{ inst }}: - arch_aarch64_native {% endmacro %} -{{ aws_x86_64(builders.aws.x86_64[0] + builders.aws.armhfp[0] + builders.aws.s390x[0], - builders.aws.x86_64[1] + builders.aws.armhfp[1] + builders.aws.s390x[1], - builders.aws.x86_64[2] + builders.aws.armhfp[2] + builders.aws.s390x[2]) +{{ aws_x86_64(builders.aws.x86_64[0], builders.aws.x86_64[1], + builders.aws.x86_64[2]) }} -{{ aws_aarch64(builders.aws.aarch64[0], - builders.aws.aarch64[1], +{{ aws_aarch64(builders.aws.aarch64[0], builders.aws.aarch64[1], builders.aws.aarch64[2]) }} -{{ aws_x86_64(builders.aws_spot.x86_64[0] + builders.aws_spot.armhfp[0] + builders.aws_spot.s390x[0], - builders.aws_spot.x86_64[1] + builders.aws_spot.armhfp[1] + builders.aws_spot.s390x[1], - builders.aws_spot.x86_64[2] + builders.aws_spot.armhfp[2] + builders.aws_spot.s390x[2], - True) +{{ aws_x86_64(builders.aws_spot.x86_64[0], builders.aws_spot.x86_64[1], + builders.aws_spot.x86_64[2], True) }} -{{ aws_aarch64(builders.aws_spot.aarch64[0], - builders.aws_spot.aarch64[1], - builders.aws_spot.aarch64[2], - True) +{{ aws_aarch64(builders.aws_spot.aarch64[0], builders.aws_spot.aarch64[1], + builders.aws_spot.aarch64[2], True) }} {% if env == "production" %} From 07d2d46d96cf2d9fb6d61b12bfbb906e7df32eeb Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Sun, 23 May 2021 22:44:52 +0200 Subject: [PATCH 074/189] copr-be: fix prod inventory for previous commit --- inventory/group_vars/copr_aws | 4 ---- 1 file changed, 4 deletions(-) diff --git a/inventory/group_vars/copr_aws b/inventory/group_vars/copr_aws index 6bc9e1f4a6..ae13254e6c 100644 --- a/inventory/group_vars/copr_aws +++ b/inventory/group_vars/copr_aws @@ -29,13 +29,9 @@ builders: # max|max_spawn|max_prealloc aws: x86_64: [100,15,15] - armhfp: [20,3,3] - s390x: [5,1,1] aarch64: [20,6,6] aws_spot: x86_64: [0,1,1] - armhfp: [0,0,0] - s390x: [0,0,0] aarch64: [0,2,2] #x86_hypervisor_01: From fe888acea909078ffae8c86969a78a71a64bbf48 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Sun, 23 May 2021 22:47:57 +0200 Subject: [PATCH 075/189] copr-be: typo in config --- roles/copr/backend/templates/copr-be.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/copr/backend/templates/copr-be.conf.j2 b/roles/copr/backend/templates/copr-be.conf.j2 index dab60b1baa..ba4e0fea64 100644 --- a/roles/copr/backend/templates/copr-be.conf.j2 +++ b/roles/copr/backend/templates/copr-be.conf.j2 @@ -46,7 +46,7 @@ builds_max_workers_arch= x86_64={{ max_x86_64_workers }}, aarch64={{ max_aarch64_workers }}, armhfp={{ (max_x86_64_workers|int / 4)|int }}, - s390x={{ (max_x86_64_workers|int / 4)|int }}, + s390x={{ (max_x86_64_workers|int / 4)|int }} # Maximum number of concurrently running tasks per project owner. {% if env == 'production' %} From a1b9ee23a348705dce7766dc360888d641a00086 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Sun, 23 May 2021 22:54:44 +0200 Subject: [PATCH 076/189] copr-be: limit max tickets for 32bit x86 --- roles/copr/backend/templates/copr-be.conf.j2 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/roles/copr/backend/templates/copr-be.conf.j2 b/roles/copr/backend/templates/copr-be.conf.j2 index ba4e0fea64..9a06f2f3df 100644 --- a/roles/copr/backend/templates/copr-be.conf.j2 +++ b/roles/copr/backend/templates/copr-be.conf.j2 @@ -42,11 +42,16 @@ builds_max_workers={{ max_workers }} # Maximum number of concurrently running tasks per architecture. Since we # emulate s390x and armhfp on x86_64, we give it ~25% of the x86_64 quota. +# The 32bit x86 variants (i386, i586 and i686) are in minority now, so let's +# limit them as well. builds_max_workers_arch= x86_64={{ max_x86_64_workers }}, aarch64={{ max_aarch64_workers }}, armhfp={{ (max_x86_64_workers|int / 4)|int }}, - s390x={{ (max_x86_64_workers|int / 4)|int }} + s390x={{ (max_x86_64_workers|int / 4)|int }}, + i386={{ (max_x86_64_workers|int / 4)|int }}, + i586={{ (max_x86_64_workers|int / 4)|int }}, + i686={{ (max_x86_64_workers|int / 4)|int }} # Maximum number of concurrently running tasks per project owner. {% if env == 'production' %} From c112053767e374758a00a66472ec77afc36033ef Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Sun, 23 May 2021 22:59:35 +0200 Subject: [PATCH 077/189] copr-be: turn on spot instances in production --- inventory/group_vars/copr_aws | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/inventory/group_vars/copr_aws b/inventory/group_vars/copr_aws index ae13254e6c..20ff125aad 100644 --- a/inventory/group_vars/copr_aws +++ b/inventory/group_vars/copr_aws @@ -28,22 +28,22 @@ nm_controlled_resolv: True builders: # max|max_spawn|max_prealloc aws: - x86_64: [100,15,15] - aarch64: [20,6,6] + x86_64: [30, 5, 5] + aarch64: [8, 2, 2] aws_spot: - x86_64: [0,1,1] - aarch64: [0,2,2] + x86_64: [60, 10, 20] + aarch64: [30, 4, 6] #x86_hypervisor_01: - # x86_64: [2,1,1] + # x86_64: [20, 4, 20] x86_hypervisor_02: - x86_64: [20,3,20] + x86_64: [20, 4, 20] x86_hypervisor_03: - x86_64: [20,3,20] + x86_64: [20, 4, 20] x86_hypervisor_04: - x86_64: [20,3,20] + x86_64: [20, 4, 20] rpm_vendor_copr_name: Fedora Copr From 47f17055ecbc61c822798e1e84d3599a50b2e7a8 Mon Sep 17 00:00:00 2001 From: Brendan Early Date: Sun, 23 May 2021 16:15:10 -0500 Subject: [PATCH 078/189] packages-static: enable simple query parser --- .../templates/packages/solrconfig.xml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/roles/openshift-apps/fedora-packages-static/templates/packages/solrconfig.xml b/roles/openshift-apps/fedora-packages-static/templates/packages/solrconfig.xml index 0fc3f8b6cb..f87a30319f 100644 --- a/roles/openshift-apps/fedora-packages-static/templates/packages/solrconfig.xml +++ b/roles/openshift-apps/fedora-packages-static/templates/packages/solrconfig.xml @@ -368,11 +368,11 @@ - +