From d5cc71560bb1969c5b2805b65c5b37dd3a09567a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20P=C3=A1ral?= Date: Thu, 18 Jan 2018 16:30:02 +0100 Subject: [PATCH 001/242] buildmaster-configure: fix gzipping artifacts Used to fail for symlinks. See https://pagure.io/taskotron/issue/247 --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index cabe0cd97c..c052754869 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -240,7 +240,7 @@ factory.addStep(DirectoryUpload(slavesrc=Interpolate('/var/lib/taskotron/artifac masterdest=Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/task_output'))) # gzip artifacts -factory.addStep(MasterShellCommand(command=Interpolate('gzip -r {{ public_artifacts_dir }}/%(prop:uuid)s/task_output/*'), +factory.addStep(MasterShellCommand(command=Interpolate('find {{ public_artifacts_dir }}/%(prop:uuid)s/task_output/ -type f -exec gzip {} \;'), descriptionDone=['gzip artifacs dir content'])) {% if deployment_type in ['local'] %} @@ -317,7 +317,7 @@ distgit_factory.addStep(DirectoryUpload(slavesrc=Interpolate('/var/lib/taskotron masterdest=Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/task_output'))) # gzip artifacts -distgit_factory.addStep(MasterShellCommand(command=Interpolate('gzip -r {{ public_artifacts_dir }}/%(prop:uuid)s/task_output/*'), +distgit_factory.addStep(MasterShellCommand(command=Interpolate('find {{ public_artifacts_dir }}/%(prop:uuid)s/task_output/ -type f -exec gzip {} \;'), descriptionDone=['gzip artifacs dir content'])) {% endif %} From 52c1c426c6409d8e1c901086e0901917c3d03e00 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 18 Jan 2018 15:55:22 +0000 Subject: [PATCH 002/242] Bodhi 3.2.0 is GO Signed-off-by: Patrick Uiterwijk --- inventory/host_vars/bodhi-backend01.phx2.fedoraproject.org | 4 ++-- inventory/host_vars/bodhi-backend02.phx2.fedoraproject.org | 4 ++-- inventory/host_vars/bodhi03.phx2.fedoraproject.org | 4 ++-- inventory/host_vars/bodhi04.phx2.fedoraproject.org | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/inventory/host_vars/bodhi-backend01.phx2.fedoraproject.org b/inventory/host_vars/bodhi-backend01.phx2.fedoraproject.org index a6981c6b39..c85cc70f8f 100644 --- a/inventory/host_vars/bodhi-backend01.phx2.fedoraproject.org +++ b/inventory/host_vars/bodhi-backend01.phx2.fedoraproject.org @@ -2,8 +2,8 @@ nm: 255.255.255.0 gw: 10.5.125.254 dns: 10.5.126.21 -ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-26 -ks_repo: http://10.5.126.23/pub/fedora/linux/releases/26/Server/x86_64/os/ +ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-27 +ks_repo: http://10.5.126.23/pub/fedora/linux/releases/27/Server/x86_64/os/ volgroup: /dev/vg_host01 eth0_ip: 10.5.125.135 eth1_ip: 10.5.127.61 diff --git a/inventory/host_vars/bodhi-backend02.phx2.fedoraproject.org b/inventory/host_vars/bodhi-backend02.phx2.fedoraproject.org index 14592e3bb5..b52771da0c 100644 --- a/inventory/host_vars/bodhi-backend02.phx2.fedoraproject.org +++ b/inventory/host_vars/bodhi-backend02.phx2.fedoraproject.org @@ -2,8 +2,8 @@ nm: 255.255.255.0 gw: 10.5.125.254 dns: 10.5.126.21 -ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-26 -ks_repo: http://10.5.126.23/pub/fedora/linux/releases/26/Server/x86_64/os/ +ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-27 +ks_repo: http://10.5.126.23/pub/fedora/linux/releases/27/Server/x86_64/os/ volgroup: /dev/vg_bvirthost06 eth0_ip: 10.5.125.136 eth1_ip: 10.5.127.62 diff --git a/inventory/host_vars/bodhi03.phx2.fedoraproject.org b/inventory/host_vars/bodhi03.phx2.fedoraproject.org index 4978f13135..19dedb83b6 100644 --- a/inventory/host_vars/bodhi03.phx2.fedoraproject.org +++ b/inventory/host_vars/bodhi03.phx2.fedoraproject.org @@ -2,8 +2,8 @@ nm: 255.255.255.0 gw: 10.5.126.254 dns: 10.5.126.21 -ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-26 -ks_repo: http://10.5.126.23/pub/fedora/linux/releases/26/Server/x86_64/os/ +ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-27 +ks_repo: http://10.5.126.23/pub/fedora/linux/releases/27/Server/x86_64/os/ volgroup: /dev/vg_virthost01 eth0_ip: 10.5.126.115 vmhost: virthost01.phx2.fedoraproject.org diff --git a/inventory/host_vars/bodhi04.phx2.fedoraproject.org b/inventory/host_vars/bodhi04.phx2.fedoraproject.org index e7948f32e0..843366304a 100644 --- a/inventory/host_vars/bodhi04.phx2.fedoraproject.org +++ b/inventory/host_vars/bodhi04.phx2.fedoraproject.org @@ -2,8 +2,8 @@ nm: 255.255.255.0 gw: 10.5.126.254 dns: 10.5.126.21 -ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-26 -ks_repo: http://10.5.126.23/pub/fedora/linux/releases/26/Server/x86_64/os/ +ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-27 +ks_repo: http://10.5.126.23/pub/fedora/linux/releases/27/Server/x86_64/os/ volgroup: /dev/vg_guests eth0_ip: 10.5.126.116 vmhost: virthost02.phx2.fedoraproject.org From c9aa5425678920a19ea9216e0e2c185848e1bc13 Mon Sep 17 00:00:00 2001 From: Randy Barlow Date: Thu, 18 Jan 2018 16:01:56 +0000 Subject: [PATCH 003/242] Run bodhi-check-policies on production. Signed-off-by: Randy Barlow --- roles/bodhi2/backend/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/bodhi2/backend/tasks/main.yml b/roles/bodhi2/backend/tasks/main.yml index 795ee43b92..6a153ab649 100644 --- a/roles/bodhi2/backend/tasks/main.yml +++ b/roles/bodhi2/backend/tasks/main.yml @@ -322,7 +322,7 @@ cron: name="bodhi-check-policies" hour="*/6" minute=0 user="apache" job="/usr/bin/bodhi-check-policies > /dev/null" cron_file=bodhi-check-policies-job - when: inventory_hostname.startswith('bodhi-backend01') and env == "staging" + when: inventory_hostname.startswith('bodhi-backend01') tags: - config - bodhi From e561d139e6b28460848fa00ebb1c3f6ffea89b37 Mon Sep 17 00:00:00 2001 From: Randy Barlow Date: Thu, 18 Jan 2018 16:08:57 +0000 Subject: [PATCH 004/242] Run bodhi-check-policies on backend02 for production. Signed-off-by: Randy Barlow --- roles/bodhi2/backend/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/bodhi2/backend/tasks/main.yml b/roles/bodhi2/backend/tasks/main.yml index 6a153ab649..3a2dac2408 100644 --- a/roles/bodhi2/backend/tasks/main.yml +++ b/roles/bodhi2/backend/tasks/main.yml @@ -322,7 +322,7 @@ cron: name="bodhi-check-policies" hour="*/6" minute=0 user="apache" job="/usr/bin/bodhi-check-policies > /dev/null" cron_file=bodhi-check-policies-job - when: inventory_hostname.startswith('bodhi-backend01') + when: (inventory_hostname.startswith('bodhi-backend01') and env == "staging") or (inventory_hostname.startswith('bodhi-backend02') and env == "production") tags: - config - bodhi From 3a78f3deeb9715cf4d266158ee182c57d207b4b1 Mon Sep 17 00:00:00 2001 From: Randy Barlow Date: Thu, 18 Jan 2018 16:09:34 +0000 Subject: [PATCH 005/242] Enable Bodhi's test gating based on Greenwave decisions. Signed-off-by: Randy Barlow --- roles/bodhi2/base/templates/production.ini.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/bodhi2/base/templates/production.ini.j2 b/roles/bodhi2/base/templates/production.ini.j2 index dd70eed050..bf067f1279 100644 --- a/roles/bodhi2/base/templates/production.ini.j2 +++ b/roles/bodhi2/base/templates/production.ini.j2 @@ -298,7 +298,7 @@ krb_keytab = /etc/krb5.bodhi_bodhi{{ env_suffix }}.fedoraproject.org.keytab # Set this to True to enable gating based on policies enforced by Greenwave. If you set this to True, # be sure to add a cron job to run the bodhi-check-policies CLI periodically. -test_gating.required = False +test_gating.required = True # If this is set to a URL, a "More information about test gating" link will appear on update pages for users # to click and learn more. From 12a1603323042d54136403614e470fc4989fc4bd Mon Sep 17 00:00:00 2001 From: Jeremy Cline Date: Thu, 18 Jan 2018 12:49:48 -0500 Subject: [PATCH 006/242] Add a fedmsg consumer for release-monitoring.org Signed-off-by: Jeremy Cline --- .../release-monitoring/files/buildconfig.yml | 45 ++++++++++++++++ .../files/deploymentconfig.yml | 53 +++++++++++++++++++ .../release-monitoring/files/imagestream.yml | 6 +++ 3 files changed, 104 insertions(+) diff --git a/roles/openshift-apps/release-monitoring/files/buildconfig.yml b/roles/openshift-apps/release-monitoring/files/buildconfig.yml index 3fcfc8a61a..436c5d0a2e 100644 --- a/roles/openshift-apps/release-monitoring/files/buildconfig.yml +++ b/roles/openshift-apps/release-monitoring/files/buildconfig.yml @@ -49,5 +49,50 @@ items: to: kind: ImageStreamTag name: release-monitoring-web:latest +- apiVersion: v1 + kind: BuildConfig + metadata: + labels: + build: fedmsg-hub-build + name: fedmsg-hub-build + spec: + runPolicy: Serial + source: + dockerfile: |- + FROM fedora:27 + LABEL \ + name="fedmsg-hub" \ + vendor="Fedora Infrastructure" \ + license="MIT" + RUN dnf install -y \ + git \ + python3-blinker \ + python3-dateutil \ + python3-fedmsg \ + python3-flask \ + python3-flask-wtf \ + python3-flask-login \ + python3-flask-restful \ + python3-flask-openid \ + python3-gunicorn \ + python3-openid \ + python3-pip \ + python3-psycopg2 \ + python3-setuptools \ + python3-straight-plugin \ + python3-sqlalchemy \ + python3-wtforms && \ + dnf autoremove -y && \ + dnf clean all -y + RUN pip-3 install git+https://github.com/release-monitoring/anitya.git@master + ENV USER=fedmsg + ENTRYPOINT fedmsg-hub + type: Dockerfile + strategy: + type: Docker + output: + to: + kind: ImageStreamTag + name: fedmsg-hub:latest kind: List metadata: {} diff --git a/roles/openshift-apps/release-monitoring/files/deploymentconfig.yml b/roles/openshift-apps/release-monitoring/files/deploymentconfig.yml index cdf826edc0..ba3ad2a846 100644 --- a/roles/openshift-apps/release-monitoring/files/deploymentconfig.yml +++ b/roles/openshift-apps/release-monitoring/files/deploymentconfig.yml @@ -67,5 +67,58 @@ items: namespace: release-monitoring type: ImageChange - type: ConfigChange +- apiVersion: v1 + kind: DeploymentConfig + metadata: + labels: + app: fedmsg-hub + service: fedmsg + name: fedmsg-hub + spec: + replicas: 1 + selector: + deploymentconfig: fedmsg-hub + strategy: + activeDeadlineSeconds: 21600 + recreateParams: + timeoutSeconds: 600 + resources: {} + rollingParams: + intervalSeconds: 1 + maxSurge: 25% + maxUnavailable: 25% + timeoutSeconds: 600 + updatePeriodSeconds: 1 + type: Rolling + template: + metadata: + creationTimestamp: null + labels: + app: fedmsg-hub + deploymentconfig: fedmsg-hub + spec: + containers: + - name: fedmsg-hub + image: release-monitoring/fedmsg-hub:latest + resources: {} + volumeMounts: + - name: config-volume + mountPath: /etc/anitya + readOnly: true + volumes: + - name: config-volume + configMap: + name: release-monitoring-configmap + triggers: + - imageChangeParams: + automatic: true + containerNames: + - fedmsg-hub + from: + kind: ImageStreamTag + name: fedmsg-hub:latest + namespace: release-monitoring + type: ImageChange + - type: ConfigChange kind: List metadata: {} diff --git a/roles/openshift-apps/release-monitoring/files/imagestream.yml b/roles/openshift-apps/release-monitoring/files/imagestream.yml index 44856d1c34..f09d401176 100644 --- a/roles/openshift-apps/release-monitoring/files/imagestream.yml +++ b/roles/openshift-apps/release-monitoring/files/imagestream.yml @@ -6,5 +6,11 @@ items: name: release-monitoring-web labels: build: release-monitoring-web +- apiVersion: v1 + kind: ImageStream + metadata: + name: fedmsg-hub + labels: + build: fedmsg-hub kind: List metadata: {} From db9301cf93b2d4104852e959c10c10e478385eed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kalu=C5=BEa?= Date: Fri, 19 Jan 2018 06:51:55 +0000 Subject: [PATCH 007/242] Allow only x86_64 architectures on ODCS staging --- .../odcs/base/templates/etc/odcs/raw_config_wrapper.conf.j2 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/roles/odcs/base/templates/etc/odcs/raw_config_wrapper.conf.j2 b/roles/odcs/base/templates/etc/odcs/raw_config_wrapper.conf.j2 index 30bbed4058..6d5a35fd19 100644 --- a/roles/odcs/base/templates/etc/odcs/raw_config_wrapper.conf.j2 +++ b/roles/odcs/base/templates/etc/odcs/raw_config_wrapper.conf.j2 @@ -12,9 +12,12 @@ createrepo_deltas = False # In runroot, we cannot use guestmount, but have to use mount -o loop. buildinstall_use_guestmount=False +{% if env == 'staging' %} # We cannot use hardlinks on stg, because it uses different volume and copy # just takes lot of storage. -{% if env == 'staging' %} link_type = 'symlink' + +Staging is used only for testing, so allow only x86_64 composes +tree_arches = ['x86_64'] {% endif %} From a4b9280479fa381f8ee5d610ff03ef171cc6ed96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20P=C3=A1ral?= Date: Fri, 19 Jan 2018 10:21:37 +0100 Subject: [PATCH 008/242] taskotron-trigger: merge trigger rules --- .../templates/trigger_rules.yml.j2 | 22 ++++++++------- .../templates/trigger_rules.yml.j2.dev | 27 ------------------- 2 files changed, 12 insertions(+), 37 deletions(-) delete mode 100644 roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2.dev diff --git a/roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2 b/roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2 index da91b20a27..c155f5d946 100644 --- a/roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2 +++ b/roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2 @@ -3,10 +3,6 @@ do: - {tasks: [rpmlint, rpmgrill, python-versions]} -- when: {message_type: KojiBuildPackageCompleted, name: docker} - do: - - {tasks: [dockerautotest]} - - when: message_type: KojiBuildPackageCompleted name: @@ -24,17 +20,24 @@ do: - {tasks: [check_modulemd]} -- when: - message_type: ModuleBuildComplete - do: - - {tasks: [modularity-testing-framework]} - - when: message_type: GitHubPullRequestOpened repo_name: {$regex: '/^container-images\/.+/'} do: - {tasks: [mtf-containers]} +{% if deployment_type in ['stg', 'prod'] %} +{# these tasks are not ansiblized yet #} +- when: + message_type: ModuleBuildComplete + do: + - {tasks: [modularity-testing-framework]} + +- when: {message_type: KojiBuildPackageCompleted, name: docker} + do: + - {tasks: [dockerautotest]} +{% endif %} + {# disabled due to missing nested virt: https://pagure.io/taskotron/issue/239 - when: message_type: AtomicCompose @@ -46,4 +49,3 @@ do: - {tasks: [fedora-cloud-tests]} #} - diff --git a/roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2.dev b/roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2.dev deleted file mode 100644 index 43da94c0fb..0000000000 --- a/roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2.dev +++ /dev/null @@ -1,27 +0,0 @@ ---- -- when: {message_type: KojiBuildPackageCompleted} - do: - - {tasks: [rpmlint, rpmgrill, python-versions]} - -- when: - message_type: KojiBuildPackageCompleted - name: - $nin: [{{ trigger_abicheck_blacklist | join(',') }}] - do: - - {tasks: [abicheck]} - -- when: - message_type: KojiTagChanged - tag: {$regex: '/^f[0-9]{2}-updates(-testing)?-pending$$/'} - do: - - {tasks: [rpmdeplint]} - -- when: {message_type: DistGitCommit, namespace: modules} - do: - - {tasks: [check_modulemd]} - -- when: - message_type: GitHubPullRequestOpened - repo_name: {$regex: '/^container-images\/.+/'} - do: - - {tasks: [mtf-containers]} From 64ccbfe3dea0c0fd18a74c82928545efe25188e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kalu=C5=BEa?= Date: Fri, 19 Jan 2018 10:20:18 +0000 Subject: [PATCH 009/242] Fix typo in raw_config_wrapper.conf --- roles/odcs/base/templates/etc/odcs/raw_config_wrapper.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/odcs/base/templates/etc/odcs/raw_config_wrapper.conf.j2 b/roles/odcs/base/templates/etc/odcs/raw_config_wrapper.conf.j2 index 6d5a35fd19..c498c13c69 100644 --- a/roles/odcs/base/templates/etc/odcs/raw_config_wrapper.conf.j2 +++ b/roles/odcs/base/templates/etc/odcs/raw_config_wrapper.conf.j2 @@ -17,7 +17,7 @@ buildinstall_use_guestmount=False # just takes lot of storage. link_type = 'symlink' -Staging is used only for testing, so allow only x86_64 composes +# Staging is used only for testing, so allow only x86_64 composes tree_arches = ['x86_64'] {% endif %} From c2cd3ae32b3c5f56144dbd16af64c94db94bb04e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kalu=C5=BEa?= Date: Fri, 19 Jan 2018 12:33:53 +0000 Subject: [PATCH 010/242] Enable runroot only when config.bootable is set --- roles/odcs/base/files/pungi.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/odcs/base/files/pungi.conf b/roles/odcs/base/files/pungi.conf index 3b01255259..0519d78760 100644 --- a/roles/odcs/base/files/pungi.conf +++ b/roles/odcs/base/files/pungi.conf @@ -23,9 +23,11 @@ sigkeys = [None] hashed_directories = True # RUNROOT settings +{%- if config.bootable %} runroot = True runroot_tag = "f26-build" runroot_channel = "compose" +{%- endif %} # PDC settings pdc_url = '{{ config.pdc_url }}' From b3df9abc8acd3e3bbb745a504ceb111ff0e18944 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Fri, 19 Jan 2018 13:02:40 +0000 Subject: [PATCH 011/242] Remove eth1 from odcs-backend01 for now. NFS will go over main if Signed-off-by: Patrick Uiterwijk --- inventory/host_vars/odcs-backend01.phx2.fedoraproject.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/host_vars/odcs-backend01.phx2.fedoraproject.org b/inventory/host_vars/odcs-backend01.phx2.fedoraproject.org index 097be7b958..fe5603cb74 100644 --- a/inventory/host_vars/odcs-backend01.phx2.fedoraproject.org +++ b/inventory/host_vars/odcs-backend01.phx2.fedoraproject.org @@ -7,7 +7,7 @@ ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-26 ks_repo: http://10.5.126.23/pub/fedora/linux/releases/26/Server/x86_64/os/ eth0_ip: 10.5.126.65 -eth1_ip: 10.5.127.114 +#eth1_ip: 10.5.127.114 volgroup: /dev/vg_guests vmhost: virthost19.phx2.fedoraproject.org From 55e65fbbaf41a9db5664388b45fd69e016a7837a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kalu=C5=BEa?= Date: Fri, 19 Jan 2018 13:05:49 +0000 Subject: [PATCH 012/242] Set runroot to False when config.bootable is False --- roles/odcs/base/files/pungi.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/odcs/base/files/pungi.conf b/roles/odcs/base/files/pungi.conf index 0519d78760..1b49522fd6 100644 --- a/roles/odcs/base/files/pungi.conf +++ b/roles/odcs/base/files/pungi.conf @@ -27,6 +27,8 @@ hashed_directories = True runroot = True runroot_tag = "f26-build" runroot_channel = "compose" +{%- else %} +runroot = False {%- endif %} # PDC settings From 0433a955da05c07240f4fd77522add2f3bcb37ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kalu=C5=BEa?= Date: Fri, 19 Jan 2018 13:30:30 +0000 Subject: [PATCH 013/242] Allow sgallagh to generate ODCS composes to test modules. --- inventory/group_vars/odcs-frontend | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/inventory/group_vars/odcs-frontend b/inventory/group_vars/odcs-frontend index bb9b350dfb..7907969c39 100644 --- a/inventory/group_vars/odcs-frontend +++ b/inventory/group_vars/odcs-frontend @@ -39,7 +39,9 @@ fedmsg_certs: odcs_target_dir_url: https://odcs.fedoraproject.org/composes # Give access to jscotka to be able to develop module testing integration # for taskotron. -odcs_allowed_clients_users: ["jscotka"] +# Give access to sgallagh to be able to generate testing composes for new +# modules. +odcs_allowed_clients_users: ["jscotka", "sgallagh"] # For the MOTD csi_security_category: Low From 35bb27240efb4b2333eafd4c4237d2bac728d1f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20P=C3=A1ral?= Date: Fri, 19 Jan 2018 14:32:10 +0100 Subject: [PATCH 014/242] taskotron-trigger: make rules more readable But expanding the structured fields. --- .../templates/trigger_rules.yml.j2 | 50 +++++++++++++------ 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2 b/roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2 index c155f5d946..fa6d5cb93f 100644 --- a/roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2 +++ b/roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2 @@ -1,51 +1,71 @@ --- -- when: {message_type: KojiBuildPackageCompleted} +- when: + message_type: KojiBuildPackageCompleted do: - - {tasks: [rpmlint, rpmgrill, python-versions]} + - tasks: + - python-versions + - rpmgrill + - rpmlint - when: message_type: KojiBuildPackageCompleted name: - $nin: [{{ trigger_abicheck_blacklist | join(',') }}] + $nin: + - "{{ trigger_abicheck_blacklist | join(',') }}" do: - - {tasks: [abicheck]} + - tasks: + - abicheck - when: message_type: KojiTagChanged - tag: {$regex: '/^f[0-9]{2}-updates(-testing)?-pending$$/'} + tag: + $regex: '/^f[0-9]{2}-updates(-testing)?-pending$$/' do: - - {tasks: [rpmdeplint]} + - tasks: + - rpmdeplint -- when: {message_type: DistGitCommit, namespace: modules} +- when: + message_type: DistGitCommit + namespace: modules do: - - {tasks: [check_modulemd]} + - tasks: + - check_modulemd - when: message_type: GitHubPullRequestOpened - repo_name: {$regex: '/^container-images\/.+/'} + repo_name: + $regex: '/^container-images\/.+/' do: - - {tasks: [mtf-containers]} + - tasks: + - mtf-containers {% if deployment_type in ['stg', 'prod'] %} {# these tasks are not ansiblized yet #} - when: message_type: ModuleBuildComplete do: - - {tasks: [modularity-testing-framework]} + - tasks: + - modularity-testing-framework -- when: {message_type: KojiBuildPackageCompleted, name: docker} +- when: + message_type: KojiBuildPackageCompleted + name: docker do: - - {tasks: [dockerautotest]} + - tasks: + - dockerautotest {% endif %} {# disabled due to missing nested virt: https://pagure.io/taskotron/issue/239 - when: message_type: AtomicCompose do: - - {tasks: [upstream-atomic, fedora-cloud-tests]} + - tasks: + - upstream-atomic + - fedora-cloud-tests - when: message_type: CloudCompose do: - - {tasks: [fedora-cloud-tests]} + - tasks: + - fedora-cloud-tests #} From 357d6b69d1ef7c8d861cf85d6134523add7f94c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kalu=C5=BEa?= Date: Fri, 19 Jan 2018 13:48:31 +0000 Subject: [PATCH 015/242] Set proper translate_paths for ODCS according to env. --- roles/odcs/base/files/pungi.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/odcs/base/files/pungi.conf b/roles/odcs/base/files/pungi.conf index 1b49522fd6..8cd558e841 100644 --- a/roles/odcs/base/files/pungi.conf +++ b/roles/odcs/base/files/pungi.conf @@ -87,7 +87,11 @@ skip_phases = [ "ostree"] translate_paths = [ +{%- if config.koji_profile == "odcs_stg" %} + ('/mnt/koji/compose/', 'http://kojipkgs.stg.fedoraproject.org/compose/'), +{%- else %} ('/mnt/koji/compose/', 'http://kojipkgs.fedoraproject.org/compose/'), +{%- endif %} ] koji_profile = '{{ config.koji_profile }}' From 476ca97e748c7e950ef3310a4029bb8527a49f0e Mon Sep 17 00:00:00 2001 From: Todd Zullinger Date: Sat, 23 Dec 2017 02:31:58 -0500 Subject: [PATCH 016/242] git (post-receive-alternativearch): use unicode for git output The commit message and diff is stored in `full_change` as a string, via `read_output()`. This is passed to `TEXT` which is a unicode string. When a commit message or diff contains non-ascii characters we get: UnicodeDecodeError: 'ascii' codec can't decode byte ... Encode git output returned from `read_output()` as a unicode string and define `full_change` as unicode for completeness. Fixes #6040. --- roles/git/hooks/files/post-receive-alternativearch | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/git/hooks/files/post-receive-alternativearch b/roles/git/hooks/files/post-receive-alternativearch index 16d8cef4dd..7bc4a9a6a0 100755 --- a/roles/git/hooks/files/post-receive-alternativearch +++ b/roles/git/hooks/files/post-receive-alternativearch @@ -66,7 +66,7 @@ def read_output(cmd, abspath, input=None, keepends=False, **kw): print(err) if not keepends: out = out.rstrip('\n\r') - return out + return out.decode('utf-8') def read_git_output(args, abspath, input=None, keepends=False, **kw): @@ -164,7 +164,7 @@ def run_as_post_receive_hook(): if DEBUG: print('List of commits:', new_commits_list) - full_change = '' + full_change = u'' exclude_arch = {} for commit in new_commits_list: if DEBUG: From b53181dce936571287413a50f6d93830a11ae541 Mon Sep 17 00:00:00 2001 From: Dan Callaghan Date: Sat, 20 Jan 2018 01:59:54 +1000 Subject: [PATCH 017/242] greenwave: use 8 gunicorn workers instead of the default 1 Trying to avoid Openshift killing the Greenwave pods because of the liveness check timing out, if it gets stuck behind other slow HTTP requests. https://pagure.io/greenwave/issue/116 --- roles/openshift-apps/greenwave/templates/buildconfig.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/openshift-apps/greenwave/templates/buildconfig.yml b/roles/openshift-apps/greenwave/templates/buildconfig.yml index b81da5fba5..af3e79f65d 100644 --- a/roles/openshift-apps/greenwave/templates/buildconfig.yml +++ b/roles/openshift-apps/greenwave/templates/buildconfig.yml @@ -34,7 +34,7 @@ spec: RUN chmod 777 /var/run/fedmsg/ ENV USER=openshift EXPOSE 8080 - ENTRYPOINT gunicorn --bind 0.0.0.0:8080 --access-logfile=- greenwave.wsgi:app + ENTRYPOINT gunicorn --workers 8 --bind 0.0.0.0:8080 --access-logfile=- greenwave.wsgi:app strategy: type: Docker output: From d96ccc806d57a1763d3d728587af2e5c017c338b Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Fri, 19 Jan 2018 21:06:53 +0000 Subject: [PATCH 018/242] add libkcapi to increase a kernel limit on arm builders. Ticket 6636 --- roles/koji_builder/tasks/main.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/roles/koji_builder/tasks/main.yml b/roles/koji_builder/tasks/main.yml index 2cc469a234..038e3efb17 100644 --- a/roles/koji_builder/tasks/main.yml +++ b/roles/koji_builder/tasks/main.yml @@ -287,3 +287,10 @@ tags: - koji_builder when: env == "staging" + +# https://pagure.io/fedora-infrastructure/issue/6636 +- name: install libkcapi to get increased sockets on armv7 + dnf: name=libkcapi enablerepo=updates-testing state=present + tags: + - koji_builder + when: ansible_architecture == 'armv7l' From 0b138f9111dd93b3f7f405db1c7d2a48a3cefd74 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Fri, 19 Jan 2018 21:32:19 +0000 Subject: [PATCH 019/242] add mdapi and greenwave monitoring. tickets 6639 and 6643 --- .../templates/check_datanommer_history.cfg.j2 | 2 ++ roles/nagios_server/files/nagios/services/fedmsg.cfg | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/roles/nagios_client/templates/check_datanommer_history.cfg.j2 b/roles/nagios_client/templates/check_datanommer_history.cfg.j2 index 2b1c6cbbec..85bacf91af 100644 --- a/roles/nagios_client/templates/check_datanommer_history.cfg.j2 +++ b/roles/nagios_client/templates/check_datanommer_history.cfg.j2 @@ -41,6 +41,8 @@ command[check_datanommer_faf]={{libdir}}/nagios/plugins/check_datanommer_timesin command[check_datanommer_koschei]={{libdir}}/nagios/plugins/check_datanommer_timesince.py koschei 86400 604800 command[check_datanommer_autocloud]={{libdir}}/nagios/plugins/check_datanommer_timesince.py autocloud 259200 1814400 command[check_datanommer_twoweekatomic]=/usr/lib64/nagios/plugins/check_datanommer_timesince.py org.fedoraproject.prod.releng.atomic.twoweek.complete 1296000 1382400 +command[check_datanommer_mdapi]=/usr/lib64/nagios/plugins/check_datanommer_timesince.py mdapi 14400 86400 +command[check_datanommer_greenwave]=/usr/lib64/nagios/plugins/check_datanommer_timesince.py greenwave 14400 86400 # This one is retired since it times out all the time. Too few messages. #command[check_datanommer_nuancier]={{libdir}}/nagios/plugins/check_datanommer_timesince.py nuancier 23652000 31536000 diff --git a/roles/nagios_server/files/nagios/services/fedmsg.cfg b/roles/nagios_server/files/nagios/services/fedmsg.cfg index d7eb667e17..de71e68fd7 100644 --- a/roles/nagios_server/files/nagios/services/fedmsg.cfg +++ b/roles/nagios_server/files/nagios/services/fedmsg.cfg @@ -347,6 +347,18 @@ define service { check_command check_by_nrpe!check_datanommer_twoweekatomic use defaulttemplate } +define service { + host_name busgateway01.phx2.fedoraproject.org + service_description Check datanommer for recent mdapi messages + check_command check_by_nrpe!check_datanommer_mdapi + use defaulttemplate +} +define service { + host_name busgateway01.phx2.fedoraproject.org + service_description Check datanommer for recent greenwave messages + check_command check_by_nrpe!check_datanommer_greenwave + use defaulttemplate +} # BEGIN, check consumers and producers From ab455cd57d58a79ccaa1ff1771413f06e00bc3fb Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Fri, 19 Jan 2018 23:40:00 +0000 Subject: [PATCH 020/242] Remove proxy07 from inventory for now so it doesn't get tkey Signed-off-by: Patrick Uiterwijk --- inventory/inventory | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inventory/inventory b/inventory/inventory index 01f4f93c13..1d4eeda9b2 100644 --- a/inventory/inventory +++ b/inventory/inventory @@ -496,7 +496,7 @@ proxy03.fedoraproject.org proxy04.fedoraproject.org proxy05.fedoraproject.org proxy06.fedoraproject.org -proxy07.fedoraproject.org +#proxy07.fedoraproject.org proxy08.fedoraproject.org proxy09.fedoraproject.org proxy10.phx2.fedoraproject.org @@ -656,7 +656,7 @@ proxy03.fedoraproject.org proxy04.fedoraproject.org proxy05.fedoraproject.org proxy06.fedoraproject.org -proxy07.fedoraproject.org +#proxy07.fedoraproject.org proxy08.fedoraproject.org proxy09.fedoraproject.org proxy10.phx2.fedoraproject.org From 8c816c031e3913f93de0569d58692c0a88ec67e3 Mon Sep 17 00:00:00 2001 From: clime Date: Sat, 20 Jan 2018 18:54:57 +0100 Subject: [PATCH 021/242] copr-frontend: reeanble alembic upgrade head --- roles/copr/frontend/tasks/main.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/roles/copr/frontend/tasks/main.yml b/roles/copr/frontend/tasks/main.yml index b20e2043a4..78e9f4dc33 100644 --- a/roles/copr/frontend/tasks/main.yml +++ b/roles/copr/frontend/tasks/main.yml @@ -21,6 +21,7 @@ tags: - packages + # we install python-alembic because https://bugzilla.redhat.com/show_bug.cgi?id=1536058 - name: install additional pkgs for copr-frontend dnf: state=present pkg={{ item }} with_items: @@ -28,6 +29,7 @@ - "mod_ssl" - redis - pxz + - python-alembic tags: - packages @@ -60,12 +62,12 @@ - import_tasks: "psql_setup.yml" -#- name: upgrade db to head -# command: alembic upgrade head -# become: yes -# become_user: copr-fe -# args: -# chdir: /usr/share/copr/coprs_frontend/ +- name: upgrade db to head + command: alembic upgrade head + become: yes + become_user: copr-fe + args: + chdir: /usr/share/copr/coprs_frontend/ - name: set up admins command: ./manage.py alter_user --admin {{ item }} From ead8042cfd17b2c905f7cf71267daf454d3d15d5 Mon Sep 17 00:00:00 2001 From: clime Date: Sat, 20 Jan 2018 19:13:01 +0100 Subject: [PATCH 022/242] copr-frontend: requiring certain versions of a package should be in copr-frontend.spec --- roles/copr/frontend/tasks/main.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/roles/copr/frontend/tasks/main.yml b/roles/copr/frontend/tasks/main.yml index 78e9f4dc33..b708a34dd7 100644 --- a/roles/copr/frontend/tasks/main.yml +++ b/roles/copr/frontend/tasks/main.yml @@ -16,11 +16,6 @@ tags: - packages -- name: ensure python2-flask-whooshee is latest - dnf: state=latest name=python2-flask-whooshee - tags: - - packages - # we install python-alembic because https://bugzilla.redhat.com/show_bug.cgi?id=1536058 - name: install additional pkgs for copr-frontend dnf: state=present pkg={{ item }} From cf6dc8a3487f7b75f0d7a0d07137519562c4ff48 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Mon, 22 Jan 2018 14:23:34 +0000 Subject: [PATCH 023/242] mbs: change REBUILD_STRATEGY from the default 'changed-and-after' to 'only-changed' --- roles/mbs/common/templates/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/mbs/common/templates/config.py b/roles/mbs/common/templates/config.py index a57a6351b1..97a909739e 100644 --- a/roles/mbs/common/templates/config.py +++ b/roles/mbs/common/templates/config.py @@ -96,6 +96,7 @@ class ProdConfiguration(BaseConfiguration): 'releng', ] + REBUILD_STRATEGY = 'only-changed' REBUILD_STRATEGY_ALLOW_OVERRIDE = True {% if env == 'staging' %} From 50e65f4d47cf0c04114f5b715d1e634e204ad940 Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Tue, 23 Jan 2018 13:18:28 +0000 Subject: [PATCH 024/242] Update the edit-badge script to edit the badge tags --- roles/badges/backend/files/edit-badge | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/roles/badges/backend/files/edit-badge b/roles/badges/backend/files/edit-badge index 2b9079c421..acfffb8951 100644 --- a/roles/badges/backend/files/edit-badge +++ b/roles/badges/backend/files/edit-badge @@ -27,12 +27,13 @@ def parse_args(): parser.add_argument('--description', default=None, help='Description..') parser.add_argument('--criteria', default=None, help='Criteria link') parser.add_argument('--image', default=None, help='Image link') + parser.add_argument('--tags', default=None, help='Badge Tags') args = parser.parse_args() if not args.badge: print "You must specify a badge id." sys.exit(1) - if not args.name and not args.description and not args.criteria and not args.image: - print "You must specify either name, description or criteria or image to edit." + if not args.name and not args.description and not args.criteria and not args.image and not args.tags: + print "You must specify either name, description or criteria, tags or image to edit." sys.exit(1) return args @@ -51,7 +52,7 @@ def initialize(): return tahrir -def main(tahrir, badge_id, name, description, criteria, image): +def main(tahrir, badge_id, name, description, criteria, image, tags): badge = tahrir.get_badge(badge_id) if not badge: @@ -75,6 +76,11 @@ def main(tahrir, badge_id, name, description, criteria, image): if image: badge.image = image print "Setting image on %r to %r" % (badge_id, image) + + if tags: + badge.tags = tags + print "Setting tags on %r to %r" % (badge_id, tags) + tahrir.session.commit() transaction.commit() @@ -82,4 +88,5 @@ def main(tahrir, badge_id, name, description, criteria, image): if __name__ == '__main__': args = parse_args() tahrir = initialize() - main(tahrir, args.badge, args.name, args.description, args.criteria, args.image) + main(tahrir, args.badge, args.name, args.description, args.criteria, + args.image, args.tags) From 2ee431319bb840bb870894c55ac1e86d201f8acc Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Tue, 23 Jan 2018 17:08:17 +0000 Subject: [PATCH 025/242] Add epel releases to the greenwave policy. --- roles/openshift-apps/greenwave/templates/configmap.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/openshift-apps/greenwave/templates/configmap.yml b/roles/openshift-apps/greenwave/templates/configmap.yml index b80fbe5ba4..2b4fd4c0da 100644 --- a/roles/openshift-apps/greenwave/templates/configmap.yml +++ b/roles/openshift-apps/greenwave/templates/configmap.yml @@ -94,6 +94,8 @@ data: - fedora-27 - fedora-26 - fedora-25 + - fedora-epel-7 + - fedora-epel-6 decision_context: bodhi_update_push_testing blacklist: [] relevance_value: koji_build @@ -105,6 +107,8 @@ data: - fedora-27 - fedora-26 - fedora-25 + - fedora-epel-7 + - fedora-epel-6 decision_context: bodhi_update_push_stable blacklist: [] relevance_value: koji_build @@ -116,6 +120,8 @@ data: - fedora-27 - fedora-26 - fedora-25 + - fedora-epel-7 + - fedora-epel-6 decision_context: bodhi_update_push_stable relevance_value: koji_build # abicheck only runs on a subset of all packages. We borrow the list from From 465f155d140a9fbe34f0f51dbfc2137b2900a6f8 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Tue, 23 Jan 2018 17:16:34 +0000 Subject: [PATCH 026/242] Remove the abicheck from greenwave's policy. We have some test run issues to sort out first. --- .../greenwave/templates/configmap.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/roles/openshift-apps/greenwave/templates/configmap.yml b/roles/openshift-apps/greenwave/templates/configmap.yml index 2b4fd4c0da..15e2680a42 100644 --- a/roles/openshift-apps/greenwave/templates/configmap.yml +++ b/roles/openshift-apps/greenwave/templates/configmap.yml @@ -115,25 +115,6 @@ data: rules: - !PassingTestCaseRule {test_case_name: dist.rpmdeplint} --- !Policy - id: "taskotron_release_critical_tasks_for_stable_with_blacklist" - product_versions: - - fedora-27 - - fedora-26 - - fedora-25 - - fedora-epel-7 - - fedora-epel-6 - decision_context: bodhi_update_push_stable - relevance_value: koji_build - # abicheck only runs on a subset of all packages. We borrow the list from - # taskotron's ansible vars. See discussion in https://pagure.io/greenwave/issue/68 -{% if env == 'staging' %} - blacklist: [{{ hostvars[groups['taskotron-stg'][0]]['trigger_abicheck_blacklist'] | join (',') }}] -{% else %} - blacklist: [{{ hostvars[groups['taskotron-prod'][0]]['trigger_abicheck_blacklist'] | join (',') }}] -{% endif %} - rules: - - !PassingTestCaseRule {test_case_name: dist.abicheck} - --- !Policy # Fedora Atomic CI pipeline # http://fedoraproject.org/wiki/CI id: "atomic_ci_pipeline_results" From 87b84c41ca0d60fcbd0cd1694c9ef21b8c5b6bc2 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Wed, 24 Jan 2018 06:03:36 +0000 Subject: [PATCH 027/242] Deploy fedoradocsredirect in staging mediawiki Signed-off-by: Patrick Uiterwijk --- roles/mediawiki/tasks/main.yml | 6 ++++++ roles/mediawiki/templates/LocalSettings.php.fp.j2 | 3 +++ 2 files changed, 9 insertions(+) diff --git a/roles/mediawiki/tasks/main.yml b/roles/mediawiki/tasks/main.yml index ac6608f629..00838a0e8e 100644 --- a/roles/mediawiki/tasks/main.yml +++ b/roles/mediawiki/tasks/main.yml @@ -53,6 +53,12 @@ - packages - mediawiki +- name: Install mediawiki-fedoradocsredirect on staging + package: name=mediawiki-fedoradocsredirect state=present + tags: + - packages + - mediawiki + - name: adding fedmsg emit copy: src=fedmsg-emit.php dest=/usr/share/{{ wikiver }}/extensions/fedmsg-emit.php owner=root group=root mode=775 tags: diff --git a/roles/mediawiki/templates/LocalSettings.php.fp.j2 b/roles/mediawiki/templates/LocalSettings.php.fp.j2 index 78ffe5eab0..4ca78776e3 100644 --- a/roles/mediawiki/templates/LocalSettings.php.fp.j2 +++ b/roles/mediawiki/templates/LocalSettings.php.fp.j2 @@ -328,6 +328,9 @@ require_once "$IP/extensions/fedmsg-emit.php"; require_once "$IP/extensions/HTTP302Found/HTTP302Found.php"; require_once "$IP/extensions/RSS/RSS.php"; require_once "$IP/extensions/BassetSubmitter.php"; +{% if env == "staging" %} +require_once "$IP/extensions/FedoraDocsRedirect/FedoraDocsRedirect.php"; +{% endif %} {% if env == "staging" %} $basset_url = 'http://basset01.stg.phx2.fedoraproject.org/basset'; From baa418b6927dfdd080cf1380cfcf9bee332f58c7 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Wed, 24 Jan 2018 06:20:08 +0000 Subject: [PATCH 028/242] Only add pkg on staging for now Signed-off-by: Patrick Uiterwijk --- roles/mediawiki/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/mediawiki/tasks/main.yml b/roles/mediawiki/tasks/main.yml index 00838a0e8e..18977b682f 100644 --- a/roles/mediawiki/tasks/main.yml +++ b/roles/mediawiki/tasks/main.yml @@ -55,6 +55,7 @@ - name: Install mediawiki-fedoradocsredirect on staging package: name=mediawiki-fedoradocsredirect state=present + when: env == "staging" tags: - packages - mediawiki From 09f15fbc783ef77e8eb245ae495a41b371b091d0 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 24 Jan 2018 08:40:17 +0000 Subject: [PATCH 029/242] Greenwave should require nothing for EPEL. --- .../greenwave/templates/configmap.yml | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/roles/openshift-apps/greenwave/templates/configmap.yml b/roles/openshift-apps/greenwave/templates/configmap.yml index 15e2680a42..67bbd3d1a1 100644 --- a/roles/openshift-apps/greenwave/templates/configmap.yml +++ b/roles/openshift-apps/greenwave/templates/configmap.yml @@ -94,8 +94,6 @@ data: - fedora-27 - fedora-26 - fedora-25 - - fedora-epel-7 - - fedora-epel-6 decision_context: bodhi_update_push_testing blacklist: [] relevance_value: koji_build @@ -107,8 +105,6 @@ data: - fedora-27 - fedora-26 - fedora-25 - - fedora-epel-7 - - fedora-epel-6 decision_context: bodhi_update_push_stable blacklist: [] relevance_value: koji_build @@ -185,3 +181,21 @@ data: greenwave_api_url='https://greenwave-web-greenwave.app.os.fedoraproject.org/api/v1.0' {% endif %} ) + --- !Policy + id: "no_requirements_for_epel_testing" + product_versions: + - fedora-epel-7 + - fedora-epel-6 + decision_context: bodhi_update_push_testing + blacklist: [] + relevance_value: koji_build + rules: [] + --- !Policy + id: "no_requirements_for_epel_stable" + product_versions: + - fedora-epel-7 + - fedora-epel-6 + decision_context: bodhi_update_push_stable + blacklist: [] + relevance_value: koji_build + rules: [] From e7accbe0664a15f3bbb0d7bc759f806da3e0a6ff Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 24 Jan 2018 08:53:02 +0000 Subject: [PATCH 030/242] Move this to the right place in the file. --- .../greenwave/templates/configmap.yml | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/roles/openshift-apps/greenwave/templates/configmap.yml b/roles/openshift-apps/greenwave/templates/configmap.yml index 67bbd3d1a1..6f882b3573 100644 --- a/roles/openshift-apps/greenwave/templates/configmap.yml +++ b/roles/openshift-apps/greenwave/templates/configmap.yml @@ -111,6 +111,24 @@ data: rules: - !PassingTestCaseRule {test_case_name: dist.rpmdeplint} --- !Policy + id: "no_requirements_for_epel_testing" + product_versions: + - fedora-epel-7 + - fedora-epel-6 + decision_context: bodhi_update_push_testing + blacklist: [] + relevance_value: koji_build + rules: [] + --- !Policy + id: "no_requirements_for_epel_stable" + product_versions: + - fedora-epel-7 + - fedora-epel-6 + decision_context: bodhi_update_push_stable + blacklist: [] + relevance_value: koji_build + rules: [] + --- !Policy # Fedora Atomic CI pipeline # http://fedoraproject.org/wiki/CI id: "atomic_ci_pipeline_results" @@ -181,21 +199,3 @@ data: greenwave_api_url='https://greenwave-web-greenwave.app.os.fedoraproject.org/api/v1.0' {% endif %} ) - --- !Policy - id: "no_requirements_for_epel_testing" - product_versions: - - fedora-epel-7 - - fedora-epel-6 - decision_context: bodhi_update_push_testing - blacklist: [] - relevance_value: koji_build - rules: [] - --- !Policy - id: "no_requirements_for_epel_stable" - product_versions: - - fedora-epel-7 - - fedora-epel-6 - decision_context: bodhi_update_push_stable - blacklist: [] - relevance_value: koji_build - rules: [] From 858fcf681b362221fd36c8496e6cc101e940994d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Wed, 24 Jan 2018 10:06:34 +0100 Subject: [PATCH 031/242] copr-be-dev-data volume is back again, so we can mount it --- roles/copr/backend/tasks/mount_fs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/copr/backend/tasks/mount_fs.yml b/roles/copr/backend/tasks/mount_fs.yml index 3e3cbcb248..bbd1411dc7 100644 --- a/roles/copr/backend/tasks/mount_fs.yml +++ b/roles/copr/backend/tasks/mount_fs.yml @@ -3,7 +3,6 @@ - name: mount up disk of copr repo mount: name=/var/lib/copr/public_html src='LABEL=copr-repo' fstype=ext4 state=mounted - when: env != "staging" - name: mount /tmp/ mount: name=/tmp src='tmpfs' fstype=tmpfs state=mounted From 8f2dd30ae2e5f1eaa545c6971ec6a2b346212fdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20P=C3=A1ral?= Date: Wed, 24 Jan 2018 13:36:39 +0100 Subject: [PATCH 032/242] taskotron-buildmaster: send TERM signal before KILL Also follow heartbeat.log. --- .../templates/taskotron.master.cfg.j2 | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index c052754869..41418cf258 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -214,18 +214,24 @@ factory.addStep(ShellCommand(command=["runtask", {% endif %} descriptionDone=[Interpolate('%(prop:taskname)s on %(prop:item)s')], name='runtask', - timeout=2400, -{% if deployment_type in ['dev', 'stg', 'prod'] %} + timeout=20*60, +{% if deployment_type in ['dev'] %} + sigtermTime=5*60, + logfiles={ + 'heartbeat.log': {'filename': Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/tests.yml/taskotron/heartbeat.log')} # FIXME: tests.yml hardcoded + } +{% endif %} +{% if deployment_type in ['stg', 'prod'] %} logfiles={ 'taskotron-overlord.log': {'filename': Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/taskotron-overlord.log')}, 'taskotron-stdio.log': {'filename': Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/taskotron-stdio.log')}, 'taskotron.log': {'filename': Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/taskotron.log')} } -)) {% endif %} {% if deployment_type in ['local'] %} - logfiles={'taskotron.log': {'filename': '/var/log/taskotron/taskotron.log', }})) + logfiles={'taskotron.log': {'filename': '/var/log/taskotron/taskotron.log', }} {% endif %} +)) factory.addStep(ShellCommand(command=Interpolate('testcloud instance remove --force taskotron-%(prop:uuid)s; true'), From c89d481d51f93dc297b8d625cd2ee56e1a9aaf1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Wed, 24 Jan 2018 13:27:09 +0000 Subject: [PATCH 033/242] Hubs: update the playbook --- .../hosts/hubs-dev.fedorainfracloud.org.yml | 4 +- roles/hubs/defaults/main.yml | 8 +- roles/hubs/files/logging.ini | 9 +- roles/hubs/handlers/main.yml | 17 ++- roles/hubs/handlers/webserver.yml | 7 - roles/hubs/tasks/db-postgresql.yml | 2 +- roles/hubs/tasks/dev.yml | 30 ++++ roles/hubs/tasks/main.yml | 133 ++++++------------ roles/hubs/tasks/prod.yml | 28 ++++ roles/hubs/tasks/webserver.yml | 22 +-- roles/hubs/templates/bashrc | 14 +- roles/hubs/templates/fedmsg_config | 9 +- roles/hubs/templates/honcho-env | 3 + roles/hubs/templates/honcho-procfile | 7 + .../hubs/templates/hubs-fedmsg-relay.service | 14 ++ 15 files changed, 180 insertions(+), 127 deletions(-) delete mode 100644 roles/hubs/handlers/webserver.yml create mode 100644 roles/hubs/tasks/dev.yml create mode 100644 roles/hubs/tasks/prod.yml create mode 100644 roles/hubs/templates/honcho-env create mode 100644 roles/hubs/templates/honcho-procfile create mode 100644 roles/hubs/templates/hubs-fedmsg-relay.service diff --git a/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml b/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml index 6d1b6265ed..36386dd71a 100644 --- a/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml +++ b/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml @@ -55,10 +55,12 @@ main_user: fedora hubs_url_hostname: "{{ ansible_fqdn }}" hubs_secret_key: demotestinghubsmachine - hubs_db_type: sqlite + hubs_db_type: postgresql hubs_dev_mode: false hubs_ssl_cert: /etc/letsencrypt/live/{{ ansible_fqdn }}/fullchain.pem hubs_ssl_key: /etc/letsencrypt/live/{{ ansible_fqdn }}/privkey.pem + hubs_fas_username: "{{ fedoraDummyUser }}", + hubs_fas_password: "{{ fedoraDummyUserPassword }}", tasks: diff --git a/roles/hubs/defaults/main.yml b/roles/hubs/defaults/main.yml index 2ee32214e4..f3d14cb644 100644 --- a/roles/hubs/defaults/main.yml +++ b/roles/hubs/defaults/main.yml @@ -8,7 +8,9 @@ hubs_venv_dir: "{{ hubs_base_dir }}/venv" hubs_var_dir: "{{ hubs_base_dir }}/var" hubs_db_type: sqlite hubs_db_password: changeme -hubs_url_hostname: localhost +hubs_url_hostname: "{{ ansible_fqdn }}" hubs_url: http{% if not hubs_dev_mode %}s{% endif %}://{{ hubs_url_hostname }}{% if hubs_dev_mode %}:5000{% endif %} -hubs_ssl_cert: /etc/pki/tls/certs/localhost.crt -hubs_ssl_key: /etc/pki/tls/private/localhost.key +hubs_ssl_cert: /etc/pki/tls/certs/{{ hubs_url_hostname }}.crt +hubs_ssl_key: /etc/pki/tls/private/{{ hubs_url_hostname }}.key +hubs_fas_username: changeme +hubs_fas_password: changeme diff --git a/roles/hubs/files/logging.ini b/roles/hubs/files/logging.ini index 3512fa6ee9..f1a3bc0ddd 100644 --- a/roles/hubs/files/logging.ini +++ b/roles/hubs/files/logging.ini @@ -1,4 +1,3 @@ -# From https://docs.python.org/2/howto/logging.html [loggers] keys=root @@ -6,7 +5,7 @@ keys=root keys=console [formatters] -keys=simple +keys=simple,minimal [logger_root] level=DEBUG @@ -15,9 +14,13 @@ handlers=console [handler_console] class=StreamHandler level=DEBUG -formatter=simple +formatter=minimal args=(sys.stdout,) [formatter_simple] format=[%(asctime)s][%(process)d][%(levelname)s] (%(name)s) %(message)s datefmt=%H:%M:%S + +[formatter_minimal] +format=[%(levelname)s] (%(name)s) %(message)s +datefmt=%H:%M:%S diff --git a/roles/hubs/handlers/main.yml b/roles/hubs/handlers/main.yml index c6935af51c..f71ee8fd8c 100644 --- a/roles/hubs/handlers/main.yml +++ b/roles/hubs/handlers/main.yml @@ -4,18 +4,33 @@ - name: restart the hubs-specific fedmsg-hub service: name=hubs-fedmsg-hub state=restarted listen: "hubs configuration change" + when: not hubs_dev_mode + +- name: restart the hubs-specific fedmsg-relay + service: name=hubs-fedmsg-relay state=restarted + listen: "hubs configuration change" + when: not hubs_dev_mode - name: restart hubs triage service: name=hubs-triage@* state=restarted listen: "hubs configuration change" + when: not hubs_dev_mode - name: restart hubs workers service: name=hubs-worker@* state=restarted listen: "hubs configuration change" + when: not hubs_dev_mode - name: restart hubs SSE server service: name=hubs-sse state=restarted listen: "hubs configuration change" + when: not hubs_dev_mode # Webserver -- import_tasks: webserver.yml +- name: restart hubs webapp + service: name=hubs-webapp state=restarted + listen: "hubs configuration change" + when: not hubs_dev_mode + +- name: restart nginx + service: name=nginx state=restarted diff --git a/roles/hubs/handlers/webserver.yml b/roles/hubs/handlers/webserver.yml deleted file mode 100644 index e6cb871791..0000000000 --- a/roles/hubs/handlers/webserver.yml +++ /dev/null @@ -1,7 +0,0 @@ -- name: restart hubs webapp - service: name=hubs-webapp state=restarted - listen: "hubs configuration change" - when: not hubs_dev_mode - -- name: restart nginx - service: name=nginx state=restarted diff --git a/roles/hubs/tasks/db-postgresql.yml b/roles/hubs/tasks/db-postgresql.yml index e8a560105c..2a9eec915c 100644 --- a/roles/hubs/tasks/db-postgresql.yml +++ b/roles/hubs/tasks/db-postgresql.yml @@ -52,5 +52,5 @@ environment: HUBS_CONFIG: "{{ hubs_conf_dir }}/hubs_config.py" become_user: "{{ main_user }}" - when: db_creation|succeeded + when: db_creation|succeeded and db_creation|changed diff --git a/roles/hubs/tasks/dev.yml b/roles/hubs/tasks/dev.yml new file mode 100644 index 0000000000..6b01c9b8fc --- /dev/null +++ b/roles/hubs/tasks/dev.yml @@ -0,0 +1,30 @@ +- name: Install Fedora Hubs development packages + dnf: name={{ item }} state=present + with_items: + - gcc + - gcc-c++ + - libffi-devel + - openssl-devel + - python-sphinx + - python2-devel + - python3-devel + - python3-honcho + - python3-tox + - redhat-rpm-config + - sqlite-devel + +- name: Install a custom bashrc + template: src=bashrc dest=/home/{{ main_user }}/.bashrc + +- name: Install Honcho's env file + template: src=honcho-env dest={{ hubs_base_dir }}/.env + +- name: Install Honcho's procfile + template: src=honcho-procfile dest={{ hubs_base_dir }}/Procfile + +- name: Link to the FAS credentials file if any + file: + state: link + path: "/etc/fedmsg.d/fas_credentials.py" + src: "{{ hubs_code_dir }}/fedmsg.d/fas_credentials.py" + notify: "hubs configuration change" diff --git a/roles/hubs/tasks/main.yml b/roles/hubs/tasks/main.yml index a06608abee..db877d2c8d 100644 --- a/roles/hubs/tasks/main.yml +++ b/roles/hubs/tasks/main.yml @@ -11,54 +11,33 @@ - npm - redis - fedmsg-hub - - python-virtualenv + - fedmsg-relay + - python3-virtualenv - python3-flask-oidc - -- name: Install Fedora Hubs development packages - dnf: name={{ item }} state=present - with_items: - - gcc - - gcc-c++ - - libffi-devel - - openssl-devel - - python-sphinx - - python2-devel - - python3-devel - - redhat-rpm-config - - sqlite-devel - when: hubs_dev_mode + - postfix - name: Install the distribution versions of requirements.txt dnf: name={{ item }} state=present with_items: - - python-alembic - - python-arrow - - python-bleach - - python-decorator - - python-dogpile-cache - - python-fedmsg-core - - python-fedmsg-meta-fedora-infrastructure - - python-flask - - python-flask-oidc - - python-fmn-lib - - python-fmn-rules - - python-futures - - python-html5lib - - python-munch - - pytz - - python-sqlalchemy - - python-markdown - - python2-pkgwat-api - - python-six - - python-pygments - - python-pygments-markdown-lexer - - python-retask - - -# Add various helpful configuration files -- name: Install a custom bashrc - template: src=bashrc dest=/home/{{ main_user }}/.bashrc - when: hubs_dev_mode + - python3-alembic + - python3-arrow + - python3-bleach + - python3-decorator + - python3-dogpile-cache + - python3-fedmsg-core + - python3-fedmsg-meta-fedora-infrastructure + - python3-flask + - python3-flask-oidc + - python3-html5lib + - python3-munch + - python3-pytz + - python3-sqlalchemy + - python3-markdown + - python3-pkgwat-api + - python3-six + - python3-pygments + - python3-pygments-markdown-lexer + - python3-retask # Create directory structure @@ -84,6 +63,7 @@ requirements: "{{ hubs_code_dir }}/requirements.txt" virtualenv: "{{ hubs_venv_dir}}" virtualenv_site_packages: yes + virtualenv_command: virtualenv-3 - name: Install Fedora Hubs test-requirements.txt into hubs virtualenv become_user: "{{ main_user }}" @@ -91,6 +71,7 @@ requirements: "{{ hubs_code_dir }}/test-requirements.txt" virtualenv: "{{ hubs_venv_dir}}" virtualenv_site_packages: yes + virtualenv_command: virtualenv-3 - name: Install other packages into hubs virtualenv become_user: "{{ main_user }}" @@ -98,6 +79,7 @@ name: "{{ item }}" virtualenv: "{{ hubs_venv_dir }}" virtualenv_site_packages: yes + virtualenv_command: virtualenv-3 with_items: - bleach @@ -105,7 +87,7 @@ become_user: "{{ main_user }}" command: "{{ hubs_venv_dir }}/bin/pip install -e {{ hubs_code_dir }}" args: - creates: "{{ hubs_venv_dir }}/lib/python2.7/site-packages/fedora-hubs.egg-link" + creates: "{{ hubs_venv_dir }}/lib/python3.6/site-packages/fedora-hubs.egg-link" - name: Set bin file context in the virtualenv become_user: "{{ main_user }}" @@ -144,8 +126,14 @@ creates: "{{ hubs_conf_dir }}/client_secrets.json" +- name: Start and enable the common services + service: name={{ item }} state=started enabled=yes + with_items: + - redis + - postfix + # Set up, create, and populate the database. -- import_tasks: db-{{ hubs_db_type }}.yml +- include_tasks: db-{{ hubs_db_type }}.yml # Set up JavaScript requirements @@ -156,7 +144,7 @@ creates: node_modules chdir: "{{ hubs_code_dir }}/hubs/static/client" -- name: Build JavaScript assests +- name: Build JavaScript assets command: npm run build become_user: "{{ main_user }}" args: @@ -164,51 +152,18 @@ creates: "{{ hubs_code_dir }}/hubs/static/js/build/common.js" -- name: Fix permissions if necessary - file: - path: "{{ item }}" - state: directory - owner: "{{ main_user }}" - group: "{{ main_user }}" - recurse: yes - #setype: httpd_sys_content_rw_t - with_items: - - "{{ hubs_base_dir }}" - - "{{ hubs_conf_dir }}" - - "{{ hubs_var_dir }}" - - # Services -- name: Disable the system-wide fedmsg-hub - service: name=fedmsg-hub state=stopped enabled=no - -- name: Install the service files - template: - src: "{{ item }}.service" - dest: /etc/systemd/system/{{ item }}.service +- name: Disable the system-wide fedmsg daemons + service: name={{ item }} state=stopped enabled=no with_items: - - hubs-triage@ - - hubs-worker@ - - hubs-sse - - hubs-fedmsg-hub - register: service_installed - -- name: reload systemd - command: systemctl daemon-reload - when: service_installed|changed - -- name: Start and enable the services - service: name={{ item }} state=started enabled=yes - with_items: - - redis - - hubs-triage@1 - - hubs-triage@2 - - hubs-worker@1 - - hubs-worker@2 - - hubs-sse - - hubs-fedmsg-hub + - fedmsg-hub + - fedmsg-relay -# Webserver -- import_tasks: webserver.yml +# Include mode-specific tasks + +- include_tasks: dev.yml + when: hubs_dev_mode + +- include_tasks: prod.yml when: not hubs_dev_mode diff --git a/roles/hubs/tasks/prod.yml b/roles/hubs/tasks/prod.yml new file mode 100644 index 0000000000..0cc4894eab --- /dev/null +++ b/roles/hubs/tasks/prod.yml @@ -0,0 +1,28 @@ +- name: Install the service files + template: + src: "{{ item }}.service" + dest: /etc/systemd/system/{{ item }}.service + with_items: + - hubs-triage@ + - hubs-worker@ + - hubs-sse + - hubs-fedmsg-hub + - hubs-fedmsg-relay + register: service_installed + +- name: reload systemd + command: systemctl daemon-reload + when: service_installed|changed + +- name: Start and enable the services in prod mode + service: name={{ item }} state=started enabled=yes + with_items: + - hubs-triage@1 + - hubs-triage@2 + - hubs-worker@1 + - hubs-worker@2 + - hubs-sse + - hubs-fedmsg-hub + - hubs-fedmsg-relay + +- include_tasks: webserver.yml diff --git a/roles/hubs/tasks/webserver.yml b/roles/hubs/tasks/webserver.yml index 3db6d340e4..3aea933ed2 100644 --- a/roles/hubs/tasks/webserver.yml +++ b/roles/hubs/tasks/webserver.yml @@ -8,6 +8,17 @@ - libsemanage-python +- name: install python2-certbot-nginx + dnf: name=python2-certbot-nginx state=present + +- name: get the letencrypt cert + command: certbot certonly -n --nginx -d {{ ansible_fqdn }} --agree-tos --email admin@fedoraproject.org + args: + creates: /etc/letsencrypt/live/{{ ansible_fqdn }}/privkey.pem + notify: + - restart nginx + + - name: Gunicorn logging configuration copy: src: logging.ini @@ -37,17 +48,6 @@ - restart nginx -- name: install python2-certbot-nginx - dnf: name=python2-certbot-nginx state=present - -- name: get the letencrypt cert - command: certbot certonly -n --nginx -d {{ ansible_fqdn }} --agree-tos --email admin@fedoraproject.org - args: - creates: /etc/letsencrypt/live/{{ ansible_fqdn }}/privkey.pem - notify: - - restart nginx - - - name: Nginx proxy configuration copy: src: "{{ item }}" diff --git a/roles/hubs/templates/bashrc b/roles/hubs/templates/bashrc index 89027290a4..c1f32e2910 100644 --- a/roles/hubs/templates/bashrc +++ b/roles/hubs/templates/bashrc @@ -13,6 +13,10 @@ fi # by defining a variable with name __help containing the help text +# Honcho has issues outputing UTF-8 in Vagrant SSH +# https://github.com/nickstenning/honcho/issues/51 +export PYTHONIOENCODING=utf-8 + export HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py export FLASK_APP={{ hubs_code_dir }}/hubs/app.py @@ -23,14 +27,9 @@ workon() { cd {{ hubs_code_dir }} } -hup() { - source {{ hubs_venv_dir }}/bin/activate - pushd {{ hubs_code_dir }} - FLASK_DEBUG=1 flask run --host 0.0.0.0 --port 5000 -} +alias hup="pushd ~ ; honcho start ; popd" hreset() { - source {{ hubs_venv_dir }}/bin/activate {% if hubs_db_type == "postgresql" %} sudo -u postgres dropdb hubs sudo -u postgres createdb -O hubs hubs @@ -39,7 +38,6 @@ hreset() { {% endif %} rm {{ hubs_var_dir }}/cache.db pushd {{ hubs_code_dir }} - python populate.py + {{ hubs_venv_dir }}/bin/python populate.py popd - deactivate } diff --git a/roles/hubs/templates/fedmsg_config b/roles/hubs/templates/fedmsg_config index 064401878d..1ab54b00a5 100644 --- a/roles/hubs/templates/fedmsg_config +++ b/roles/hubs/templates/fedmsg_config @@ -20,10 +20,13 @@ config = { 'hubs.consumer.enabled': True, 'hubs.redis.triage-queue-name': 'fedora-hubs-triage-queue', - # FAS + # Use fedmsg-relay to publish messages + 'active': True, + + # FAS credentials 'fas_credentials': { - 'username': '{{ fedoraDummyUser }}', - 'password': '{{ fedoraDummyUserPassword }}', + 'username': '{{ hubs_fas_username }}', + 'password': '{{ hubs_fas_password }}', }, } diff --git a/roles/hubs/templates/honcho-env b/roles/hubs/templates/honcho-env new file mode 100644 index 0000000000..352c551383 --- /dev/null +++ b/roles/hubs/templates/honcho-env @@ -0,0 +1,3 @@ +FLASK_DEBUG=1 +FLASK_APP={{ hubs_code_dir }}/hubs/app.py +HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py diff --git a/roles/hubs/templates/honcho-procfile b/roles/hubs/templates/honcho-procfile new file mode 100644 index 0000000000..937ef33b6f --- /dev/null +++ b/roles/hubs/templates/honcho-procfile @@ -0,0 +1,7 @@ +web: {{ hubs_venv_dir }}/bin/python /usr/bin/flask-3 run --host 0.0.0.0 --port 5000 +triage: {{ hubs_venv_dir }}/bin/fedora-hubs-triage +worker: {{ hubs_venv_dir }}/bin/fedora-hubs-worker +sse: {{ hubs_venv_dir }}/bin/python /usr/bin/twistd -l - --pidfile= -ny {{ hubs_code_dir }}/hubs/backend/sse_server.tac +fedmsg_hub: {{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-hub +fedmsg_relay: {{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-relay +js_build: cd {{ hubs_code_dir }}/hubs/static/client && npm run dev diff --git a/roles/hubs/templates/hubs-fedmsg-relay.service b/roles/hubs/templates/hubs-fedmsg-relay.service new file mode 100644 index 0000000000..d8fe0ca491 --- /dev/null +++ b/roles/hubs/templates/hubs-fedmsg-relay.service @@ -0,0 +1,14 @@ +[Unit] +Description=Hubs-specific fedmsg processing relay +After=network.target +Documentation=https://fedmsg.readthedocs.org/ + +[Service] +ExecStart={{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-relay +Type=simple +User=fedmsg +Group=fedmsg +Restart=on-failure + +[Install] +WantedBy=multi-user.target From 45d373507cab7cd6d110b33864d12a4dc66c3aca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20P=C3=A1ral?= Date: Wed, 24 Jan 2018 15:22:37 +0100 Subject: [PATCH 034/242] taskotron-buildmaster: improve path for heartbeat.log It's now placed in artifacts dir root instead of per-playbook artifacts dir. --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 41418cf258..1da3ee837a 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -218,7 +218,7 @@ factory.addStep(ShellCommand(command=["runtask", {% if deployment_type in ['dev'] %} sigtermTime=5*60, logfiles={ - 'heartbeat.log': {'filename': Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/tests.yml/taskotron/heartbeat.log')} # FIXME: tests.yml hardcoded + 'heartbeat.log': {'filename': Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/taskotron/heartbeat.log')} } {% endif %} {% if deployment_type in ['stg', 'prod'] %} From 400fd88e139437dc2d8a4f3c5e385e6cb433a47e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Wed, 24 Jan 2018 15:27:48 +0000 Subject: [PATCH 035/242] Hubs: fix playbook --- playbooks/hosts/hubs-dev.fedorainfracloud.org.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml b/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml index 36386dd71a..82a4b5eac8 100644 --- a/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml +++ b/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml @@ -59,8 +59,8 @@ hubs_dev_mode: false hubs_ssl_cert: /etc/letsencrypt/live/{{ ansible_fqdn }}/fullchain.pem hubs_ssl_key: /etc/letsencrypt/live/{{ ansible_fqdn }}/privkey.pem - hubs_fas_username: "{{ fedoraDummyUser }}", - hubs_fas_password: "{{ fedoraDummyUserPassword }}", + hubs_fas_username: "{{ fedoraDummyUser }}" + hubs_fas_password: "{{ fedoraDummyUserPassword }}" tasks: From c4ac38563830a0d61299864f87198dbde1ea11b9 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 24 Jan 2018 15:56:27 +0000 Subject: [PATCH 036/242] add some dynamic ips for moonshot in the 129 net --- roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org index 250a3b040b..e8d4957ea7 100644 --- a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org +++ b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org @@ -201,6 +201,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { option domain-name-servers 10.5.126.21, 10.5.126.22; option routers 10.5.129.254; option log-servers 10.5.126.29; + range 10.5.129.190 10.5.129.210; host ppc8-01 { hardware ethernet 40:f2:e9:5d:39:43; From fdef430f4e466381d7abf0a0bea6a8cf69a9bbd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Wed, 24 Jan 2018 16:03:32 +0000 Subject: [PATCH 037/242] Hubs: don't rely on become being set globally --- roles/hubs/tasks/db-postgresql.yml | 3 +++ roles/hubs/tasks/db-sqlite.yml | 1 + roles/hubs/tasks/main.yml | 9 +++++++++ 3 files changed, 13 insertions(+) diff --git a/roles/hubs/tasks/db-postgresql.yml b/roles/hubs/tasks/db-postgresql.yml index 2a9eec915c..017674c5f1 100644 --- a/roles/hubs/tasks/db-postgresql.yml +++ b/roles/hubs/tasks/db-postgresql.yml @@ -28,6 +28,7 @@ name: hubs password: "{{ hubs_db_password }}" role_attr_flags: NOSUPERUSER,NOCREATEROLE,NOCREATEDB + become: true become_user: postgres - name: Create the database @@ -35,6 +36,7 @@ name: hubs owner: hubs register: db_creation + become: true become_user: postgres - name: Ease local access to the database @@ -51,6 +53,7 @@ chdir: "{{ hubs_code_dir }}" environment: HUBS_CONFIG: "{{ hubs_conf_dir }}/hubs_config.py" + become: true become_user: "{{ main_user }}" when: db_creation|succeeded and db_creation|changed diff --git a/roles/hubs/tasks/db-sqlite.yml b/roles/hubs/tasks/db-sqlite.yml index 624c726019..9d15e816d5 100644 --- a/roles/hubs/tasks/db-sqlite.yml +++ b/roles/hubs/tasks/db-sqlite.yml @@ -5,4 +5,5 @@ chdir: "{{ hubs_code_dir }}" environment: HUBS_CONFIG: "{{ hubs_conf_dir }}/hubs_config.py" + become: true become_user: "{{ main_user }}" diff --git a/roles/hubs/tasks/main.yml b/roles/hubs/tasks/main.yml index db877d2c8d..534c13fa79 100644 --- a/roles/hubs/tasks/main.yml +++ b/roles/hubs/tasks/main.yml @@ -58,6 +58,7 @@ # Set up the Python development environment - name: Install Fedora Hubs requirements.txt into hubs virtualenv + become: true become_user: "{{ main_user }}" pip: requirements: "{{ hubs_code_dir }}/requirements.txt" @@ -66,6 +67,7 @@ virtualenv_command: virtualenv-3 - name: Install Fedora Hubs test-requirements.txt into hubs virtualenv + become: true become_user: "{{ main_user }}" pip: requirements: "{{ hubs_code_dir }}/test-requirements.txt" @@ -74,6 +76,7 @@ virtualenv_command: virtualenv-3 - name: Install other packages into hubs virtualenv + become: true become_user: "{{ main_user }}" pip: name: "{{ item }}" @@ -84,12 +87,14 @@ - bleach - name: Install Fedora Hubs into the virtualenv + become: true become_user: "{{ main_user }}" command: "{{ hubs_venv_dir }}/bin/pip install -e {{ hubs_code_dir }}" args: creates: "{{ hubs_venv_dir }}/lib/python3.6/site-packages/fedora-hubs.egg-link" - name: Set bin file context in the virtualenv + become: true become_user: "{{ main_user }}" file: path: "{{ hubs_venv_dir }}/bin" @@ -104,6 +109,7 @@ with_first_found: - hubs_config.{{ ansible_hostname }} - hubs_config + become: true become_user: "{{ main_user }}" notify: "hubs configuration change" @@ -121,6 +127,7 @@ oidc-register --output-file {{ hubs_conf_dir }}/client_secrets.json https://iddev.fedorainfracloud.org/ {{ hubs_url }} + become: true become_user: "{{ main_user }}" args: creates: "{{ hubs_conf_dir }}/client_secrets.json" @@ -139,6 +146,7 @@ # Set up JavaScript requirements - name: Install npm packages command: npm install + become: true become_user: "{{ main_user }}" args: creates: node_modules @@ -146,6 +154,7 @@ - name: Build JavaScript assets command: npm run build + become: true become_user: "{{ main_user }}" args: chdir: "{{ hubs_code_dir }}/hubs/static/client" From 555b49499ecd6ea991f7a274f6363b5705d46f72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Wed, 24 Jan 2018 16:10:56 +0000 Subject: [PATCH 038/242] Hubs: require twisted --- roles/hubs/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/hubs/tasks/main.yml b/roles/hubs/tasks/main.yml index 534c13fa79..b10a52eed2 100644 --- a/roles/hubs/tasks/main.yml +++ b/roles/hubs/tasks/main.yml @@ -38,6 +38,7 @@ - python3-pygments - python3-pygments-markdown-lexer - python3-retask + - python3-twisted # Create directory structure From c032c55921a2e8e55c956b8b68c67e57047b889b Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 24 Jan 2018 16:11:10 +0000 Subject: [PATCH 039/242] Drop dynamic range and just update carts mac addrs --- .../files/dhcpd.conf.noc01.phx2.fedoraproject.org | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org index e8d4957ea7..1e60ab8b8a 100644 --- a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org +++ b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org @@ -201,7 +201,6 @@ subnet 10.5.129.0 netmask 255.255.255.0 { option domain-name-servers 10.5.126.21, 10.5.126.22; option routers 10.5.129.254; option log-servers 10.5.126.29; - range 10.5.129.190 10.5.129.210; host ppc8-01 { hardware ethernet 40:f2:e9:5d:39:43; @@ -296,7 +295,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { } host aarch64-c09n1 { - hardware ethernet 14:58:D0:58:E5:B2; + hardware ethernet 14:58:D0:58:A5:52; fixed-address 10.5.129.109; next-server 10.5.126.41; option host-name "aarch64-c09n1"; @@ -336,7 +335,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { } host aarch64-c14n1 { - hardware ethernet 14:58:D0:58:75:32; + hardware ethernet 14:58:D0:58:65:E2; fixed-address 10.5.129.114; next-server 10.5.126.41; option host-name "aarch64-c14n1"; @@ -360,7 +359,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { } host aarch64-c17n1 { - hardware ethernet 14:58:D0:58:C4:F2; + hardware ethernet 14:58:D0:58:C5:F2; fixed-address 10.5.129.117; next-server 10.5.126.41; option host-name "aarch64-c17n1"; From 4a2901f8d9063b8a96c47f28c93a274362260cbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Wed, 24 Jan 2018 16:13:42 +0000 Subject: [PATCH 040/242] Hubs: fix more python3 deps --- roles/hubs/tasks/webserver.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/hubs/tasks/webserver.yml b/roles/hubs/tasks/webserver.yml index 3aea933ed2..e502057055 100644 --- a/roles/hubs/tasks/webserver.yml +++ b/roles/hubs/tasks/webserver.yml @@ -3,7 +3,7 @@ - name: Install the webserver packages dnf: name={{ item }} state=present with_items: - - python-gunicorn + - python3-gunicorn - nginx - libsemanage-python From 7e5db19f663f94a4d67ef094526b40540a9af5d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Wed, 24 Jan 2018 16:15:37 +0000 Subject: [PATCH 041/242] Hubs: fix even more python3 deps --- roles/hubs/tasks/db-postgresql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/hubs/tasks/db-postgresql.yml b/roles/hubs/tasks/db-postgresql.yml index 017674c5f1..3e299bf2bc 100644 --- a/roles/hubs/tasks/db-postgresql.yml +++ b/roles/hubs/tasks/db-postgresql.yml @@ -4,7 +4,7 @@ dnf: name={{ item }} state=present with_items: - postgresql-server - - python-psycopg2 + - python3-psycopg2 - name: Set up postgresql database command: postgresql-setup --initdb From 134c02a499a5296dfcebf5900bcc4682ef3ff4e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Wed, 24 Jan 2018 16:25:20 +0000 Subject: [PATCH 042/242] Hubs: actually use the local fedmsg-relay (python2) --- roles/hubs/handlers/main.yml | 5 ----- roles/hubs/tasks/main.yml | 1 - roles/hubs/tasks/prod.yml | 3 +-- roles/hubs/templates/hubs-fedmsg-relay.service | 14 -------------- 4 files changed, 1 insertion(+), 22 deletions(-) delete mode 100644 roles/hubs/templates/hubs-fedmsg-relay.service diff --git a/roles/hubs/handlers/main.yml b/roles/hubs/handlers/main.yml index f71ee8fd8c..7f7235b2e8 100644 --- a/roles/hubs/handlers/main.yml +++ b/roles/hubs/handlers/main.yml @@ -6,11 +6,6 @@ listen: "hubs configuration change" when: not hubs_dev_mode -- name: restart the hubs-specific fedmsg-relay - service: name=hubs-fedmsg-relay state=restarted - listen: "hubs configuration change" - when: not hubs_dev_mode - - name: restart hubs triage service: name=hubs-triage@* state=restarted listen: "hubs configuration change" diff --git a/roles/hubs/tasks/main.yml b/roles/hubs/tasks/main.yml index b10a52eed2..84b225e1cc 100644 --- a/roles/hubs/tasks/main.yml +++ b/roles/hubs/tasks/main.yml @@ -167,7 +167,6 @@ service: name={{ item }} state=stopped enabled=no with_items: - fedmsg-hub - - fedmsg-relay # Include mode-specific tasks diff --git a/roles/hubs/tasks/prod.yml b/roles/hubs/tasks/prod.yml index 0cc4894eab..e9393e8a37 100644 --- a/roles/hubs/tasks/prod.yml +++ b/roles/hubs/tasks/prod.yml @@ -7,7 +7,6 @@ - hubs-worker@ - hubs-sse - hubs-fedmsg-hub - - hubs-fedmsg-relay register: service_installed - name: reload systemd @@ -17,12 +16,12 @@ - name: Start and enable the services in prod mode service: name={{ item }} state=started enabled=yes with_items: + - fedmsg-relay - hubs-triage@1 - hubs-triage@2 - hubs-worker@1 - hubs-worker@2 - hubs-sse - hubs-fedmsg-hub - - hubs-fedmsg-relay - include_tasks: webserver.yml diff --git a/roles/hubs/templates/hubs-fedmsg-relay.service b/roles/hubs/templates/hubs-fedmsg-relay.service deleted file mode 100644 index d8fe0ca491..0000000000 --- a/roles/hubs/templates/hubs-fedmsg-relay.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Hubs-specific fedmsg processing relay -After=network.target -Documentation=https://fedmsg.readthedocs.org/ - -[Service] -ExecStart={{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-relay -Type=simple -User=fedmsg -Group=fedmsg -Restart=on-failure - -[Install] -WantedBy=multi-user.target From e8bab5cba62ddfc1b69f1d87062ab06c703bfd67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20P=C3=A1ral?= Date: Wed, 24 Jan 2018 17:41:57 +0100 Subject: [PATCH 043/242] taskotron-buildmaster: use lazylogfiles --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 1da3ee837a..66b946ffb3 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -217,6 +217,7 @@ factory.addStep(ShellCommand(command=["runtask", timeout=20*60, {% if deployment_type in ['dev'] %} sigtermTime=5*60, + lazylogfiles=True, logfiles={ 'heartbeat.log': {'filename': Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/taskotron/heartbeat.log')} } From 70be01e16c444f4524f55e20b4b92d3caa62a137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Wed, 24 Jan 2018 16:42:55 +0000 Subject: [PATCH 044/242] Hubs: fix even moooore python3 deps --- roles/hubs/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/hubs/tasks/main.yml b/roles/hubs/tasks/main.yml index 84b225e1cc..bb7a79ee79 100644 --- a/roles/hubs/tasks/main.yml +++ b/roles/hubs/tasks/main.yml @@ -14,6 +14,7 @@ - fedmsg-relay - python3-virtualenv - python3-flask-oidc + - python3-moksha-common - postfix - name: Install the distribution versions of requirements.txt From e27614fde6e34296c0550737968a3bf80b00107c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Wed, 24 Jan 2018 20:31:48 +0000 Subject: [PATCH 045/242] Hubs: fix playbook --- playbooks/hosts/hubs-dev.fedorainfracloud.org.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml b/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml index 82a4b5eac8..6bb2759453 100644 --- a/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml +++ b/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml @@ -36,6 +36,8 @@ - dnf: name={{item}} state=present with_items: - git + # for certbot + - httpd - name: create the code directory file: dest=/srv/hubs state=directory owner=fedora group=fedora From b05b9bcb80bb5511e4e584bae2dcc13044b1c7bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Wed, 24 Jan 2018 20:38:18 +0000 Subject: [PATCH 046/242] Hubs: fix playbook again --- playbooks/hosts/hubs-dev.fedorainfracloud.org.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml b/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml index 6bb2759453..6c9b7fcdc3 100644 --- a/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml +++ b/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml @@ -46,6 +46,7 @@ git: repo=https://pagure.io/fedora-hubs.git dest=/srv/hubs/fedora-hubs version=develop + become: true become_user: fedora #ignore_errors: true From cf69510ab91ff35056e0d9202342522a406e1cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Wed, 24 Jan 2018 20:42:56 +0000 Subject: [PATCH 047/242] Hubs: fix playbook 3, the revenge --- roles/hubs/tasks/db-postgresql.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/hubs/tasks/db-postgresql.yml b/roles/hubs/tasks/db-postgresql.yml index 3e299bf2bc..e3af324019 100644 --- a/roles/hubs/tasks/db-postgresql.yml +++ b/roles/hubs/tasks/db-postgresql.yml @@ -5,6 +5,8 @@ with_items: - postgresql-server - python3-psycopg2 + # For the ansible modules + - python-psycopg2 - name: Set up postgresql database command: postgresql-setup --initdb From bb11108fe86639783bdf13964f6b8cd8889a6fb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Wed, 24 Jan 2018 20:50:38 +0000 Subject: [PATCH 048/242] Hubs: fix playbook 4, the mission --- roles/hubs/tasks/webserver.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/hubs/tasks/webserver.yml b/roles/hubs/tasks/webserver.yml index e502057055..5021673c0d 100644 --- a/roles/hubs/tasks/webserver.yml +++ b/roles/hubs/tasks/webserver.yml @@ -8,8 +8,8 @@ - libsemanage-python -- name: install python2-certbot-nginx - dnf: name=python2-certbot-nginx state=present +- name: install python3-certbot-nginx + dnf: name=python3-certbot-nginx state=present - name: get the letencrypt cert command: certbot certonly -n --nginx -d {{ ansible_fqdn }} --agree-tos --email admin@fedoraproject.org From 1a0590e5fd2d4808e676d388eec037fa0eca6d2d Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Wed, 24 Jan 2018 21:49:51 +0100 Subject: [PATCH 049/242] Add multitenancy to staging registry Signed-off-by: Patrick Uiterwijk --- .../templates/reversepassproxy.registry.conf | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf b/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf index 4b65819173..c9d501657c 100644 --- a/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf +++ b/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf @@ -6,7 +6,16 @@ ProxyPreserveHost On RewriteEngine on RewriteRule ^/signatures/(.*) /srv/web/registry-signatures/$1 [L] -RewriteRule ^/v2/latest/(.*) /v2/f27/$1 [R,L] +{% if env == "staging" %} +RewriteCond %{HTTP_HOST} "registry{{env_suffix}}.fedoraproject.org" +RewriteRule ^/v2/(.*) /v2/fedora/$1 + +RewriteCond %{HTTP_HOST} "registry{{env_suffix}}.centos.org" +RewriteRule ^/v2/(.*) /v2/centos/$1 +{% endif %} + + +RewriteRule ^/v2/fedora/latest/(.*) /v2/fedora/f27/$1 [R,L] {% if env == "production" %} RewriteCond %{HTTP:VIA} !cdn77 @@ -48,8 +57,25 @@ SSLOptions +FakeBasicAuth # Write access to docker-deployer only - - Require valid-user - + {% if env == "staging" %} + + + Require user docker-registry-internal-stg + + + + + + Require user docker-registry-centos-stg + + + + Require all denied + + {% else %} + + require valid-user + + {% endif %} From f1ef844715b7e580cb9379580b215092fa47adea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Wed, 24 Jan 2018 21:03:03 +0000 Subject: [PATCH 050/242] Hubs: fix playbook 5: legacy --- roles/hubs/tasks/webserver.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/hubs/tasks/webserver.yml b/roles/hubs/tasks/webserver.yml index 5021673c0d..23152c9ab0 100644 --- a/roles/hubs/tasks/webserver.yml +++ b/roles/hubs/tasks/webserver.yml @@ -12,7 +12,7 @@ dnf: name=python3-certbot-nginx state=present - name: get the letencrypt cert - command: certbot certonly -n --nginx -d {{ ansible_fqdn }} --agree-tos --email admin@fedoraproject.org + command: certbot certonly -n --standalone --pre-hook "systemctl stop nginx" --post-hook "systemctl start nginx" -d {{ ansible_fqdn }} --agree-tos --email admin@fedoraproject.org args: creates: /etc/letsencrypt/live/{{ ansible_fqdn }}/privkey.pem notify: From 445d4f0919fc5b1d311587b73ef73a0f52784b4e Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Wed, 24 Jan 2018 22:04:57 +0100 Subject: [PATCH 051/242] Move Signed-off-by: Patrick Uiterwijk --- .../templates/reversepassproxy.registry.conf | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf b/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf index c9d501657c..9d7c1ace0d 100644 --- a/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf +++ b/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf @@ -55,9 +55,10 @@ SSLOptions +FakeBasicAuth Require all granted + - # Write access to docker-deployer only - {% if env == "staging" %} +# Write access to docker-deployer only +{% if env == "staging" %} Require user docker-registry-internal-stg @@ -72,10 +73,8 @@ SSLOptions +FakeBasicAuth Require all denied - {% else %} - - require valid-user - - {% endif %} - - +{% else %} + + require valid-user + +{% endif %} From 2bb12e25e666eac8547f1ef120182261e45f1eb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Wed, 24 Jan 2018 21:16:44 +0000 Subject: [PATCH 052/242] Hubs: fix playbook strikes back --- roles/hubs/templates/hubs-webapp.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/hubs/templates/hubs-webapp.service b/roles/hubs/templates/hubs-webapp.service index 59586d35b4..ae6d046fcd 100644 --- a/roles/hubs/templates/hubs-webapp.service +++ b/roles/hubs/templates/hubs-webapp.service @@ -6,7 +6,7 @@ Documentation=https://pagure.io/fedora-hubs/ [Service] ExecStart= \ {{ hubs_venv_dir }}/bin/python \ - /usr/bin/gunicorn -b 127.0.0.1:8000 --threads 12 \ + /usr/bin/python3-gunicorn -b 127.0.0.1:8000 --threads 12 \ --log-config {{ hubs_conf_dir }}/logging.ini \ {% if hubs_dev_mode %}--reload{% endif %} \ hubs.app:app From e342afe8d34639ae7007a3fe24f3b5349df09705 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Wed, 24 Jan 2018 22:29:28 +0100 Subject: [PATCH 053/242] Deploy the registry multi-tenant in staging Signed-off-by: Patrick Uiterwijk --- playbooks/include/proxies-reverseproxy.yml | 9 ++- playbooks/include/proxies-websites.yml | 6 ++ .../reversepassproxy.registry-fedora.conf | 33 ++++++++ .../reversepassproxy.registry-generic.conf | 34 ++++++++ .../templates/reversepassproxy.registry.conf | 80 ------------------- 5 files changed, 81 insertions(+), 81 deletions(-) create mode 100644 roles/httpd/reverseproxy/templates/reversepassproxy.registry-fedora.conf create mode 100644 roles/httpd/reverseproxy/templates/reversepassproxy.registry-generic.conf delete mode 100644 roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf diff --git a/playbooks/include/proxies-reverseproxy.yml b/playbooks/include/proxies-reverseproxy.yml index 4cac5a18ac..187213d2c9 100644 --- a/playbooks/include/proxies-reverseproxy.yml +++ b/playbooks/include/proxies-reverseproxy.yml @@ -545,7 +545,14 @@ - role: httpd/reverseproxy website: registry.fedoraproject.org - destname: registry + destname: registry-fedora + # proxyurl in this one is totally ignored, because Docker. + # (turns out it uses PATCH requests that Varnish cannot deal with) + proxyurl: "{{ varnish_url }}" + + - role: httpd/reverseproxy + website: registry.centos.org + destname: registry-centos # proxyurl in this one is totally ignored, because Docker. # (turns out it uses PATCH requests that Varnish cannot deal with) proxyurl: "{{ varnish_url }}" diff --git a/playbooks/include/proxies-websites.yml b/playbooks/include/proxies-websites.yml index 6541f4c6de..3dccad7a77 100644 --- a/playbooks/include/proxies-websites.yml +++ b/playbooks/include/proxies-websites.yml @@ -568,6 +568,12 @@ sslonly: true cert_name: "{{wildcard_cert_name}}" + - role: httpd/website + name: registry.centos.org + server_aliases: [registry.stg.centos.org] + sslonly: true + cert_name: "{{wildcard_cert_name}}" + - role: httpd/website name: candidate-registry.fedoraproject.org server_aliases: [candidate-registry.stg.fedoraproject.org] diff --git a/roles/httpd/reverseproxy/templates/reversepassproxy.registry-fedora.conf b/roles/httpd/reverseproxy/templates/reversepassproxy.registry-fedora.conf new file mode 100644 index 0000000000..abe388b26f --- /dev/null +++ b/roles/httpd/reverseproxy/templates/reversepassproxy.registry-fedora.conf @@ -0,0 +1,33 @@ +RewriteEngine on + +RewriteRule ^/v2/latest/(.*) /v2/f27/$1 [R,L] + +{% if env == "staging" %} +RewriteRule ^/v2/(.*) /v2/fedora/$1 +{% endif %} + +RewriteRule ^/signatures/(.*) /srv/web/registry-signatures/$1 [L] + + + Require all granted + + +{% include './reversepassproxy.registry-generic.conf' %} + +# Write access to docker-deployer only +{% if env == "staging" %} + + + Require user docker-registry-internal-stg + + + + Require all denied + + +{% else %} + + + require valid-user + +{% endif %} diff --git a/roles/httpd/reverseproxy/templates/reversepassproxy.registry-generic.conf b/roles/httpd/reverseproxy/templates/reversepassproxy.registry-generic.conf new file mode 100644 index 0000000000..da8b016c4a --- /dev/null +++ b/roles/httpd/reverseproxy/templates/reversepassproxy.registry-generic.conf @@ -0,0 +1,34 @@ +RequestHeader set X-Forwarded-Scheme https early +RequestHeader set X-Scheme https early +RequestHeader set X-Forwarded-Proto https early +ProxyPreserveHost On + + +{% if env == "production" %} +RewriteCond %{HTTP:VIA} !cdn77 +RewriteCond %{REQUEST_METHOD} !^(PATCH|POST|PUT|DELETE|HEAD)$ +RewriteRule ^/v2/(.*)/blobs/([a-zA-Z0-9:]*) https://cdn.registry.fedoraproject.org/v2/$1/blobs/$2 [R] +{% endif %} + +# This is terible, but Docker. +RewriteCond %{REQUEST_METHOD} ^(PATCH|POST|PUT|DELETE)$ +RewriteRule ^/v2/(.*)$ http://docker-registry02:5000/v2/$1 [P,L] +RewriteRule ^/v2/(.*)$ http://localhost:6081/v2/$1 [P,L] + +DocumentRoot /srv/web/registry-index/ + + + Require all granted + + +SSLVerifyClient optional +SSLVerifyDepth 1 +SSLCACertificateFile /etc/pki/httpd/registry-ca-{{env}}.cert +SSLOptions +FakeBasicAuth + + + + AuthName "Registry Authentication" + AuthType Basic + AuthUserFile /etc/httpd/conf.d/registry.fedoraproject.org/passwd + diff --git a/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf b/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf deleted file mode 100644 index 9d7c1ace0d..0000000000 --- a/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf +++ /dev/null @@ -1,80 +0,0 @@ -RequestHeader set X-Forwarded-Scheme https early -RequestHeader set X-Scheme https early -RequestHeader set X-Forwarded-Proto https early -ProxyPreserveHost On - -RewriteEngine on -RewriteRule ^/signatures/(.*) /srv/web/registry-signatures/$1 [L] - -{% if env == "staging" %} -RewriteCond %{HTTP_HOST} "registry{{env_suffix}}.fedoraproject.org" -RewriteRule ^/v2/(.*) /v2/fedora/$1 - -RewriteCond %{HTTP_HOST} "registry{{env_suffix}}.centos.org" -RewriteRule ^/v2/(.*) /v2/centos/$1 -{% endif %} - - -RewriteRule ^/v2/fedora/latest/(.*) /v2/fedora/f27/$1 [R,L] - -{% if env == "production" %} -RewriteCond %{HTTP:VIA} !cdn77 -RewriteCond %{REQUEST_METHOD} !^(PATCH|POST|PUT|DELETE|HEAD)$ -RewriteRule ^/v2/(.*)/blobs/([a-zA-Z0-9:]*) https://cdn.registry.fedoraproject.org/v2/$1/blobs/$2 [R] -{% endif %} - -# This is terible, but Docker. -RewriteCond %{REQUEST_METHOD} ^(PATCH|POST|PUT|DELETE)$ -RewriteRule ^/v2/(.*)$ http://docker-registry02:5000/v2/$1 [P,L] -RewriteRule ^/v2/(.*)$ http://localhost:6081/v2/$1 [P,L] - -DocumentRoot /srv/web/registry-index/ - - - Require all granted - - -SSLVerifyClient optional -SSLVerifyDepth 1 -SSLCACertificateFile /etc/pki/httpd/registry-ca-{{env}}.cert -SSLOptions +FakeBasicAuth - - - Require all granted - - - - - Order deny,allow - Allow from all - AuthName "Registry Authentication" - AuthType Basic - AuthUserFile /etc/httpd/conf.d/registry.fedoraproject.org/passwd - - # Anyone can read - - Require all granted - - - -# Write access to docker-deployer only -{% if env == "staging" %} - - - Require user docker-registry-internal-stg - - - - - - Require user docker-registry-centos-stg - - - - Require all denied - -{% else %} - - require valid-user - -{% endif %} From 8cd7d733fb98dacfe53dca61e9b2896a2430f723 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 25 Jan 2018 13:29:17 +0000 Subject: [PATCH 054/242] switch this location --- .../dhcpd.conf.noc01.phx2.fedoraproject.org | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org index 1e60ab8b8a..8d57ea802b 100644 --- a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org +++ b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org @@ -235,7 +235,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.101; next-server 10.5.126.41; option host-name "aarch64-c01n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c02n1 { @@ -243,7 +243,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.102; next-server 10.5.126.41; option host-name "aarch64-c02n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c03n1 { @@ -251,7 +251,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.103; next-server 10.5.126.41; option host-name "aarch64-c03n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c04n1 { @@ -259,7 +259,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.104; next-server 10.5.126.41; option host-name "aarch64-c04n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c05n1 { @@ -267,7 +267,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.105; next-server 10.5.126.41; option host-name "aarch64-c05n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c06n1 { @@ -275,7 +275,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.106; next-server 10.5.126.41; option host-name "aarch64-c06n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c07n1 { @@ -283,7 +283,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.107; next-server 10.5.126.41; option host-name "aarch64-c07n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c08n1 { @@ -291,7 +291,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.108; next-server 10.5.126.41; option host-name "aarch64-c08n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c09n1 { @@ -299,7 +299,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.109; next-server 10.5.126.41; option host-name "aarch64-c09n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c10n1 { @@ -307,7 +307,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.110; next-server 10.5.126.41; option host-name "aarch64-c10n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c11n1 { @@ -315,7 +315,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.111; next-server 10.5.126.41; option host-name "aarch64-c11n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c12n1 { @@ -323,7 +323,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.112; next-server 10.5.126.41; option host-name "aarch64-c12n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c13n1 { @@ -331,7 +331,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.113; next-server 10.5.126.41; option host-name "aarch64-c13n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c14n1 { @@ -339,7 +339,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.114; next-server 10.5.126.41; option host-name "aarch64-c14n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c15n1 { @@ -347,7 +347,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.115; next-server 10.5.126.41; option host-name "aarch64-c15n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c16n1 { @@ -355,7 +355,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.116; next-server 10.5.126.41; option host-name "aarch64-c16n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c17n1 { @@ -363,7 +363,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.117; next-server 10.5.126.41; option host-name "aarch64-c17n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c18n1 { @@ -371,7 +371,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.118; next-server 10.5.126.41; option host-name "aarch64-c18n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c19n1 { @@ -379,7 +379,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.119; next-server 10.5.126.41; option host-name "aarch64-c19n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c20n1 { @@ -387,7 +387,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.120; next-server 10.5.126.41; option host-name "aarch64-c20n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c21n1 { @@ -395,7 +395,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.121; next-server 10.5.126.41; option host-name "aarch64-c21n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c22n1 { @@ -403,7 +403,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.122; next-server 10.5.126.41; option host-name "aarch64-c22n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c23n1 { @@ -411,7 +411,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.123; next-server 10.5.126.41; option host-name "aarch64-c23n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c24n1 { @@ -419,7 +419,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.124; next-server 10.5.126.41; option host-name "aarch64-c24n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-c25n1 { @@ -427,7 +427,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { fixed-address 10.5.129.125; next-server 10.5.126.41; option host-name "aarch64-c25n1"; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } } @@ -1777,7 +1777,7 @@ subnet 10.5.78.0 netmask 255.255.255.0 { fixed-address 10.5.78.70; option host-name "compose-aarch64-01"; next-server 10.5.126.41; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-02a { @@ -1785,7 +1785,7 @@ subnet 10.5.78.0 netmask 255.255.255.0 { fixed-address 10.5.78.75; option host-name "aarch64-02a"; next-server 10.5.126.41; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-03a { @@ -1793,7 +1793,7 @@ subnet 10.5.78.0 netmask 255.255.255.0 { fixed-address 10.5.78.80; option host-name "aarch64-03a"; next-server 10.5.126.41; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-04a { @@ -1801,7 +1801,7 @@ subnet 10.5.78.0 netmask 255.255.255.0 { fixed-address 10.5.78.85; option host-name "aarch64-04a"; next-server 10.5.126.41; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-05a { @@ -1809,7 +1809,7 @@ subnet 10.5.78.0 netmask 255.255.255.0 { fixed-address 10.5.78.150; option host-name "aarch64-05a"; next-server 10.5.126.41; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-06a { @@ -1817,7 +1817,7 @@ subnet 10.5.78.0 netmask 255.255.255.0 { fixed-address 10.5.78.155; option host-name "aarch64-06a"; next-server 10.5.126.41; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-07a { @@ -1825,7 +1825,7 @@ subnet 10.5.78.0 netmask 255.255.255.0 { fixed-address 10.5.78.160; option host-name "aarch64-07a"; next-server 10.5.126.41; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-08a { @@ -1833,7 +1833,7 @@ subnet 10.5.78.0 netmask 255.255.255.0 { fixed-address 10.5.78.165; option host-name "aarch64-08a"; next-server 10.5.126.41; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-09a { @@ -1841,7 +1841,7 @@ subnet 10.5.78.0 netmask 255.255.255.0 { fixed-address 10.5.78.170; option host-name "aarch64-09a"; next-server 10.5.126.41; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-10a { @@ -1849,7 +1849,7 @@ subnet 10.5.78.0 netmask 255.255.255.0 { fixed-address 10.5.78.175; option host-name "aarch64-10a"; next-server 10.5.126.41; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-11a { @@ -1857,7 +1857,7 @@ subnet 10.5.78.0 netmask 255.255.255.0 { fixed-address 10.5.78.180; option host-name "aarch64-11a"; next-server 10.5.126.41; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-12a { @@ -1865,7 +1865,7 @@ subnet 10.5.78.0 netmask 255.255.255.0 { fixed-address 10.5.78.185; option host-name "aarch64-12a"; next-server 10.5.126.41; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-13a { @@ -1873,7 +1873,7 @@ subnet 10.5.78.0 netmask 255.255.255.0 { fixed-address 10.5.78.190; option host-name "aarch64-13a"; next-server 10.5.126.41; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-14a { @@ -1881,7 +1881,7 @@ subnet 10.5.78.0 netmask 255.255.255.0 { fixed-address 10.5.78.195; option host-name "aarch64-14a"; next-server 10.5.126.41; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } host aarch64-15a { @@ -1889,7 +1889,7 @@ subnet 10.5.78.0 netmask 255.255.255.0 { fixed-address 10.5.78.200; option host-name "aarch64-15a"; next-server 10.5.126.41; - filename "grubaa64.efi"; + filename "/uefi/grubaa64.efi"; } } From 0126a1272b16404045d86d760b223cd35f8bd2a9 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 25 Jan 2018 14:57:48 +0000 Subject: [PATCH 055/242] add birthdays to virthosts --- playbooks/groups/virthost.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/playbooks/groups/virthost.yml b/playbooks/groups/virthost.yml index ee20c955bd..6c309e1446 100644 --- a/playbooks/groups/virthost.yml +++ b/playbooks/groups/virthost.yml @@ -2,6 +2,8 @@ # NOTE: should be used with --limit most of the time # NOTE: most of these vars_path come from group_vars/backup_server or from hostvars +- import_playbook: "/srv/web/infra/ansible/playbooks/include/happy_birthday.yml myhosts=virthost:bvirthost:buildvmhost:virthost-comm:colo-virt" + - name: make virthost server system hosts: virthost:bvirthost:buildvmhost:virthost-comm:colo-virt user: root From 9cfe2d113bf5e8d1fc246447dc724ea0c4f38c77 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Fri, 26 Jan 2018 13:14:30 +0000 Subject: [PATCH 056/242] add python26 and python35 on jenkins f26 worker --- inventory/group_vars/jenkins-slave | 2 ++ 1 file changed, 2 insertions(+) diff --git a/inventory/group_vars/jenkins-slave b/inventory/group_vars/jenkins-slave index 32582efe40..276c4f4927 100644 --- a/inventory/group_vars/jenkins-slave +++ b/inventory/group_vars/jenkins-slave @@ -278,3 +278,5 @@ f25_only: f26_only: - python2-koji # Needed for pyrpkg - python3-koji # Needed for pyrpkg +- python26 +- python35 From ce5c96c0943f266e1b2d323efc80980eca589d97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20P=C3=A1ral?= Date: Fri, 26 Jan 2018 16:48:10 +0100 Subject: [PATCH 057/242] taskotron-client: sync taskotron.yaml.j2 with latest upstream content --- .../templates/taskotron.yaml.j2 | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/roles/taskotron/taskotron-client/templates/taskotron.yaml.j2 b/roles/taskotron/taskotron-client/templates/taskotron.yaml.j2 index 7a5d012326..0d026cad41 100644 --- a/roles/taskotron/taskotron-client/templates/taskotron.yaml.j2 +++ b/roles/taskotron/taskotron-client/templates/taskotron.yaml.j2 @@ -2,7 +2,7 @@ ## The file is in YAML syntax, read more about it at: ## http://en.wikipedia.org/wiki/Yaml ## libtaskotron docs live at: -## https://docs.qadevel.cloud.fedoraproject.org/libtaskotron/latest/ +## https://docs.qa.fedoraproject.org/libtaskotron/latest/ ## ==== GENERAL section ==== @@ -38,6 +38,12 @@ runtask_mode: libvirt runtask_mode: local {% endif %} +## Supported machine architectures. This is mostly used by generic, +## arch-independent tasks to determine which arches to test and report against. +## You can still run an arch-specific task on any other arch using the command +## line. +#supported_arches: ['x86_64', 'armhfp'] + ## ==== SCHEDULING section ==== ## This section holds options related to the scheduling and execution system, @@ -51,8 +57,8 @@ buildbot_task_step: 'runtask' ## This section controls which result reports you want to send after the test ## execution is complete. -## Whether to send test results to the configured ResultsDB server. See also -## 'reporting_enabled' option. +## Whether to send test results to the configured ResultsDB server. +## [default: True for production, False for development] report_to_resultsdb: True @@ -101,16 +107,6 @@ artifacts_baseurl: {{ artifacts_base_url }} #download_cache_enabled: False -## ==== BODHI EMAIL section ==== -## These configuration options affect how Taskotron decideds to send emails -## through Bodhi in specific situations. - -## How long (in minutes) should we wait before allowing consequent test to -## re-post a 'FAILED' comment into Bodhi once again. -## By default 3 days (3*24*60 = 4320). -#bodhi_posting_comments_span: 4320 - - ## ==== PATHS section ==== ## Location of various pieces of the project. @@ -133,7 +129,7 @@ artifacts_baseurl: {{ artifacts_base_url }} ## File names need to adhere to the naming standard of: ## YYMMDD_HHMM-fedora-RELEASE-FLAVOR-ARCH.(qcow2|raw|img) ## For example: -## 160301_1030-fedora-23-taskotron_cloud-x86_64.img +## 160301_1030-fedora-25-taskotron_cloud-x86_64.img ## Variables disposable_(release|flavor|arch) set in this config file ## define what kind of image is looked for. ## The newest (by YYMMDD_HHMM) image of the respective R-F-A is used. @@ -148,11 +144,10 @@ force_imageurl: False ## Default distro/release/flavor/arch for the disposable images discovery #default_disposable_distro: fedora -default_disposable_release: "26" +default_disposable_release: '26' #default_disposable_flavor: taskotron_cloud #default_disposable_arch: x86_64 - ## Additional repos for minion to install packages from minion_repos: - https://fedorapeople.org/groups/qa/taskotron-repos/taskotron-production-override/taskotron-production-override.repo From 7b3eeba89b66966b2d29a02b898cdc16bb429c02 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 27 Jan 2018 08:51:38 +0000 Subject: [PATCH 058/242] Ramp this down to see if we can keep out of swap. --- roles/pdc/frontend/templates/pdc.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/pdc/frontend/templates/pdc.conf b/roles/pdc/frontend/templates/pdc.conf index 17add4bf7a..8bedf17966 100644 --- a/roles/pdc/frontend/templates/pdc.conf +++ b/roles/pdc/frontend/templates/pdc.conf @@ -1,7 +1,7 @@ Alias /docs/ /usr/share/doc/pdc/docs/build/html/ Alias /saml2protected /usr/share/ipsilon/ui/saml2sp -WSGIDaemonProcess pdc user=apache group=apache maximum-requests=1000 display-name=pdc processes={{ wsgi_procs - 1}} threads={{ wsgi_threads }} +WSGIDaemonProcess pdc user=apache group=apache maximum-requests=100 display-name=pdc processes={{ wsgi_procs - 1}} threads={{ wsgi_threads }} WSGISocketPrefix run/wsgi WSGIRestrictStdout On WSGIRestrictSignal Off From 4d8ad2041926ffdc7fe6e6afa2221603af81fff3 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 27 Jan 2018 08:53:37 +0000 Subject: [PATCH 059/242] This script seems to be working now. --- roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 index 17893492c7..0a55a86992 100644 --- a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 +++ b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 @@ -96,8 +96,6 @@ BUGZILLA_OVERRIDE_REPO = 'releng/fedora-scm-requests' NOTIFYEMAIL = [ 'kevin@fedoraproject.org', 'pingou@fedoraproject.org', - 'ralph@fedoraproject.org', - 'mprahl@fedoraproject.org', ] VERBOSE = False DRYRUN = False From 53d318abc2471d9c6eb8b19daadedc524c316edb Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Sat, 27 Jan 2018 14:15:03 +0000 Subject: [PATCH 060/242] Fix cron-refresh-groups on koschei-backend --- roles/koschei/backend/templates/cron-refresh-groups.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/koschei/backend/templates/cron-refresh-groups.j2 b/roles/koschei/backend/templates/cron-refresh-groups.j2 index a8cd97020d..08423ce93c 100644 --- a/roles/koschei/backend/templates/cron-refresh-groups.j2 +++ b/roles/koschei/backend/templates/cron-refresh-groups.j2 @@ -1,6 +1,7 @@ SHELL=/bin/bash MAILTO=sysadmin-koschei-members@fedoraproject.org 0 0-23/3 * * * koschei /usr/local/bin/koschei-refresh-group php 'php*' + {%- if env != 'stg' %}{# rust packages are not synced on stg yet #} 5 0-23/3 * * * koschei /usr/local/bin/koschei-refresh-distgit-group rust-sig rust-sig {%- endif %} From 571ca7275a5fb5af9473a72f0c6ed959fb789485 Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Sat, 27 Jan 2018 15:00:04 +0000 Subject: [PATCH 061/242] Second try on fixing cron-refresh-groups on koschei-backend --- roles/koschei/backend/templates/cron-refresh-groups.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/koschei/backend/templates/cron-refresh-groups.j2 b/roles/koschei/backend/templates/cron-refresh-groups.j2 index 08423ce93c..7a607adb01 100644 --- a/roles/koschei/backend/templates/cron-refresh-groups.j2 +++ b/roles/koschei/backend/templates/cron-refresh-groups.j2 @@ -2,7 +2,7 @@ SHELL=/bin/bash MAILTO=sysadmin-koschei-members@fedoraproject.org 0 0-23/3 * * * koschei /usr/local/bin/koschei-refresh-group php 'php*' -{%- if env != 'stg' %}{# rust packages are not synced on stg yet #} +{% if env != 'stg' %}{# rust packages are not synced on stg yet #} 5 0-23/3 * * * koschei /usr/local/bin/koschei-refresh-distgit-group rust-sig rust-sig {%- endif %} From 69ed1bb5549156ef8e3c52c72e0a50c6bf545a53 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Sun, 28 Jan 2018 01:08:44 +0100 Subject: [PATCH 062/242] openqa: disable createhdds cron jobs (#1539330) Yeah, daily crashes are bad. Signed-off-by: Adam Williamson --- roles/openqa/server/tasks/main.yml | 9 +++++++-- roles/openqa/worker/tasks/createhdds.yml | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/roles/openqa/server/tasks/main.yml b/roles/openqa/server/tasks/main.yml index 676c8ebb7e..5b44bb454d 100644 --- a/roles/openqa/server/tasks/main.yml +++ b/roles/openqa/server/tasks/main.yml @@ -142,8 +142,13 @@ - /var/lib/openqa/share/factory/repo - /var/lib/openqa/share/factory/other -- name: Set up createhdds cron job - copy: src=createhdds dest=/etc/cron.daily/createhdds owner=root group=root mode=0755 +#- name: Set up createhdds cron job +# copy: src=createhdds dest=/etc/cron.daily/createhdds owner=root group=root mode=0755 + +# While #1539330 is a thing, we probably don't want the servers +# crashing every day... +- name: Remove createhdds cron job (#1539330) + file: path=/etc/cron.daily/createhdds state=absent - name: Check if any hard disk images need (re)building command: "/root/createhdds/createhdds.py check" diff --git a/roles/openqa/worker/tasks/createhdds.yml b/roles/openqa/worker/tasks/createhdds.yml index 20a637999d..44addbb6fb 100644 --- a/roles/openqa/worker/tasks/createhdds.yml +++ b/roles/openqa/worker/tasks/createhdds.yml @@ -39,8 +39,13 @@ repo: https://pagure.io/fedora-qa/createhdds.git dest: /root/createhdds -- name: Set up createhdds cron job - copy: src=createhdds dest=/etc/cron.daily/createhdds owner=root group=root mode=0755 +#- name: Set up createhdds cron job +# copy: src=createhdds dest=/etc/cron.daily/createhdds owner=root group=root mode=0755 + +# While #1539330 is a thing, we probably don't want these boxes +# crashing every day... +- name: Remove createhdds cron job (#1539330) + file: path=/etc/cron.daily/createhdds state=absent - name: Check if any hard disk images need (re)building command: "/root/createhdds/createhdds.py check" From ffa7656480af958c79b3743263bcce269a04fe80 Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Mon, 29 Jan 2018 20:35:50 +0800 Subject: [PATCH 063/242] freshmaker: add freshmaker playbook and roles --- playbooks/groups/freshmaker.yml | 8 +- roles/freshmaker/backend/defaults/main.yml | 3 + roles/freshmaker/backend/meta/main.yml | 3 + roles/freshmaker/backend/tasks/main.yml | 47 ++++ .../etc/koji.conf.d/freshmaker.conf.j2 | 13 + roles/freshmaker/base/defaults/main.yml | 27 ++ roles/freshmaker/base/tasks/main.yml | 52 ++++ .../templates/etc/freshmaker/config.py.j2 | 255 ++++++++++++++++++ roles/freshmaker/frontend/defaults/main.yml | 8 + roles/freshmaker/frontend/meta/main.yml | 3 + roles/freshmaker/frontend/tasks/main.yml | 47 ++++ .../etc/httpd/conf.d/freshmaker.conf.j2 | 49 ++++ 12 files changed, 512 insertions(+), 3 deletions(-) create mode 100644 roles/freshmaker/backend/defaults/main.yml create mode 100644 roles/freshmaker/backend/meta/main.yml create mode 100644 roles/freshmaker/backend/tasks/main.yml create mode 100644 roles/freshmaker/backend/templates/etc/koji.conf.d/freshmaker.conf.j2 create mode 100644 roles/freshmaker/base/defaults/main.yml create mode 100644 roles/freshmaker/base/tasks/main.yml create mode 100644 roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 create mode 100644 roles/freshmaker/frontend/defaults/main.yml create mode 100644 roles/freshmaker/frontend/meta/main.yml create mode 100644 roles/freshmaker/frontend/tasks/main.yml create mode 100644 roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 diff --git a/playbooks/groups/freshmaker.yml b/playbooks/groups/freshmaker.yml index 37677c14f8..968eebc5a6 100644 --- a/playbooks/groups/freshmaker.yml +++ b/playbooks/groups/freshmaker.yml @@ -46,7 +46,7 @@ handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml" -- name: Set up apache on the frontend MBS API app +- name: set up Freshmaker frontend hosts: freshmaker-frontend:freshmaker-frontend-stg user: root gather_facts: True @@ -58,12 +58,13 @@ roles: - mod_wsgi + - freshmaker/frontend handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml" -- name: set up fedmsg configuration and common freshmaker files - hosts: freshmaker:freshmaker-stg +- name: set up Freshmaker backend + hosts: freshmaker-backend:freshmaker-backend-stg user: root gather_facts: True @@ -74,6 +75,7 @@ roles: - fedmsg/base + - freshmaker/backend handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml" diff --git a/roles/freshmaker/backend/defaults/main.yml b/roles/freshmaker/backend/defaults/main.yml new file mode 100644 index 0000000000..1adc701cc4 --- /dev/null +++ b/roles/freshmaker/backend/defaults/main.yml @@ -0,0 +1,3 @@ +--- +freshmaker_upgrade: False +freshmaker_migrate_db: False diff --git a/roles/freshmaker/backend/meta/main.yml b/roles/freshmaker/backend/meta/main.yml new file mode 100644 index 0000000000..4a5c132f49 --- /dev/null +++ b/roles/freshmaker/backend/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - { role: freshmaker/base } diff --git a/roles/freshmaker/backend/tasks/main.yml b/roles/freshmaker/backend/tasks/main.yml new file mode 100644 index 0000000000..febb901756 --- /dev/null +++ b/roles/freshmaker/backend/tasks/main.yml @@ -0,0 +1,47 @@ +--- +- name: install the latest Freshmaker package + yum: + name: freshmaker + state: latest + update_cache: yes + with_items: + - freshmaker + - python2-odcs-client + when: freshmaker_upgrade + notify: + - restart fedmsg-hub + tags: + - freshmaker + - freshmaker/backend + +- name: generate the Freshmaker koji config + template: + src: etc/koji.conf.d/freshmaker.conf.j2 + dest: /etc/koji.conf.d/freshmaker.conf + owner: fedmsg + group: fedmsg + mode: 0440 + notify: + - restart fedmsg-hub + tags: + - freshmaker + - freshmaker/backend + +- name: ensure fedmsg-hub starts on boot + service: + name: "fedmsg-hub" + enabled: yes + +# This will initialize Alembic if the database is empty, and migrate to the +# latest revision +- name: migrate the database + command: "{{ item }}" + with_items: + - freshmaker-manager upgradedb + - freshmaker-manager db migrate + become: yes + become_user: fedmsg + when: freshmaker_migrate_db + tags: + - freshmaker + - freshmaker/backend diff --git a/roles/freshmaker/backend/templates/etc/koji.conf.d/freshmaker.conf.j2 b/roles/freshmaker/backend/templates/etc/koji.conf.d/freshmaker.conf.j2 new file mode 100644 index 0000000000..ed2dcc8653 --- /dev/null +++ b/roles/freshmaker/backend/templates/etc/koji.conf.d/freshmaker.conf.j2 @@ -0,0 +1,13 @@ +[freshmaker_prod] +server = https://koji.fedoraproject.org/kojihub +weburl = https://koji.fedoraproject.org/koji +topurl = https://kojipkgs.fedoraproject.org/ +authtype = kerberos +krb_rdns = false + +[freshmaker_stg] +server = https://koji.stg.fedoraproject.org/kojihub +weburl = https://koji.stg.fedoraproject.org/koji +topurl = https://kojipkgs.stg.fedoraproject.org/ +authtype = kerberos +krb_rdns = false diff --git a/roles/freshmaker/base/defaults/main.yml b/roles/freshmaker/base/defaults/main.yml new file mode 100644 index 0000000000..44d59bbea4 --- /dev/null +++ b/roles/freshmaker/base/defaults/main.yml @@ -0,0 +1,27 @@ +--- +freshmaker_force_postgres_ssl: False +freshmaker_handler_build_whitelist: null +freshmaker_handler_build_blacklist: null +freshmaker_pdc_insecure: False +freshmaker_stg_krb_auth_client_keytab: "/etc/krb5.freshmaker_freshmaker.stg.fedoraproject.org.keytab" +freshmaker_stg_krb_auth_principal: "freshmaker/freshmaker.stg.fedoraproject.org@STG.FEDORAPROJECT.ORG" +freshmaker_prod_krb_auth_client_keytab: "/etc/krb5.freshmaker_freshmaker.fedoraproject.org.keytab" +freshmaker_prod_krb_auth_principal: "freshmaker/freshmaker.fedoraproject.org@FEDORAPROJECT.ORG" +freshmaker_stg_git_base_url: git://pkgs.stg.fedoraproject.org +freshmaker_stg_git_ssh_base_url: ssh://%s@pkgs.stg.fedoraproject.org +freshmaker_stg_git_user: null +freshmaker_prod_git_base_url: git://pkgs.fedoraproject.org +freshmaker_prod_git_ssh_base_url: ssh://%s@pkgs.fedoraproject.org +freshmaker_prod_git_user: null +freshmaker_stg_odcs_server_url: https://odcs.fedoraproject.org +freshmaker_prod_odcs_server_url: https://odcs.stg.fedoraproject.org +freshmaker_stg_odcs_sigkeys: [] +freshmaker_prod_odcs_sigkeys: [] +freshmaker_dry_run: False +freshmaker_admins: {"users": [], "groups": []} +freshmaker_log_level: info +freshmaker_servername: localhost + +freshmaker_messaging_topic_prefix: [] +freshmaker_parsers: [] +freshmaker_handlers: [] diff --git a/roles/freshmaker/base/tasks/main.yml b/roles/freshmaker/base/tasks/main.yml new file mode 100644 index 0000000000..2922e77483 --- /dev/null +++ b/roles/freshmaker/base/tasks/main.yml @@ -0,0 +1,52 @@ +--- +- name: install the packages required for Freshmaker frontend + yum: + name: "{{ item }}" + state: present + with_items: + - httpd + - mod_wsgi + - mod_auth_openidc + - libsemanage-python + - python-psycopg2 + - freshmaker + when: inventory_hostname.startswith('freshmaker-frontend') + tags: + - freshmaker + +- name: install the packages required for Freshmaker backend + yum: + name: "{{ item }}" + state: present + with_items: + - python-psycopg2 + - freshmaker + when: inventory_hostname.startswith('freshmaker-backend') + tags: + - freshmaker + +- name: generate Freshmaker app config for frontend + template: + src: etc/freshmaker/config.py.j2 + dest: /etc/freshmaker/config.py + owner: apache + group: apache + mode: 0440 + notify: + - restart apache + when: inventory_hostname.startswith('freshmaker-frontend') + tags: + - freshmaker + +- name: generate Freshmaker app config for backend + template: + src: etc/freshmaker/config.py.j2 + dest: /etc/freshmaker/config.py + owner: fedmsg + group: fedmsg + mode: 0440 + notify: + - restart fedmsg-hub + when: inventory_hostname.startswith('freshmaker-backend') + tags: + - freshmaker diff --git a/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 b/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 new file mode 100644 index 0000000000..8a03a4aaa5 --- /dev/null +++ b/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 @@ -0,0 +1,255 @@ +# -*- coding: utf-8 -*- + +from os import path, environ + +confdir = path.abspath(path.dirname(__file__)) +# use parent dir as dbdir else fallback to current dir +dbdir = path.abspath(path.join(confdir, '..')) if confdir.endswith('conf') \ + else confdir + + +class BaseConfiguration(object): + # Make this random (used to generate session keys) + SECRET_KEY = '74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0' + SQLALCHEMY_DATABASE_URI = 'sqlite:///{0}'.format(path.join( + dbdir, 'freshmaker.db')) + SQLALCHEMY_TRACK_MODIFICATIONS = False + + HOST = '0.0.0.0' + PORT = 5001 + + DEBUG = False + # Global network-related values, in seconds + NET_TIMEOUT = 120 + NET_RETRY_INTERVAL = 30 + + SYSTEM = 'koji' + MESSAGING = 'fedmsg' # or amq + + # Available backends are: console, file, journal. + LOG_BACKEND = 'journal' + + # Path to log file when LOG_BACKEND is set to "file". + LOG_FILE = 'freshmaker.log' + + # Available log levels are: debug, info, warn, error. + LOG_LEVEL = 'info' + + MESSAGING_TOPIC_PREFIX = ['org.fedoraproject.prod'] + + # Parsers defined for parse specific messages + PARSERS = [ + 'freshmaker.parsers.bodhi:BodhiUpdateCompleteStableParser', + 'freshmaker.parsers.git:GitReceiveParser', + 'freshmaker.parsers.koji:KojiTaskStateChangeParser', + 'freshmaker.parsers.mbs:MBSModuleStateChangeParser', + ] + + # List of enabled composing handlers. + HANDLERS = [ + "freshmaker.handlers.bodhi:BodhiUpdateCompleteStableHandler", + "freshmaker.handlers.git:GitDockerfileChangeHandler", + "freshmaker.handlers.git:GitModuleMetadataChangeHandler", + "freshmaker.handlers.git:GitRPMSpecChangeHandler", + "freshmaker.handlers.koji:KojiTaskStateChangeHandler", + "freshmaker.handlers.mbs:MBSModuleStateChangeHandler", + ] + + # Base URL of git repository with source artifacts. + GIT_BASE_URL = "git://pkgs.fedoraproject.org" + + # SSH base URL of git repository + GIT_SSH_BASE_URL = "ssh://%s@pkgs.fedoraproject.org/" + + # GIT user for cloning and pushing repo + GIT_USER = "" + + # PDC API URL + PDC_URL = 'http://pdc.fedoraproject.org/rest_api/v1' + + # Read Koji configuration from profile instead of reading them from + # configuration file directly. For staging Koji, it is stg. + KOJI_PROFILE = 'koji' + KOJI_PROXYUSER = False + KOJI_BUILD_OWNER = 'freshmaker' + + # Settings for docker image rebuild handler + KOJI_CONTAINER_SCRATCH_BUILD = False + + SSL_ENABLED = False + + # whitelist and blacklist for handlers to decide whether an artifact + # can be built. + # + # In format of: + # + # { : + # { : } + # } + # + # Here is an example of allowing MBSModuleStateChangeHandler to build + # any module that module name matches 'base-.*' but not: + # 1. module name matches 'base-test-module' + # or: + # 2. module from branch 'rawhide' + # + # HANDLER_BUILD_WHITELIST = { + # "MBSModuleStateChangeHandler": { + # "module": [ + # { + # 'name': 'base-.*', + # }, + # ], + # }, + # } + # HANDLER_BUILD_BLACKLIST = { + # "MBSModuleStateChangeHandler": { + # "module": [ + # { + # 'name': 'base-test-module', + # }, + # { + # 'branch': 'rawhide', + # }, + # ], + # }, + # } + + +class DevConfiguration(BaseConfiguration): + DEBUG = True + LOG_BACKEND = 'console' + LOG_LEVEL = 'debug' + + MESSAGING_TOPIC_PREFIX = ['org.fedoraproject.dev', 'org.fedoraproject.stg'] + + # Global network-related values, in seconds + NET_TIMEOUT = 5 + NET_RETRY_INTERVAL = 1 + + KOJI_CONTAINER_SCRATCH_BUILD = True + + LIGHTBLUE_VERIFY_SSL = False + + +class TestConfiguration(BaseConfiguration): + LOG_BACKEND = 'console' + LOG_LEVEL = 'debug' + DEBUG = True + + SQLALCHEMY_DATABASE_URI = 'sqlite:///{0}'.format( + path.join(dbdir, 'tests', 'test_freshmaker.db')) + + MESSAGING = 'in_memory' + PDC_URL = 'http://pdc.fedoraproject.org/rest_api/v1' + + # Global network-related values, in seconds + NET_TIMEOUT = 3 + NET_RETRY_INTERVAL = 1 + MBS_AUTH_TOKEN = "testingtoken" + + KOJI_CONTAINER_SCRATCH_BUILD = True + + LIGHTBLUE_SERVER_URL = '' # replace with real dev server url + LIGHTBLUE_VERIFY_SSL = False + + +class ProdConfiguration(BaseConfiguration): + AUTH_BACKEND = 'openidc' + # use kerberos for talking to koji + KRB_AUTH_USE_KEYTAB = True + + PDC_INSECURE = {{ freshmaker_pdc_insecure }} + # No auth is required by Freshmaker, read-only PDC accesss is enough. + PDC_DEVELOP = True + +{% if env == 'staging' %} + SECRET_KEY = "{{ freshmaker_stg_secret_key }}" + + AUTH_OPENIDC_USERINFO_URI = 'https://id.stg.fedoraproject.org/openidc/UserInfo' + + SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://freshmaker:{{freshmaker_stg_db_password}}@db-freshmaker/freshmaker{{ '?sslmode=require' if freshmaker_force_postgres_ssl else '' }}' + + KOJI_PROFILE = 'freshmaker_stg' + + PDC_URL = 'https://pdc.stg.fedoraproject.org/rest_api/v1' + + GIT_BASE_URL = "{{ freshmaker_stg_git_base_url }}" + GIT_SSH_BASE_URL = "{{ freshmaker_stg_git_ssh_base_url }}" + GIT_USER = "{{ freshmaker_stg_git_user }}" + + ODCS_SERVER_URL = "{{ freshmaker_prod_odcs_server_url }}" + ODCS_SIGKEYS = {{ freshmaker_prod_odcs_sigkeys }} + + KRB_AUTH_CLIENT_KEYTAB = "{{ freshmaker_stg_krb_auth_client_keytab }}" + KRB_AUTH_PRINCIPAL = "{{ freshmaker_stg_krb_auth_principal }}" +{% else %} + SECRET_KEY = "{{ freshmaker_prod_secret_key }}" + + AUTH_OPENIDC_USERINFO_URI = "{{ freshmaker_prod_auth_openidc_userinfo_uri }}" + + SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://freshmaker:{{freshmaker_prod_db_password}}@db-freshmaker/freshmaker{{ '?sslmode=require' if freshmaker_force_postgres_ssl else '' }}' + + KOJI_PROFILE = "freshmaker_production" + + PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1' + + GIT_BASE_URL = "{{ freshmaker_prod_git_base_url }}" + GIT_SSH_BASE_URL = "{{ freshmaker_prod_git_ssh_base_url }}" + GIT_USER = "{{ freshmaker_prod_git_user }}" + + ODCS_SERVER_URL = "{{ freshmaker_prod_odcs_server_url }}" + ODCS_SIGKEYS = {{ freshmaker_prod_odcs_sigkeys }} + + KRB_AUTH_CLIENT_KEYTAB = "{{ freshmaker_prod_krb_auth_client_keytab }}" + KRB_AUTH_PRINCIPAL = "{{ freshmaker_prod_krb_auth_principal }}" +{% endif %} + + # requests_kerberos module does not support setting keytab, but the krb5 + # library checks the KRB5_CLIENT_KTNAME environment variable to set the + # path to keytab. + environ["KRB5_CLIENT_KTNAME"] = KRB_AUTH_CLIENT_KEYTAB + + MESSAGING = 'fedmsg' + MESSAGING_SENDER = 'fedmsg' + MESSAGING_BACKENDS = { + 'fedmsg': { + 'SERVICE': 'freshmaker', + }, + 'in_memory': { + 'SERVICE': 'freshmaker', + } + } + + MESSAGING_TOPIC_PREFIX = [ + {% for prefix in freshmaker_messaging_topic_prefix %} + '{{ prefix }}', + {% endfor %} + ] + + PARSERS = [ + {% for parser in freshmaker_parsers %} + '{{ parser }}', + {% endfor %} + ] + + HANDLERS = [ + {% for handler in freshmaker_handlers %} + '{{ handler }}', + {% endfor %} + ] + +{% if freshmaker_handler_build_whitelist %} + HANDLER_BUILD_WHITELIST = {{ freshmaker_handler_build_whitelist }} +{% endif %} + +{% if freshmaker_handler_build_blacklist %} + HANDLER_BUILD_BLACKLIST = {{ freshmaker_handler_build_blacklist }} +{% endif %} + + DRY_RUN = {{ freshmaker_dry_run }} + + ADMINS = {{ freshmaker_admins }} + + LOG_LEVEL = "{{ freshmaker_log_level }}" + SERVER_NAME = "{{ freshmaker_servername }}" diff --git a/roles/freshmaker/frontend/defaults/main.yml b/roles/freshmaker/frontend/defaults/main.yml new file mode 100644 index 0000000000..2513ba4fc6 --- /dev/null +++ b/roles/freshmaker/frontend/defaults/main.yml @@ -0,0 +1,8 @@ +--- +freshmaker_upgrade: False +freshmaker_migrate_db: False +freshmaker_force_ssl: True +freshmaker_endpoint: '' +freshmaker_allowed_named_hosts: [] +freshmaker_allowed_hosts: [] +freshmaker_servername: localhost diff --git a/roles/freshmaker/frontend/meta/main.yml b/roles/freshmaker/frontend/meta/main.yml new file mode 100644 index 0000000000..4a5c132f49 --- /dev/null +++ b/roles/freshmaker/frontend/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - { role: freshmaker/base } diff --git a/roles/freshmaker/frontend/tasks/main.yml b/roles/freshmaker/frontend/tasks/main.yml new file mode 100644 index 0000000000..8da7c575eb --- /dev/null +++ b/roles/freshmaker/frontend/tasks/main.yml @@ -0,0 +1,47 @@ +--- +- name: install the latest Freshmaker package + yum: + name: freshmaker + state: latest + update_cache: yes + with_items: + - freshmaker + when: freshmaker_upgrade + notify: + - restart apache + tags: + - freshmaker + - freshmaker/frontend + +- name: ensure selinux lets httpd talk to postgres + seboolean: name={{item}} state=yes persistent=yes + with_items: + - httpd_can_network_connect_db + - httpd_can_network_connect + when: "'enabled' in ansible_selinux.status" + tags: + - freshmaker + - freshmaker/frontend + - selinux + +- name: make httpd logs world readable + file: + name: /var/log/httpd + state: directory + mode: 0755 + tags: + - freshmaker + - freshmaker/frontend + +- name: generate the Freshmaker httpd config + template: + src: etc/httpd/conf.d/freshmaker.conf.j2 + dest: /etc/httpd/conf.d/freshmaker.conf + owner: apache + group: apache + mode: 0440 + notify: + - restart apache + tags: + - freshmaker + - freshmaker/frontend diff --git a/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 b/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 new file mode 100644 index 0000000000..213002c28e --- /dev/null +++ b/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 @@ -0,0 +1,49 @@ +{% if freshmaker_force_ssl %} +# Force SSL +RewriteEngine On +RewriteCond %{HTTPS} off +RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI} +{% endif %} + +WSGIDaemonProcess freshmaker user=fedmsg group=fedmsg threads=5 home=/usr/share/freshmaker +WSGIScriptAlias /{{ freshmaker_endpoint }} /usr/share/freshmaker/freshmaker.wsgi + +{% if freshmaker_servername != inventory_hostname %} +# Redirect from the hostname of this machine to user-visible hostname. +RewriteEngine On + +RewriteRule (.*) "%{REQUEST_SCHEME}://{{ freshmaker_servername }}%{REQUEST_URI}" [R,L] + +{% endif %} + +{% if env == 'staging' %} +OIDCOAuthClientID {{ freshmaker_stg_oidc_client_id }} +OIDCOAuthClientSecret {{ freshmaker_stg_oidc_client_secret }} +OIDCOAuthIntrospectionEndpoint https://id.stg.fedoraproject.org/openidc/TokenInfo +{% else %} +OIDCOAuthClientID {{ freshmaker_prod_oidc_client_id }} +OIDCOAuthClientSecret {{ freshmaker_prod_oidc_client_secret }} +OIDCOAuthIntrospectionEndpoint https://id.fedoraproject.org/openidc/TokenInfo +{% endif %} + +OIDCOAuthIntrospectionEndpointAuth client_secret_post +OIDCOAuthIntrospectionEndpointParams token_type_hint=Bearer + + + WSGIProcessGroup freshmaker + WSGIApplicationGroup %{GLOBAL} + + {% if freshmaker_allowed_named_hosts or freshmaker_allowed_hosts %} + # Only requests from following hosts/ips are allowed. + + {{ 'Require host ' ~ freshmaker_allowed_named_hosts|join(' ') if freshmaker_allowed_named_hosts else '' }} + {{ 'Require ip ' ~ freshmaker_allowed_hosts|join(' ') if freshmaker_allowed_hosts else '' }} + + {% endif %} + + {% if not freshmaker_allowed_named_hosts and not freshmaker_allowed_hosts %} + # No auth mechanism configured, so everyone is allowed to access Freshmaker. + Require all granted + {% endif %} + + From 479a0aedca14d7f15ce1d96fe2bdbcc2735d914e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Mon, 29 Jan 2018 14:26:33 +0100 Subject: [PATCH 064/242] retrace: remove f25 --- roles/abrt/faf-local/tasks/cron.yml | 4 ++-- roles/abrt/retrace-local/defaults/main.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/abrt/faf-local/tasks/cron.yml b/roles/abrt/faf-local/tasks/cron.yml index 5cad1d4cae..29eaba1fe3 100644 --- a/roles/abrt/faf-local/tasks/cron.yml +++ b/roles/abrt/faf-local/tasks/cron.yml @@ -49,7 +49,6 @@ state: present when: not devel with_items: - - "25" - "26" - "27" @@ -63,6 +62,7 @@ when: not devel with_items: - "24" + - "25" - name: koops_to_xorg.py cron: @@ -82,7 +82,6 @@ state: present when: not devel with_items: - - "25" - "26" - "27" @@ -96,6 +95,7 @@ when: not devel with_items: - "24" + - "25" - name: update BZ bugs fedora cron: diff --git a/roles/abrt/retrace-local/defaults/main.yml b/roles/abrt/retrace-local/defaults/main.yml index cd32bde882..89b833be06 100644 --- a/roles/abrt/retrace-local/defaults/main.yml +++ b/roles/abrt/retrace-local/defaults/main.yml @@ -1,8 +1,8 @@ --- # List of fedora versions for reposync -rs_internal_fedora_vers: [25, 26, 27, rawhide] -rs_internal_fedora_vers_removed: [24] +rs_internal_fedora_vers: [26, 27, rawhide] +rs_internal_fedora_vers_removed: [24, 25] # List of architectures for reposync # armhfp disabled untill we get more space From b27f0508df1aa140e462f2ac2b57c5e8e43fdf4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Mon, 29 Jan 2018 14:57:11 +0100 Subject: [PATCH 065/242] retrace: do not block on long task --- roles/abrt/retrace/tasks/usefafpkgs.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/abrt/retrace/tasks/usefafpkgs.yml b/roles/abrt/retrace/tasks/usefafpkgs.yml index 5a589db84b..458c4f25b5 100644 --- a/roles/abrt/retrace/tasks/usefafpkgs.yml +++ b/roles/abrt/retrace/tasks/usefafpkgs.yml @@ -18,11 +18,15 @@ - name: ACL for user retrace acl: path="{{ faf_spool_dir }}/lob" state=present recursive=yes entity=retrace etype=user permissions=rwX + async: 21600 + pool: 0 # for files/dirs created in future - name: default ACL for user retrace acl: path="{{ faf_spool_dir }}/lob" state=present recursive=yes default=yes entity=retrace etype=user permissions=rwX + async: 21600 + pool: 0 - name: check for hardlink dir stat: path={{ rs_faf_link_dir }} From 750f0c7333a8cd2231b40872d10e016d0a01d918 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 29 Jan 2018 17:01:51 +0000 Subject: [PATCH 066/242] switch this to use package --- playbooks/manual/upgrade/fedmsg.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/manual/upgrade/fedmsg.yml b/playbooks/manual/upgrade/fedmsg.yml index 56e4017831..459b25163b 100644 --- a/playbooks/manual/upgrade/fedmsg.yml +++ b/playbooks/manual/upgrade/fedmsg.yml @@ -40,11 +40,11 @@ command: yum clean all {%if testing%} --enablerepo=infrastructure-tags-stg {%endif%} check_mode: no - name: yum update fedmsg packages from the main repo - yum: name={{item}} state=latest + package: name={{item}} state=latest when: not testing with_items: "{{packages}}" - name: yum update fedmsg packages from testing repo - yum: name={{item}} state=latest enablerepo=infrastructure-tags-stg + package: name={{item}} state=latest enablerepo=infrastructure-tags-stg when: testing with_items: "{{packages}}" From e7a8feb2d7713ae30c35524b322ab72662e1f0ff Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 29 Jan 2018 17:09:33 +0000 Subject: [PATCH 067/242] try this --- playbooks/manual/upgrade/fedmsg.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/manual/upgrade/fedmsg.yml b/playbooks/manual/upgrade/fedmsg.yml index 459b25163b..29f91bebd9 100644 --- a/playbooks/manual/upgrade/fedmsg.yml +++ b/playbooks/manual/upgrade/fedmsg.yml @@ -49,7 +49,7 @@ with_items: "{{packages}}" # Restart all the backend daemons - - include_tasks: ../restart-fedmsg-services.yml + - import_tasks: ../restart-fedmsg-services.yml # Also restart the frontend web services - name: bounce apache From 1cb7267a6a7cf83015fdf257886dcfea825be2c1 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 29 Jan 2018 17:27:02 +0000 Subject: [PATCH 068/242] disable this for now --- playbooks/manual/upgrade/fedmsg.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/manual/upgrade/fedmsg.yml b/playbooks/manual/upgrade/fedmsg.yml index 29f91bebd9..68feebb2f3 100644 --- a/playbooks/manual/upgrade/fedmsg.yml +++ b/playbooks/manual/upgrade/fedmsg.yml @@ -49,7 +49,7 @@ with_items: "{{packages}}" # Restart all the backend daemons - - import_tasks: ../restart-fedmsg-services.yml + #- import_tasks: "{{tasks_path}}../restart-fedmsg-services.yml" # Also restart the frontend web services - name: bounce apache From 9fa7b8db9f18a57d4f303174e26b1182c50b665d Mon Sep 17 00:00:00 2001 From: Andrea Veri Date: Tue, 30 Jan 2018 14:45:19 +0000 Subject: [PATCH 069/242] chooser and combobox are being decommissioned --- roles/gnome_backups/files/backup.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/gnome_backups/files/backup.sh b/roles/gnome_backups/files/backup.sh index ede7629c32..9433a5cd11 100644 --- a/roles/gnome_backups/files/backup.sh +++ b/roles/gnome_backups/files/backup.sh @@ -9,14 +9,13 @@ MACHINES='signal.gnome.org webapps2.gnome.org clutter.gnome.org blogs.gnome.org - chooser.gnome.org + palette.gnome.org git.gnome.org webapps.gnome.org cloud.gnome.org bastion.gnome.org spinner.gnome.org master.gnome.org - combobox.gnome.org restaurant.gnome.org expander.gnome.org live.gnome.org From 0a1fe3862ff44ebfd541a69b1c1c2310b0050c72 Mon Sep 17 00:00:00 2001 From: Andrea Veri Date: Tue, 30 Jan 2018 19:31:48 +0000 Subject: [PATCH 070/242] GNOME backups: s/choose/palette, combobox has been decommissioned --- roles/gnome_backups/tasks/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/gnome_backups/tasks/main.yml b/roles/gnome_backups/tasks/main.yml index 84dde09352..4f9020ad33 100644 --- a/roles/gnome_backups/tasks/main.yml +++ b/roles/gnome_backups/tasks/main.yml @@ -32,7 +32,7 @@ - view.gnome.org - puppet.gnome.org - extensions.gnome.org - - chooser.gnome.org + - palette.gnome.org - git.gnome.org - webapps.gnome.org - socket.gnome.org @@ -43,7 +43,6 @@ - spinner.gnome.org - master.gnome.org - live.gnome.org - - combobox.gnome.org - restaurant.gnome.org - expander.gnome.org - accelerator.gnome.org From 38ed0ef84ce81bddc7c4b205e2230a3ab6f3b0ec Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Wed, 31 Jan 2018 00:17:59 +0100 Subject: [PATCH 071/242] Allow openQA staging to publish 'ci' fedmsgs This is part of a Big Secret Red Hat Conspiracy. Look away now! OK, we're working on a standardized messaging format for testing related messages, and this is part of testing out how that looks with a Fedora testing system. Won't go to prod till it's a bit more locked in. Signed-off-by: Adam Williamson --- inventory/group_vars/openqa-stg | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/inventory/group_vars/openqa-stg b/inventory/group_vars/openqa-stg index 9b10299df8..176ae8ffa7 100644 --- a/inventory/group_vars/openqa-stg +++ b/inventory/group_vars/openqa-stg @@ -71,6 +71,14 @@ fedmsg_certs: - openqa.jobs.restart - openqa.job.update.result - openqa.job.done +- service: ci + owner: root + group: geekotest + can_send: + - ci.productmd-compose.test.queued + - ci.productmd-compose.test.running + - ci.productmd-compose.test.complete + - ci.productmd-compose.test.error # we need this to log with fedmsg-logger fedmsg_active: True From d5c712d465ddb752f54a6bc310a3049d8a6c536f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20P=C3=A1ral?= Date: Wed, 31 Jan 2018 15:46:53 +0100 Subject: [PATCH 072/242] taskotron-client: add taskotron-ansiblize copr into minion_repos --- roles/taskotron/taskotron-client/templates/taskotron.yaml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/taskotron/taskotron-client/templates/taskotron.yaml.j2 b/roles/taskotron/taskotron-client/templates/taskotron.yaml.j2 index 0d026cad41..97d2833a5e 100644 --- a/roles/taskotron/taskotron-client/templates/taskotron.yaml.j2 +++ b/roles/taskotron/taskotron-client/templates/taskotron.yaml.j2 @@ -157,6 +157,7 @@ minion_repos: {% endif %} {% if deployment_type == 'dev' %} - https://copr.fedorainfracloud.org/coprs/kparal/taskotron-dev/repo/fedora-26/kparal-taskotron-dev-fedora-26.repo + - https://copr.fedorainfracloud.org/coprs/kparal/taskotron-ansiblize/repo/fedora-27/kparal-taskotron-ansiblize-fedora-27.repo {% endif %} From 777b1ad6766f12209606c3b058191f6396d3a499 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Wed, 31 Jan 2018 19:21:31 +0000 Subject: [PATCH 073/242] Getting logs for the bodhi expire overrides might be useful Signed-off-by: Patrick Uiterwijk --- roles/bodhi2/backend/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/bodhi2/backend/tasks/main.yml b/roles/bodhi2/backend/tasks/main.yml index 3a2dac2408..53a76ad685 100644 --- a/roles/bodhi2/backend/tasks/main.yml +++ b/roles/bodhi2/backend/tasks/main.yml @@ -330,7 +330,7 @@ - name: bodhi-expire-overrides cron job. cron: name="bodhi-expire-overrides" hour="*" minute=0 user="apache" - job="/usr/bin/bodhi-expire-overrides /etc/bodhi/production.ini 2> /dev/null" + job="/usr/bin/bodhi-expire-overrides /etc/bodhi/production.ini" cron_file=bodhi-expire-overrides-job when: inventory_hostname.startswith('bodhi-backend02') and env == "production" tags: From c68cb601bf36e9c448d977922742188b01505689 Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Wed, 31 Jan 2018 21:30:54 +0000 Subject: [PATCH 074/242] add the httpd logs from download-ib --- roles/base/files/syncHttpLogs.sh | 1 + roles/basessh/files/syncHttpLogs.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/roles/base/files/syncHttpLogs.sh b/roles/base/files/syncHttpLogs.sh index 7b3e0752f0..0e64b0b8d8 100644 --- a/roles/base/files/syncHttpLogs.sh +++ b/roles/base/files/syncHttpLogs.sh @@ -86,6 +86,7 @@ syncHttpLogs download03.phx2.fedoraproject.org syncHttpLogs download04.phx2.fedoraproject.org syncHttpLogs download05.phx2.fedoraproject.org syncHttpLogs download-rdu01.vpn.fedoraproject.org +syncHttpLogs download-ib01.vpn.fedoraproject.org syncHttpLogs sundries01.phx2.fedoraproject.org syncHttpLogs sundries02.phx2.fedoraproject.org syncHttpLogs sundries01.stg.phx2.fedoraproject.org diff --git a/roles/basessh/files/syncHttpLogs.sh b/roles/basessh/files/syncHttpLogs.sh index 7b3e0752f0..0e64b0b8d8 100644 --- a/roles/basessh/files/syncHttpLogs.sh +++ b/roles/basessh/files/syncHttpLogs.sh @@ -86,6 +86,7 @@ syncHttpLogs download03.phx2.fedoraproject.org syncHttpLogs download04.phx2.fedoraproject.org syncHttpLogs download05.phx2.fedoraproject.org syncHttpLogs download-rdu01.vpn.fedoraproject.org +syncHttpLogs download-ib01.vpn.fedoraproject.org syncHttpLogs sundries01.phx2.fedoraproject.org syncHttpLogs sundries02.phx2.fedoraproject.org syncHttpLogs sundries01.stg.phx2.fedoraproject.org From 6875a450b0538b42d274c1f0ec082691f1e37f44 Mon Sep 17 00:00:00 2001 From: Ricky Elrod Date: Wed, 31 Jan 2018 23:52:11 +0000 Subject: [PATCH 075/242] Make www.fp.o be an alias of fp.o instead of fp.c Signed-off-by: Ricky Elrod --- playbooks/include/proxies-websites.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/include/proxies-websites.yml b/playbooks/include/proxies-websites.yml index 3dccad7a77..7e062316b5 100644 --- a/playbooks/include/proxies-websites.yml +++ b/playbooks/include/proxies-websites.yml @@ -52,6 +52,7 @@ server_aliases: - stg.fedoraproject.org - localhost + - www.fedoraproject.org # This is for all the other domains we own # that redirect to https://fedoraproject.org @@ -126,7 +127,6 @@ - www.fedoraproject.info - www.fedoraproject.net - www.fedoraproject.net.cn - - www.fedoraproject.org - www.fedoraproject.org.uk - www.fedoraproject.pe - www.fedoraproject.su From d3b961e7677231e11e3d8f651e7a6c8909385c31 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 1 Feb 2018 00:31:55 +0000 Subject: [PATCH 076/242] simplify and comment on root pw setting --- roles/base/tasks/main.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/base/tasks/main.yml b/roles/base/tasks/main.yml index 406b4ec920..ca7c7f74e6 100644 --- a/roles/base/tasks/main.yml +++ b/roles/base/tasks/main.yml @@ -108,12 +108,16 @@ - name: make sure hostname is set right on rhel7 hosts hostname: name="{{inventory_hostname}}" +# +# We set builders root password in the koji_builder role, so do not set those here +# + - name: set root passwd user: name=root password={{ rootpw }} state=present tags: - rootpw - base - when: not (inventory_hostname.startswith('rawhide') or inventory_hostname.startswith('branched') or inventory_hostname.startswith('compose') or inventory_hostname.startswith('build') or inventory_hostname.startswith('arm') or inventory_hostname.startswith('bkernel') or inventory_hostname.startswith('koji01.stg') or inventory_hostname.startswith('aarch64') or inventory_hostname.startswith('s390') or inventory_hostname.startswith('fed-cloud09') or inventory_hostname.startswith('ppc8-04')) + when: not inventory_hostname.startswith(('build','bkernel','koji01.stg','s390',fed-cloud09')) - name: add ansible root key authorized_key: user=root key="{{ item }}" From 859c75ba8d94b8c60b25bb3af34116b5749feb2b Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 1 Feb 2018 00:49:27 +0000 Subject: [PATCH 077/242] everyone loves parens --- roles/base/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/base/tasks/main.yml b/roles/base/tasks/main.yml index ca7c7f74e6..79b3cb8f25 100644 --- a/roles/base/tasks/main.yml +++ b/roles/base/tasks/main.yml @@ -117,7 +117,7 @@ tags: - rootpw - base - when: not inventory_hostname.startswith(('build','bkernel','koji01.stg','s390',fed-cloud09')) + when: not (inventory_hostname.startswith(('build','bkernel','koji01.stg','s390',fed-cloud09'))) - name: add ansible root key authorized_key: user=root key="{{ item }}" From 9d1075e993737edad3a00f860b46c60e167d8603 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 1 Feb 2018 00:52:10 +0000 Subject: [PATCH 078/242] really it was a missing quote --- roles/base/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/base/tasks/main.yml b/roles/base/tasks/main.yml index 79b3cb8f25..48d66facbd 100644 --- a/roles/base/tasks/main.yml +++ b/roles/base/tasks/main.yml @@ -117,7 +117,7 @@ tags: - rootpw - base - when: not (inventory_hostname.startswith(('build','bkernel','koji01.stg','s390',fed-cloud09'))) + when: not inventory_hostname.startswith(('build','bkernel','koji01.stg','s390','fed-cloud09')) - name: add ansible root key authorized_key: user=root key="{{ item }}" From ee65d47e584f5d9d34700a13563fbeddfa09e966 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Thu, 1 Feb 2018 09:33:45 +0000 Subject: [PATCH 079/242] Hubs: update playbook to use RPM --- .../hosts/hubs-dev.fedorainfracloud.org.yml | 21 +-- roles/hubs/defaults/main.yml | 6 +- roles/hubs/files/logging.ini | 26 ---- roles/hubs/handlers/main.yml | 13 +- roles/hubs/tasks/db-postgresql.yml | 17 +-- roles/hubs/tasks/db-sqlite.yml | 4 +- roles/hubs/tasks/dev.yml | 72 +++++++-- roles/hubs/tasks/dev_deps.yml | 64 ++++++++ roles/hubs/tasks/main.yml | 143 +++--------------- roles/hubs/tasks/prod.yml | 32 ++-- roles/hubs/tasks/prod_deps.yml | 2 + roles/hubs/tasks/webserver.yml | 30 +--- roles/hubs/templates/bashrc | 8 +- roles/hubs/templates/env | 2 + roles/hubs/templates/fedmsg_config | 15 +- roles/hubs/templates/honcho-env | 2 +- roles/hubs/templates/honcho-procfile | 12 +- roles/hubs/templates/hubs-fedmsg-hub.service | 14 -- roles/hubs/templates/hubs-sse.service | 18 --- roles/hubs/templates/hubs-triage@.service | 15 -- roles/hubs/templates/hubs-webapp.service | 20 --- roles/hubs/templates/hubs-worker@.service | 15 -- roles/hubs/templates/nginx.conf | 8 +- 23 files changed, 207 insertions(+), 352 deletions(-) delete mode 100644 roles/hubs/files/logging.ini create mode 100644 roles/hubs/tasks/dev_deps.yml create mode 100644 roles/hubs/tasks/prod_deps.yml create mode 100644 roles/hubs/templates/env delete mode 100644 roles/hubs/templates/hubs-fedmsg-hub.service delete mode 100644 roles/hubs/templates/hubs-sse.service delete mode 100644 roles/hubs/templates/hubs-triage@.service delete mode 100644 roles/hubs/templates/hubs-webapp.service delete mode 100644 roles/hubs/templates/hubs-worker@.service diff --git a/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml b/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml index 6c9b7fcdc3..ed2ec5827b 100644 --- a/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml +++ b/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml @@ -33,33 +33,18 @@ - import_tasks: "{{ tasks_path }}/yumrepos.yml" - - dnf: name={{item}} state=present - with_items: - - git - # for certbot - - httpd - - - name: create the code directory - file: dest=/srv/hubs state=directory owner=fedora group=fedora - - - name: git clone the code - git: repo=https://pagure.io/fedora-hubs.git - dest=/srv/hubs/fedora-hubs - version=develop - become: true - become_user: fedora - #ignore_errors: true - roles: - basessh - role: hubs - main_user: fedora + main_user: hubs hubs_url_hostname: "{{ ansible_fqdn }}" hubs_secret_key: demotestinghubsmachine hubs_db_type: postgresql hubs_dev_mode: false + hubs_conf_dir: /etc/fedora-hubs + hubs_var_dir: /var/lib/fedora-hubs hubs_ssl_cert: /etc/letsencrypt/live/{{ ansible_fqdn }}/fullchain.pem hubs_ssl_key: /etc/letsencrypt/live/{{ ansible_fqdn }}/privkey.pem hubs_fas_username: "{{ fedoraDummyUser }}" diff --git a/roles/hubs/defaults/main.yml b/roles/hubs/defaults/main.yml index f3d14cb644..8eb42b7a21 100644 --- a/roles/hubs/defaults/main.yml +++ b/roles/hubs/defaults/main.yml @@ -4,7 +4,6 @@ hubs_secret_key: changeme hubs_base_dir: "/srv/hubs" hubs_code_dir: "{{ hubs_base_dir }}/fedora-hubs" hubs_conf_dir: "{{ hubs_base_dir }}/config" -hubs_venv_dir: "{{ hubs_base_dir }}/venv" hubs_var_dir: "{{ hubs_base_dir }}/var" hubs_db_type: sqlite hubs_db_password: changeme @@ -12,5 +11,6 @@ hubs_url_hostname: "{{ ansible_fqdn }}" hubs_url: http{% if not hubs_dev_mode %}s{% endif %}://{{ hubs_url_hostname }}{% if hubs_dev_mode %}:5000{% endif %} hubs_ssl_cert: /etc/pki/tls/certs/{{ hubs_url_hostname }}.crt hubs_ssl_key: /etc/pki/tls/private/{{ hubs_url_hostname }}.key -hubs_fas_username: changeme -hubs_fas_password: changeme +hubs_fas_username: null +hubs_fas_password: null +hubs_oidc_url: iddev.fedorainfracloud.org diff --git a/roles/hubs/files/logging.ini b/roles/hubs/files/logging.ini deleted file mode 100644 index f1a3bc0ddd..0000000000 --- a/roles/hubs/files/logging.ini +++ /dev/null @@ -1,26 +0,0 @@ -[loggers] -keys=root - -[handlers] -keys=console - -[formatters] -keys=simple,minimal - -[logger_root] -level=DEBUG -handlers=console - -[handler_console] -class=StreamHandler -level=DEBUG -formatter=minimal -args=(sys.stdout,) - -[formatter_simple] -format=[%(asctime)s][%(process)d][%(levelname)s] (%(name)s) %(message)s -datefmt=%H:%M:%S - -[formatter_minimal] -format=[%(levelname)s] (%(name)s) %(message)s -datefmt=%H:%M:%S diff --git a/roles/hubs/handlers/main.yml b/roles/hubs/handlers/main.yml index 7f7235b2e8..4bc0f93892 100644 --- a/roles/hubs/handlers/main.yml +++ b/roles/hubs/handlers/main.yml @@ -1,29 +1,24 @@ - name: restart postgresql service: name=postgresql state=restarted -- name: restart the hubs-specific fedmsg-hub - service: name=hubs-fedmsg-hub state=restarted - listen: "hubs configuration change" - when: not hubs_dev_mode - - name: restart hubs triage - service: name=hubs-triage@* state=restarted + service: name=fedora-hubs-triage@* state=restarted listen: "hubs configuration change" when: not hubs_dev_mode - name: restart hubs workers - service: name=hubs-worker@* state=restarted + service: name=fedora-hubs-worker@* state=restarted listen: "hubs configuration change" when: not hubs_dev_mode - name: restart hubs SSE server - service: name=hubs-sse state=restarted + service: name=fedora-hubs-sse state=restarted listen: "hubs configuration change" when: not hubs_dev_mode # Webserver - name: restart hubs webapp - service: name=hubs-webapp state=restarted + service: name=fedora-hubs-webapp state=restarted listen: "hubs configuration change" when: not hubs_dev_mode diff --git a/roles/hubs/tasks/db-postgresql.yml b/roles/hubs/tasks/db-postgresql.yml index e3af324019..aabb48a80d 100644 --- a/roles/hubs/tasks/db-postgresql.yml +++ b/roles/hubs/tasks/db-postgresql.yml @@ -5,7 +5,7 @@ with_items: - postgresql-server - python3-psycopg2 - # For the ansible modules + # For the ansible module - python-psycopg2 - name: Set up postgresql database @@ -41,21 +41,12 @@ become: true become_user: postgres -- name: Ease local access to the database - copy: - content: "*:*:hubs:hubs:{{ hubs_db_password }}" - dest: /home/{{ main_user }}/.pgpass - mode: 600 - owner: "{{ main_user }}" - group: "{{ main_user }}" - - name: Populate the Fedora Hubs database - command: "{{ hubs_venv_dir }}/bin/python {{ hubs_code_dir }}/populate.py" + command: "python3 {{ hubs_code_dir }}/populate.py" args: chdir: "{{ hubs_code_dir }}" environment: - HUBS_CONFIG: "{{ hubs_conf_dir }}/hubs_config.py" + HUBS_CONFIG: "{{ hubs_conf_dir }}/hubs.py" become: true become_user: "{{ main_user }}" - when: db_creation|succeeded and db_creation|changed - + when: db_creation|succeeded and db_creation|changed and hubs_dev_mode diff --git a/roles/hubs/tasks/db-sqlite.yml b/roles/hubs/tasks/db-sqlite.yml index 9d15e816d5..e8397277e8 100644 --- a/roles/hubs/tasks/db-sqlite.yml +++ b/roles/hubs/tasks/db-sqlite.yml @@ -1,9 +1,9 @@ - name: Create and populate the Fedora Hubs database - command: "{{ hubs_venv_dir }}/bin/python {{ hubs_code_dir }}/populate.py" + command: "python3 {{ hubs_code_dir }}/populate.py" args: creates: "{{ hubs_var_dir }}/hubs.db" chdir: "{{ hubs_code_dir }}" environment: - HUBS_CONFIG: "{{ hubs_conf_dir }}/hubs_config.py" + HUBS_CONFIG: "{{ hubs_conf_dir }}/hubs.py" become: true become_user: "{{ main_user }}" diff --git a/roles/hubs/tasks/dev.yml b/roles/hubs/tasks/dev.yml index 6b01c9b8fc..2b09949be8 100644 --- a/roles/hubs/tasks/dev.yml +++ b/roles/hubs/tasks/dev.yml @@ -1,17 +1,69 @@ -- name: Install Fedora Hubs development packages +# Set up the Python development environment + +- name: Install Fedora Hubs requirements.txt into hubs virtualenv + pip: + requirements: "{{ hubs_code_dir }}/requirements.txt" + executable: pip3 + +- name: Install Fedora Hubs test-requirements.txt into hubs virtualenv + pip: + requirements: "{{ hubs_code_dir }}/test-requirements.txt" + executable: pip3 + +- name: Install other packages into hubs virtualenv + pip: + name: "{{ item }}" + executable: pip3 + with_items: + - bleach + +- name: Install Fedora Hubs into the virtualenv + command: "pip3 install -e {{ hubs_code_dir }}" + args: + creates: "/usr/lib/python3.6/site-packages/fedora-hubs.egg-link" + + +# Set up JavaScript requirements + +- name: Install npm packages + command: npm install + become: true + become_user: "{{ main_user }}" + args: + creates: node_modules + chdir: "{{ hubs_code_dir }}/hubs/static/client" + +- name: Build JavaScript assets + command: npm run build + become: true + become_user: "{{ main_user }}" + args: + chdir: "{{ hubs_code_dir }}/hubs/static/client" + creates: "{{ hubs_code_dir }}/hubs/static/js/build/common.js" + + +# Development tools + +- name: Install helpful development packages + dnf: name={{ item }} state=present + with_items: + - git + - vim-enhanced + +- name: Install Fedora Hubs development tools dnf: name={{ item }} state=present with_items: - - gcc - - gcc-c++ - - libffi-devel - - openssl-devel - - python-sphinx - - python2-devel - - python3-devel - python3-honcho - python3-tox - - redhat-rpm-config - - sqlite-devel + +- name: Ease local access to the database + copy: + content: "*:*:hubs:hubs:{{ hubs_db_password }}" + dest: /home/{{ main_user }}/.pgpass + mode: 600 + owner: "{{ main_user }}" + group: "{{ main_user }}" + when: hubs_db_type == "postgresql" - name: Install a custom bashrc template: src=bashrc dest=/home/{{ main_user }}/.bashrc diff --git a/roles/hubs/tasks/dev_deps.yml b/roles/hubs/tasks/dev_deps.yml new file mode 100644 index 0000000000..38ba4ba7f1 --- /dev/null +++ b/roles/hubs/tasks/dev_deps.yml @@ -0,0 +1,64 @@ +- name: Install Fedora Hubs development packages + dnf: name={{ item }} state=present + with_items: + - gcc + - gcc-c++ + - libffi-devel + - openssl-devel + - python-sphinx + - python2-devel + - python3-devel + - python3-virtualenv + - python3-flask-oidc + - python3-moksha-common + - redhat-rpm-config + - sqlite-devel + - npm + - fedmsg-hub + +- name: Install the distribution versions of requirements.txt + dnf: name={{ item }} state=present + with_items: + - python3-alembic + - python3-arrow + - python3-beautifulsoup4 + - python3-bleach + - python3-blinker + - python3-dateutil + - python3-decorator + - python3-dogpile-cache + - python3-fedmsg + - python3-fedmsg-meta-fedora-infrastructure + - python3-fedora + - python3-flask + - python3-flask-oidc + - python3-html5lib + - python3-humanize + - python3-iso3166 + - python3-markdown + - python3-munch + - python3-pkgwat-api + - python3-pygments + - python3-pygments-markdown-lexer + - python3-pymongo + - python3-pytz + - python3-redis + - python3-requests + - python3-retask + - python3-six + - python3-sqlalchemy + - python3-twisted + + +- name: Create the directory structure + file: + path: "{{ item.path }}" + state: directory + owner: "{{ main_user }}" + group: "{{ main_user }}" + mode: "{{ item.mode }}" + #setype: httpd_sys_content_rw_t + with_items: + - {path: "{{ hubs_base_dir }}", mode: 755} + - {path: "{{ hubs_conf_dir }}", mode: 750} + - {path: "{{ hubs_var_dir }}", mode: 750} diff --git a/roles/hubs/tasks/main.yml b/roles/hubs/tasks/main.yml index bb7a79ee79..6f858e896f 100644 --- a/roles/hubs/tasks/main.yml +++ b/roles/hubs/tasks/main.yml @@ -1,139 +1,57 @@ --- -- name: Install helpful development packages - dnf: name={{ item }} state=present - with_items: - - git - - vim-enhanced - - name: Install external dependencies dnf: name={{ item }} state=present with_items: - - npm - redis - - fedmsg-hub - - fedmsg-relay - - python3-virtualenv - - python3-flask-oidc - - python3-moksha-common + - python3-fedmsg - postfix -- name: Install the distribution versions of requirements.txt - dnf: name={{ item }} state=present - with_items: - - python3-alembic - - python3-arrow - - python3-bleach - - python3-decorator - - python3-dogpile-cache - - python3-fedmsg-core - - python3-fedmsg-meta-fedora-infrastructure - - python3-flask - - python3-flask-oidc - - python3-html5lib - - python3-munch - - python3-pytz - - python3-sqlalchemy - - python3-markdown - - python3-pkgwat-api - - python3-six - - python3-pygments - - python3-pygments-markdown-lexer - - python3-retask - - python3-twisted +- include_tasks: dev_deps.yml + when: hubs_dev_mode -# Create directory structure +- include_tasks: prod_deps.yml + when: not hubs_dev_mode -- name: Create the directory structure - file: - path: "{{ item.path }}" - state: directory - owner: "{{ main_user }}" - group: "{{ main_user }}" - mode: "{{ item.mode }}" - #setype: httpd_sys_content_rw_t - with_items: - - {path: "{{ hubs_base_dir }}", mode: 755} - - {path: "{{ hubs_conf_dir }}", mode: 750} - - {path: "{{ hubs_var_dir }}", mode: 750} - - -# Set up the Python development environment -- name: Install Fedora Hubs requirements.txt into hubs virtualenv - become: true - become_user: "{{ main_user }}" - pip: - requirements: "{{ hubs_code_dir }}/requirements.txt" - virtualenv: "{{ hubs_venv_dir}}" - virtualenv_site_packages: yes - virtualenv_command: virtualenv-3 - -- name: Install Fedora Hubs test-requirements.txt into hubs virtualenv - become: true - become_user: "{{ main_user }}" - pip: - requirements: "{{ hubs_code_dir }}/test-requirements.txt" - virtualenv: "{{ hubs_venv_dir}}" - virtualenv_site_packages: yes - virtualenv_command: virtualenv-3 - -- name: Install other packages into hubs virtualenv - become: true - become_user: "{{ main_user }}" - pip: - name: "{{ item }}" - virtualenv: "{{ hubs_venv_dir }}" - virtualenv_site_packages: yes - virtualenv_command: virtualenv-3 - with_items: - - bleach - -- name: Install Fedora Hubs into the virtualenv - become: true - become_user: "{{ main_user }}" - command: "{{ hubs_venv_dir }}/bin/pip install -e {{ hubs_code_dir }}" - args: - creates: "{{ hubs_venv_dir }}/lib/python3.6/site-packages/fedora-hubs.egg-link" - -- name: Set bin file context in the virtualenv - become: true - become_user: "{{ main_user }}" - file: - path: "{{ hubs_venv_dir }}/bin" - state: directory - recurse: true - setype: bin_t - name: Add a basic Hubs configuration file template: src: "{{ item }}" - dest: "{{ hubs_conf_dir }}/hubs_config.py" + dest: "{{ hubs_conf_dir }}/hubs.py" + owner: root + group: "{{ main_user }}" + mode: 0640 with_first_found: - hubs_config.{{ ansible_hostname }} - hubs_config - become: true - become_user: "{{ main_user }}" notify: "hubs configuration change" + - name: Add a basic fedmsg configuration file template: src: "{{ item }}" - dest: "/etc/fedmsg.d/hubs_config.py" + dest: "/etc/fedmsg.d/fedora-hubs.py" with_first_found: - fedmsg_config.{{ ansible_hostname }} - fedmsg_config notify: "hubs configuration change" + - name: Configure application to authenticate with iddev.fedorainfracloud.org command: oidc-register --output-file {{ hubs_conf_dir }}/client_secrets.json - https://iddev.fedorainfracloud.org/ {{ hubs_url }} - become: true - become_user: "{{ main_user }}" + https://{{ hubs_oidc_url }}/ {{ hubs_url }} args: creates: "{{ hubs_conf_dir }}/client_secrets.json" +- name: Set permissions on the oidc credentials file + file: + path: "{{ hubs_conf_dir }}/client_secrets.json" + owner: root + group: "{{ main_user }}" + mode: 0640 + - name: Start and enable the common services service: name={{ item }} state=started enabled=yes @@ -145,29 +63,14 @@ - include_tasks: db-{{ hubs_db_type }}.yml -# Set up JavaScript requirements -- name: Install npm packages - command: npm install - become: true - become_user: "{{ main_user }}" - args: - creates: node_modules - chdir: "{{ hubs_code_dir }}/hubs/static/client" - -- name: Build JavaScript assets - command: npm run build - become: true - become_user: "{{ main_user }}" - args: - chdir: "{{ hubs_code_dir }}/hubs/static/client" - creates: "{{ hubs_code_dir }}/hubs/static/js/build/common.js" - - # Services - name: Disable the system-wide fedmsg daemons service: name={{ item }} state=stopped enabled=no with_items: + # We use honcho in dev mode and fedmsg-hub-3 in prod mode - fedmsg-hub + # We use honcho in dev mode and fedmsg-relay-3 in prod mode + - fedmsg-relay # Include mode-specific tasks diff --git a/roles/hubs/tasks/prod.yml b/roles/hubs/tasks/prod.yml index e9393e8a37..31c29dff2a 100644 --- a/roles/hubs/tasks/prod.yml +++ b/roles/hubs/tasks/prod.yml @@ -1,27 +1,19 @@ -- name: Install the service files +- name: Install the service environment file template: - src: "{{ item }}.service" - dest: /etc/systemd/system/{{ item }}.service - with_items: - - hubs-triage@ - - hubs-worker@ - - hubs-sse - - hubs-fedmsg-hub - register: service_installed - -- name: reload systemd - command: systemctl daemon-reload - when: service_installed|changed + src: env + dest: /etc/sysconfig/fedora-hubs - name: Start and enable the services in prod mode service: name={{ item }} state=started enabled=yes with_items: - - fedmsg-relay - - hubs-triage@1 - - hubs-triage@2 - - hubs-worker@1 - - hubs-worker@2 - - hubs-sse - - hubs-fedmsg-hub + - fedmsg-relay-3 + - fedmsg-hub-3 + - fedora-hubs-triage@1 + - fedora-hubs-triage@2 + - fedora-hubs-worker@1 + - fedora-hubs-worker@2 + - fedora-hubs-worker@3 + - fedora-hubs-worker@4 + - fedora-hubs-sse - include_tasks: webserver.yml diff --git a/roles/hubs/tasks/prod_deps.yml b/roles/hubs/tasks/prod_deps.yml new file mode 100644 index 0000000000..b078a6470f --- /dev/null +++ b/roles/hubs/tasks/prod_deps.yml @@ -0,0 +1,2 @@ +- name: Install the Fedora Hubs package + dnf: name=fedora-hubs state=present diff --git a/roles/hubs/tasks/webserver.yml b/roles/hubs/tasks/webserver.yml index 23152c9ab0..4224cd2b7f 100644 --- a/roles/hubs/tasks/webserver.yml +++ b/roles/hubs/tasks/webserver.yml @@ -11,7 +11,7 @@ - name: install python3-certbot-nginx dnf: name=python3-certbot-nginx state=present -- name: get the letencrypt cert +- name: get the letsencrypt cert command: certbot certonly -n --standalone --pre-hook "systemctl stop nginx" --post-hook "systemctl start nginx" -d {{ ansible_fqdn }} --agree-tos --email admin@fedoraproject.org args: creates: /etc/letsencrypt/live/{{ ansible_fqdn }}/privkey.pem @@ -19,20 +19,10 @@ - restart nginx -- name: Gunicorn logging configuration - copy: - src: logging.ini - dest: "{{ hubs_conf_dir }}/logging.ini" - owner: "{{ main_user }}" - group: "{{ main_user }}" - notify: - - restart hubs webapp - - - name: Nginx configuration for hubs template: src: nginx.conf - dest: /etc/nginx/conf.d/hubs.conf + dest: /etc/nginx/conf.d/fedora-hubs.conf notify: - restart nginx @@ -66,22 +56,8 @@ persistent: yes -- name: Install the service files - template: - src: "{{ item }}.service" - dest: /etc/systemd/system/{{ item }}.service - with_items: - - hubs-webapp - register: service_installed - - -- name: reload systemd - command: systemctl daemon-reload - when: service_installed|changed - - - name: Start and enable the services service: name={{ item }} state=started enabled=yes with_items: - - hubs-webapp + - fedora-hubs-webapp - nginx diff --git a/roles/hubs/templates/bashrc b/roles/hubs/templates/bashrc index c1f32e2910..eaaf5d4c49 100644 --- a/roles/hubs/templates/bashrc +++ b/roles/hubs/templates/bashrc @@ -5,6 +5,9 @@ if [ -f /etc/bashrc ]; then . /etc/bashrc fi +alias vi=vim + + # Uncomment the following line if you don't like systemctl's auto-paging feature: # export SYSTEMD_PAGER= @@ -17,13 +20,12 @@ fi # https://github.com/nickstenning/honcho/issues/51 export PYTHONIOENCODING=utf-8 -export HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py +export HUBS_CONFIG={{ hubs_conf_dir }}/hubs.py export FLASK_APP={{ hubs_code_dir }}/hubs/app.py workon() { [ "$1" == "hubs" ] || ( echo "No such virtualenv."; exit 1 ) - source {{ hubs_venv_dir }}/bin/activate cd {{ hubs_code_dir }} } @@ -38,6 +40,6 @@ hreset() { {% endif %} rm {{ hubs_var_dir }}/cache.db pushd {{ hubs_code_dir }} - {{ hubs_venv_dir }}/bin/python populate.py + python3 populate.py popd } diff --git a/roles/hubs/templates/env b/roles/hubs/templates/env new file mode 100644 index 0000000000..e3c748125f --- /dev/null +++ b/roles/hubs/templates/env @@ -0,0 +1,2 @@ +HUBS_CONFIG={{ hubs_conf_dir }}/hubs.py +LOGGING_CONFIG={{ hubs_conf_dir }}/logging.ini diff --git a/roles/hubs/templates/fedmsg_config b/roles/hubs/templates/fedmsg_config index 1ab54b00a5..26335faa4e 100644 --- a/roles/hubs/templates/fedmsg_config +++ b/roles/hubs/templates/fedmsg_config @@ -16,17 +16,14 @@ config = { }, }, - # Fedmsg hub consumer - 'hubs.consumer.enabled': True, - 'hubs.redis.triage-queue-name': 'fedora-hubs-triage-queue', - - # Use fedmsg-relay to publish messages - 'active': True, - + {% if hubs_fas_username and hubs_fas_password %} # FAS credentials 'fas_credentials': { 'username': '{{ hubs_fas_username }}', 'password': '{{ hubs_fas_password }}', - }, -} + } + {% endif %} + # Use fedmsg-relay to publish messages + 'active': True, +} diff --git a/roles/hubs/templates/honcho-env b/roles/hubs/templates/honcho-env index 352c551383..a9d806fdd7 100644 --- a/roles/hubs/templates/honcho-env +++ b/roles/hubs/templates/honcho-env @@ -1,3 +1,3 @@ FLASK_DEBUG=1 FLASK_APP={{ hubs_code_dir }}/hubs/app.py -HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py +HUBS_CONFIG={{ hubs_conf_dir }}/hubs.py diff --git a/roles/hubs/templates/honcho-procfile b/roles/hubs/templates/honcho-procfile index 937ef33b6f..893a514f66 100644 --- a/roles/hubs/templates/honcho-procfile +++ b/roles/hubs/templates/honcho-procfile @@ -1,7 +1,7 @@ -web: {{ hubs_venv_dir }}/bin/python /usr/bin/flask-3 run --host 0.0.0.0 --port 5000 -triage: {{ hubs_venv_dir }}/bin/fedora-hubs-triage -worker: {{ hubs_venv_dir }}/bin/fedora-hubs-worker -sse: {{ hubs_venv_dir }}/bin/python /usr/bin/twistd -l - --pidfile= -ny {{ hubs_code_dir }}/hubs/backend/sse_server.tac -fedmsg_hub: {{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-hub -fedmsg_relay: {{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-relay +web: /usr/bin/flask-3 run --host 0.0.0.0 --port 5000 +triage: fedora-hubs-triage +worker: fedora-hubs-worker +sse: /usr/bin/twistd-3 -l - --pidfile= -n hubs-sse +fedmsg_hub: /usr/bin/fedmsg-hub-3 +fedmsg_relay: /usr/bin/fedmsg-relay-3 js_build: cd {{ hubs_code_dir }}/hubs/static/client && npm run dev diff --git a/roles/hubs/templates/hubs-fedmsg-hub.service b/roles/hubs/templates/hubs-fedmsg-hub.service deleted file mode 100644 index ca56996ca5..0000000000 --- a/roles/hubs/templates/hubs-fedmsg-hub.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Hubs-specific fedmsg processing hub -After=network.target -Documentation=https://fedmsg.readthedocs.org/ - -[Service] -ExecStart={{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-hub -Type=simple -User=fedmsg -Group=fedmsg -Restart=on-failure - -[Install] -WantedBy=multi-user.target diff --git a/roles/hubs/templates/hubs-sse.service b/roles/hubs/templates/hubs-sse.service deleted file mode 100644 index 5ff68a2522..0000000000 --- a/roles/hubs/templates/hubs-sse.service +++ /dev/null @@ -1,18 +0,0 @@ -[Unit] -Description=fedora-hubs SSE server -After=network.target -Documentation=https://pagure.io/fedora-hubs/ - -[Service] -ExecStart= \ - {{ hubs_venv_dir }}/bin/python \ - /usr/bin/twistd -l - --pidfile= \ - -ny {{ hubs_code_dir }}/hubs/backend/sse_server.tac -Environment=HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py -Type=simple -User={{ main_user }} -Group={{ main_user }} -Restart=on-failure - -[Install] -WantedBy=multi-user.target diff --git a/roles/hubs/templates/hubs-triage@.service b/roles/hubs/templates/hubs-triage@.service deleted file mode 100644 index 06ccacc05d..0000000000 --- a/roles/hubs/templates/hubs-triage@.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=fedora-hubs triage worker #%i -After=network.target -Documentation=https://pagure.io/fedora-hubs/ - -[Service] -ExecStart={{ hubs_venv_dir }}/bin/fedora-hubs-triage -Environment=HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py -Type=simple -User={{ main_user }} -Group={{ main_user }} -Restart=on-failure - -[Install] -WantedBy=multi-user.target diff --git a/roles/hubs/templates/hubs-webapp.service b/roles/hubs/templates/hubs-webapp.service deleted file mode 100644 index ae6d046fcd..0000000000 --- a/roles/hubs/templates/hubs-webapp.service +++ /dev/null @@ -1,20 +0,0 @@ -[Unit] -Description=fedora-hubs frontend webapp -After=network.target -Documentation=https://pagure.io/fedora-hubs/ - -[Service] -ExecStart= \ - {{ hubs_venv_dir }}/bin/python \ - /usr/bin/python3-gunicorn -b 127.0.0.1:8000 --threads 12 \ - --log-config {{ hubs_conf_dir }}/logging.ini \ - {% if hubs_dev_mode %}--reload{% endif %} \ - hubs.app:app -Environment=HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py -Type=simple -User={{ main_user }} -Group={{ main_user }} -Restart=on-failure - -[Install] -WantedBy=multi-user.target diff --git a/roles/hubs/templates/hubs-worker@.service b/roles/hubs/templates/hubs-worker@.service deleted file mode 100644 index 8f597f15b7..0000000000 --- a/roles/hubs/templates/hubs-worker@.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=fedora-hubs cache worker #%i -After=network.target -Documentation=https://pagure.io/fedora-hubs/ - -[Service] -ExecStart={{ hubs_venv_dir }}/bin/fedora-hubs-worker -Environment=HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py -Type=simple -User={{ main_user }} -Group={{ main_user }} -Restart=on-failure - -[Install] -WantedBy=multi-user.target diff --git a/roles/hubs/templates/nginx.conf b/roles/hubs/templates/nginx.conf index aa34310d32..853ba747e0 100644 --- a/roles/hubs/templates/nginx.conf +++ b/roles/hubs/templates/nginx.conf @@ -34,14 +34,16 @@ server { include ssl_params; keepalive_timeout 5; - # path for static files - root {{ hubs_code_dir }}/hubs/static; - location / { # checks for static file, if not found proxy to app try_files $uri @proxy_to_app; } + # path for static files + location /static { + alias /usr/lib/python3.6/site-packages/hubs/static; + } + location /sse/ { include proxy_params; proxy_pass http://hubs-sse/; From 854892a33ba148088426694788d47e2bb28fdda6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Thu, 1 Feb 2018 09:39:50 +0000 Subject: [PATCH 080/242] Hubs: fix playbook after service rename --- playbooks/hosts/hubs-dev.fedorainfracloud.org.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml b/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml index ed2ec5827b..0c0fe030d9 100644 --- a/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml +++ b/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml @@ -61,7 +61,7 @@ - name: add more hubs workers service: name={{item}} enabled=yes state=started with_items: - - hubs-triage@3 - - hubs-triage@4 - - hubs-worker@3 - - hubs-worker@4 + - fedora-hubs-triage@3 + - fedora-hubs-triage@4 + - fedora-hubs-worker@3 + - fedora-hubs-worker@4 From 303b9bed2879b89475737d0d86a1528e2918254b Mon Sep 17 00:00:00 2001 From: Andrea Veri Date: Thu, 1 Feb 2018 12:18:41 +0000 Subject: [PATCH 081/242] GNOME backups: stop backing extensions.g.o up, rename ghispano to gnome-hispano --- roles/gnome_backups/files/backup.sh | 3 +-- roles/gnome_backups/files/ssh_config | 2 +- roles/gnome_backups/tasks/main.yml | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/roles/gnome_backups/files/backup.sh b/roles/gnome_backups/files/backup.sh index 9433a5cd11..6f91fb2a38 100644 --- a/roles/gnome_backups/files/backup.sh +++ b/roles/gnome_backups/files/backup.sh @@ -19,7 +19,6 @@ MACHINES='signal.gnome.org restaurant.gnome.org expander.gnome.org live.gnome.org - extensions.gnome.org view.gnome.org puppet.gnome.org accelerator.gnome.org @@ -29,7 +28,7 @@ MACHINES='signal.gnome.org bugzilla-new.gnome.org socket.gnome.org odrs.gnome.org - ghispano.gnome.org + gnome-hispano.gnome.org scale.gnome.org sdkbuilder.gnome.org webapps3.gnome.org diff --git a/roles/gnome_backups/files/ssh_config b/roles/gnome_backups/files/ssh_config index c75e148077..74684aa5ad 100644 --- a/roles/gnome_backups/files/ssh_config +++ b/roles/gnome_backups/files/ssh_config @@ -1,4 +1,4 @@ -Host live.gnome.org extensions.gnome.org puppet.gnome.org cloud.gnome.org webapps3.gnome.org +Host live.gnome.org puppet.gnome.org cloud.gnome.org webapps3.gnome.org User root IdentityFile /usr/local/etc/gnome_backup_id.rsa ProxyCommand ssh -W %h:%p bastion.gnome.org -F /usr/local/etc/gnome_ssh_config diff --git a/roles/gnome_backups/tasks/main.yml b/roles/gnome_backups/tasks/main.yml index 4f9020ad33..0b234d3723 100644 --- a/roles/gnome_backups/tasks/main.yml +++ b/roles/gnome_backups/tasks/main.yml @@ -31,7 +31,6 @@ - blogs.gnome.org - view.gnome.org - puppet.gnome.org - - extensions.gnome.org - palette.gnome.org - git.gnome.org - webapps.gnome.org @@ -51,7 +50,7 @@ - account.gnome.org - bugzilla-new.gnome.org - odrs.gnome.org - - ghispano.gnome.org + - gnome-hispano.gnome.org - scale.gnome.org - sdkbuilder.gnome.org - webapps3.gnome.org From ad4747d69ed5e9c396e79b78508367277ff58c85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Zatloukal?= Date: Thu, 1 Feb 2018 13:35:59 +0100 Subject: [PATCH 082/242] Taskotron: Ansiblized Tasks merged to develop --- inventory/group_vars/taskotron-dev | 2 +- inventory/group_vars/taskotron-stg | 2 +- roles/taskotron/taskotron-client/templates/taskotron.yaml.j2 | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/inventory/group_vars/taskotron-dev b/inventory/group_vars/taskotron-dev index 3ea03b9f30..12a2ba5429 100644 --- a/inventory/group_vars/taskotron-dev +++ b/inventory/group_vars/taskotron-dev @@ -31,7 +31,7 @@ grokmirror_repos: - { name: fedoraqa/rpmlint, url: 'https://pagure.io/taskotron/task-rpmlint.git'} - { name: fedoraqa/upgradepath, url: 'https://pagure.io/taskotron/task-upgradepath.git'} - { name: fedoraqa/upstream-atomic, url: 'https://pagure.io/taskotron/task-upstream-atomic.git'} -grokmirror_default_branch: feature/ansiblize +grokmirror_default_branch: develop ############################################################ diff --git a/inventory/group_vars/taskotron-stg b/inventory/group_vars/taskotron-stg index 1bcff19741..b4211f2941 100644 --- a/inventory/group_vars/taskotron-stg +++ b/inventory/group_vars/taskotron-stg @@ -33,7 +33,7 @@ grokmirror_repos: - { name: fedoraqa/rpmlint, url: 'https://pagure.io/taskotron/task-rpmlint.git'} - { name: fedoraqa/upgradepath, url: 'https://pagure.io/taskotron/task-upgradepath.git'} - { name: fedoraqa/upstream-atomic, url: 'https://pagure.io/taskotron/task-upstream-atomic.git'} -grokmirror_default_branch: develop +grokmirror_default_branch: master ############################################################ diff --git a/roles/taskotron/taskotron-client/templates/taskotron.yaml.j2 b/roles/taskotron/taskotron-client/templates/taskotron.yaml.j2 index 97d2833a5e..0d026cad41 100644 --- a/roles/taskotron/taskotron-client/templates/taskotron.yaml.j2 +++ b/roles/taskotron/taskotron-client/templates/taskotron.yaml.j2 @@ -157,7 +157,6 @@ minion_repos: {% endif %} {% if deployment_type == 'dev' %} - https://copr.fedorainfracloud.org/coprs/kparal/taskotron-dev/repo/fedora-26/kparal-taskotron-dev-fedora-26.repo - - https://copr.fedorainfracloud.org/coprs/kparal/taskotron-ansiblize/repo/fedora-27/kparal-taskotron-ansiblize-fedora-27.repo {% endif %} From 4fd1ac5690931e7d3d93e0921f2d026549e0c5a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Zatloukal?= Date: Thu, 1 Feb 2018 14:49:17 +0100 Subject: [PATCH 083/242] Taskotron: Disable task python-versions on dev Ansiblized changes were reverted on develop for now. --- .../taskotron-trigger/templates/trigger_rules.yml.j2 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2 b/roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2 index fa6d5cb93f..f59bb4b917 100644 --- a/roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2 +++ b/roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2 @@ -3,7 +3,6 @@ message_type: KojiBuildPackageCompleted do: - tasks: - - python-versions - rpmgrill - rpmlint @@ -41,6 +40,12 @@ {% if deployment_type in ['stg', 'prod'] %} {# these tasks are not ansiblized yet #} +- when: + message_type: KojiBuildPackageCompleted + do: + - tasks: + - python-versions + - when: message_type: ModuleBuildComplete do: From e70400ae185b4d9596cf287a4823e256a4a842be Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 1 Feb 2018 14:53:05 +0000 Subject: [PATCH 084/242] Deploy fedoradocsredirect to prod Signed-off-by: Patrick Uiterwijk --- roles/mediawiki/tasks/main.yml | 8 +------- roles/mediawiki/templates/LocalSettings.php.fp.j2 | 2 -- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/roles/mediawiki/tasks/main.yml b/roles/mediawiki/tasks/main.yml index 18977b682f..673d872de5 100644 --- a/roles/mediawiki/tasks/main.yml +++ b/roles/mediawiki/tasks/main.yml @@ -49,13 +49,7 @@ - mediawiki-OpenIDConnect - mediawiki-OpenIDConnectAPI - php-rmccue-requests - tags: - - packages - - mediawiki - -- name: Install mediawiki-fedoradocsredirect on staging - package: name=mediawiki-fedoradocsredirect state=present - when: env == "staging" + - mediawiki-fedoradocsredirect tags: - packages - mediawiki diff --git a/roles/mediawiki/templates/LocalSettings.php.fp.j2 b/roles/mediawiki/templates/LocalSettings.php.fp.j2 index 4ca78776e3..c815f6b97c 100644 --- a/roles/mediawiki/templates/LocalSettings.php.fp.j2 +++ b/roles/mediawiki/templates/LocalSettings.php.fp.j2 @@ -328,9 +328,7 @@ require_once "$IP/extensions/fedmsg-emit.php"; require_once "$IP/extensions/HTTP302Found/HTTP302Found.php"; require_once "$IP/extensions/RSS/RSS.php"; require_once "$IP/extensions/BassetSubmitter.php"; -{% if env == "staging" %} require_once "$IP/extensions/FedoraDocsRedirect/FedoraDocsRedirect.php"; -{% endif %} {% if env == "staging" %} $basset_url = 'http://basset01.stg.phx2.fedoraproject.org/basset'; From 7797abb629dfc82bee42647d1615cc3cd6a1647a Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 1 Feb 2018 22:39:35 +0000 Subject: [PATCH 085/242] This will not work here in a role mixed into tasks --- roles/hotness/tasks/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/hotness/tasks/main.yml b/roles/hotness/tasks/main.yml index a81b2c956d..846da3ef1b 100644 --- a/roles/hotness/tasks/main.yml +++ b/roles/hotness/tasks/main.yml @@ -35,8 +35,6 @@ owner=fedmsg group=fedmsg mode=0600 with_items: - hotness.py - handlers: - - import_tasks: "{{ handlers_path }}/restart_services.yml" notify: - restart fedmsg-hub tags: From ea59d00bd77218fdbdd5fd31191cf83bc43513eb Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 1 Feb 2018 22:45:57 +0000 Subject: [PATCH 086/242] Drop some old playbooks and hosts that no longer exist --- master.yml | 1 - playbooks/groups/buildhw.yml | 4 +- playbooks/groups/mirrorlist2.yml | 73 ------------------- playbooks/groups/piwik.yml | 33 --------- playbooks/groups/postgresql-server.yml | 4 +- ...lockerbugs-dev.cloud.fedoraproject.org.yml | 39 ---------- 6 files changed, 4 insertions(+), 150 deletions(-) delete mode 100644 playbooks/groups/mirrorlist2.yml delete mode 100644 playbooks/groups/piwik.yml delete mode 100644 playbooks/hosts/blockerbugs-dev.cloud.fedoraproject.org.yml diff --git a/master.yml b/master.yml index c4217951f6..d7a99aa8df 100644 --- a/master.yml +++ b/master.yml @@ -74,7 +74,6 @@ - import_playbook: /srv/web/infra/ansible/playbooks/groups/maintainer-test.yml - import_playbook: /srv/web/infra/ansible/playbooks/groups/mariadb-server.yml - import_playbook: /srv/web/infra/ansible/playbooks/groups/mdapi.yml -- import_playbook: /srv/web/infra/ansible/playbooks/groups/mirrorlist2.yml - import_playbook: /srv/web/infra/ansible/playbooks/groups/mirrormanager.yml - import_playbook: /srv/web/infra/ansible/playbooks/groups/memcached.yml - import_playbook: /srv/web/infra/ansible/playbooks/groups/modernpaste.yml diff --git a/playbooks/groups/buildhw.yml b/playbooks/groups/buildhw.yml index a5bfe816ea..eb77006429 100644 --- a/playbooks/groups/buildhw.yml +++ b/playbooks/groups/buildhw.yml @@ -1,7 +1,7 @@ -- import_playbook: "/srv/web/infra/ansible/playbooks/include/happy_birthday.yml myhosts=buildhw:buildaarch64:bkernel" +- import_playbook: "/srv/web/infra/ansible/playbooks/include/happy_birthday.yml myhosts=buildhw:bkernel" - name: make koji builder(s) on raw hw - hosts: buildhw:buildaarch64:bkernel + hosts: buildhw:bkernel remote_user: root gather_facts: True diff --git a/playbooks/groups/mirrorlist2.yml b/playbooks/groups/mirrorlist2.yml deleted file mode 100644 index 6ed5a182e8..0000000000 --- a/playbooks/groups/mirrorlist2.yml +++ /dev/null @@ -1,73 +0,0 @@ -# create a new mirrorlist server -# NOTE: should be used with --limit most of the time -# NOTE: make sure there is room/space for this server on the vmhost -# NOTE: most of these vars_path come from group_vars/mirrorlist or from hostvars - -- import_playbook: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=mirrorlist2:mirrorlist2-stg:!mirrorlist-host1plus.fedoraproject.org" - -- name: make the box be real - hosts: mirrorlist2:mirrorlist2-stg - user: root - gather_facts: True - - vars_files: - - /srv/web/infra/ansible/vars/global.yml - - "/srv/private/ansible/vars.yml" - - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml - - pre_tasks: - - name: Install policycoreutils-python - package: name=policycoreutils-python state=present - - - name: Create /srv/web/ for all the goodies. - file: > - dest=/srv/web state=directory - owner=root group=root mode=0755 - tags: - - httpd - - httpd/website - - - name: check the selinux context of webdir - command: matchpathcon /srv/web - register: webdir - check_mode: no - changed_when: "1 != 1" - tags: - - config - - selinux - - httpd - - httpd/website - - - name: /srv/web file contexts - command: semanage fcontext -a -t httpd_sys_content_t "/srv/web(/.*)?" - when: webdir.stdout.find('httpd_sys_content_t') == -1 - tags: - - config - - selinux - - httpd - - httpd/website - - roles: - - base - - rkhunter - - nagios_client - - geoip - - hosts - - fas_client - - collectd/base - - mod_wsgi - - httpd/mod_ssl - - mirrormanager/mirrorlist2 - - sudo - - { role: openvpn/client, - when: env != "staging" } - - tasks: - # this is how you include other task lists - - import_tasks: "{{ tasks_path }}/yumrepos.yml" - - import_tasks: "{{ tasks_path }}/2fa_client.yml" - - import_tasks: "{{ tasks_path }}/motd.yml" - - - handlers: - - import_tasks: "{{ handlers_path }}/restart_services.yml" diff --git a/playbooks/groups/piwik.yml b/playbooks/groups/piwik.yml deleted file mode 100644 index 9b740037d6..0000000000 --- a/playbooks/groups/piwik.yml +++ /dev/null @@ -1,33 +0,0 @@ -# These servers run piwik - -- import_playbook: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=piwik-stg" - -- name: make the box be real - hosts: piwik-stg - user: root - gather_facts: True - - vars_files: - - /srv/web/infra/ansible/vars/global.yml - - "/srv/private/ansible/vars.yml" - - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml - - roles: - - base - - rkhunter - - nagios_client - - hosts - - fas_client - - collectd/base - - apache - - fedmsg/base - - piwik - - sudo - - tasks: - - import_tasks: "{{ tasks_path }}/yumrepos.yml" - - import_tasks: "{{ tasks_path }}/2fa_client.yml" - - import_tasks: "{{ tasks_path }}/motd.yml" - - handlers: - - import_tasks: "{{ handlers_path }}/restart_services.yml" diff --git a/playbooks/groups/postgresql-server.yml b/playbooks/groups/postgresql-server.yml index 78352cc77b..436c04814c 100644 --- a/playbooks/groups/postgresql-server.yml +++ b/playbooks/groups/postgresql-server.yml @@ -2,12 +2,12 @@ # NOTE: should be used with --limit most of the time # NOTE: most of these vars_path come from group_vars/backup_server or from hostvars -- import_playbook: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=db-datanommer01.phx2.fedoraproject.org:db-datanommer02.phx2.fedoraproject.org:db-qa01.qa.fedoraproject.org:db-koji01.phx2.fedoraproject.org:db-fas01.stg.phx2.fedoraproject.org:db-fas01.phx2.fedoraproject.org:db01.phx2.fedoraproject.org:db01.stg.phx2.fedoraproject.org:db-s390-koji01.s390.fedoraproject.org:db-arm-koji01.qa.fedoraproject.org:db-ppc-koji01.ppc.fedoraproject.org:db-qa-stg01.qa.fedoraproject.org:db-qa02.qa.fedoraproject.org:db-koji02.stg.phx2.fedoraproject.org" +- import_playbook: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=db-datanommer02.phx2.fedoraproject.org:db-qa01.qa.fedoraproject.org:db-koji01.phx2.fedoraproject.org:db-fas01.stg.phx2.fedoraproject.org:db-fas01.phx2.fedoraproject.org:db01.phx2.fedoraproject.org:db01.stg.phx2.fedoraproject.org:db-s390-koji01.s390.fedoraproject.org:db-qa-stg01.qa.fedoraproject.org:db-qa02.qa.fedoraproject.org:db-koji02.stg.phx2.fedoraproject.org" # Once the instance exists, configure it. - name: configure postgresql server system - hosts: db-datanommer01.phx2.fedoraproject.org:db-datanommer02.phx2.fedoraproject.org:db-qa01.qa.fedoraproject.org:db-koji01.phx2.fedoraproject.org:db-fas01.stg.phx2.fedoraproject.org:db-fas01.phx2.fedoraproject.org:db01.phx2.fedoraproject.org:db01.stg.phx2.fedoraproject.org:db-s390-koji01.s390.fedoraproject.org:db-arm-koji01.qa.fedoraproject.org:db-ppc-koji01.ppc.fedoraproject.org:db-qa-stg01.qa.fedoraproject.org:db-qa02.qa.fedoraproject.org:db-koji02.stg.phx2.fedoraproject.org + hosts: db-datanommer02.phx2.fedoraproject.org:db-qa01.qa.fedoraproject.org:db-koji01.phx2.fedoraproject.org:db-fas01.stg.phx2.fedoraproject.org:db-fas01.phx2.fedoraproject.org:db01.phx2.fedoraproject.org:db01.stg.phx2.fedoraproject.org:db-s390-koji01.s390.fedoraproject.org:db-qa-stg01.qa.fedoraproject.org:db-qa02.qa.fedoraproject.org:db-koji02.stg.phx2.fedoraproject.org user: root gather_facts: True diff --git a/playbooks/hosts/blockerbugs-dev.cloud.fedoraproject.org.yml b/playbooks/hosts/blockerbugs-dev.cloud.fedoraproject.org.yml deleted file mode 100644 index a5d0b48d41..0000000000 --- a/playbooks/hosts/blockerbugs-dev.cloud.fedoraproject.org.yml +++ /dev/null @@ -1,39 +0,0 @@ -- name: check/create instance - hosts: blockerbugs-dev.cloud.fedoraproject.org - user: root - gather_facts: False - - vars_files: - - /srv/web/infra/ansible/vars/global.yml - - "/srv/private/ansible/vars.yml" - - tasks: - - import_tasks: "{{ tasks_path }}/persistent_cloud.yml" - - import_tasks: "{{ tasks_path }}/growroot_cloud.yml" - - handlers: - - import_tasks: "{{ handlers_path }}/restart_services.yml" - -- name: provision instance - hosts: blockerbugs-dev.cloud.fedoraproject.org - user: root - gather_facts: True - vars: - - tcp_ports: [22, 80, 443] - - udp_ports: [] - - vars_files: - - /srv/web/infra/ansible/vars/global.yml - - "/srv/private/ansible/vars.yml" - - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml - - roles: - - basessh - - tasks: - - import_tasks: "{{ tasks_path }}/cloud_setup_basic.yml" - - name: mount up blockerbugs-dev to /srv/persistent - mount: name=/srv/persistent src='LABEL=blockerbugs-dev' fstype=ext4 state=mounted - - handlers: - - import_tasks: "{{ handlers_path }}/restart_services.yml" From 60bd57160942fa334c045c8d5aafb8c63e1de73a Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 1 Feb 2018 23:07:58 +0000 Subject: [PATCH 087/242] Add sysadmin-packages to packages --- inventory/group_vars/packages | 4 +++- inventory/group_vars/packages-stg | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/inventory/group_vars/packages b/inventory/group_vars/packages index 576d9539b2..2058d81482 100644 --- a/inventory/group_vars/packages +++ b/inventory/group_vars/packages @@ -15,7 +15,9 @@ tcp_ports: [ 80, 443, # Neeed for rsync from log01 for logs. custom_rules: [ '-A INPUT -p tcp -m tcp -s 10.5.126.13 --dport 873 -j ACCEPT', '-A INPUT -p tcp -m tcp -s 192.168.1.59 --dport 873 -j ACCEPT' ] -fas_client_groups: sysadmin-noc,sysadmin-web,sysadmin-veteran +fas_client_groups: sysadmin-noc,sysadmin-web,sysadmin-veteran,sysadmin-packages + +sudoers: "{{ private }}/files/sudo/sysadmin-packages" # These are consumed by a task in roles/fedmsg/base/main.yml fedmsg_certs: diff --git a/inventory/group_vars/packages-stg b/inventory/group_vars/packages-stg index 4f3c2db809..139053ff9a 100644 --- a/inventory/group_vars/packages-stg +++ b/inventory/group_vars/packages-stg @@ -12,7 +12,9 @@ tcp_ports: [ 80, 443, # Neeed for rsync from log01 for logs. custom_rules: [ '-A INPUT -p tcp -m tcp -s 10.5.126.13 --dport 873 -j ACCEPT', '-A INPUT -p tcp -m tcp -s 192.168.1.59 --dport 873 -j ACCEPT' ] -fas_client_groups: sysadmin-noc,sysadmin-web,fi-apprentice,sysadmin-veteran +fas_client_groups: sysadmin-noc,sysadmin-web,fi-apprentice,sysadmin-veteran,sysadmin-packages + +sudoers: "{{ private }}/files/sudo/sysadmin-packages" # These are consumed by a task in roles/fedmsg/base/main.yml fedmsg_certs: From ed66fe9f6db7e4ae19affb8164574699d937de57 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Fri, 2 Feb 2018 09:50:47 +0000 Subject: [PATCH 088/242] Update passwd for staging for new staging cert Signed-off-by: Patrick Uiterwijk --- roles/fedora-web/registry/files/passwd-staging | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/fedora-web/registry/files/passwd-staging b/roles/fedora-web/registry/files/passwd-staging index 90e491f810..fa402e79a6 100644 --- a/roles/fedora-web/registry/files/passwd-staging +++ b/roles/fedora-web/registry/files/passwd-staging @@ -1 +1,2 @@ -/C=US/ST=NM/L=Raleigh/O=Red Hat/OU=Fedora Project/CN=docker-registry-internal-stg:xxj31ZMTZzkVA +/C=US/ST=North Carolina/L=Raleigh/O=Fedora Project/OU=INTERNAL certificates/CN=Fedora STAGING registry push:xxj31ZMTZzkVA +/C=US/ST=North Carolina/L=Raleigh/O=Fedora Project/OU=INTERNAL certificates/CN=CentOS STAGING registry push:xxj31ZMTZzkVA From ad588503720be2677a5c69eff5ab0ef87178a96e Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Fri, 2 Feb 2018 10:08:14 +0000 Subject: [PATCH 089/242] Add CentOS registry info Signed-off-by: Patrick Uiterwijk --- .../reversepassproxy.registry-centos.conf | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 roles/httpd/reverseproxy/templates/reversepassproxy.registry-centos.conf diff --git a/roles/httpd/reverseproxy/templates/reversepassproxy.registry-centos.conf b/roles/httpd/reverseproxy/templates/reversepassproxy.registry-centos.conf new file mode 100644 index 0000000000..eaf39b36c2 --- /dev/null +++ b/roles/httpd/reverseproxy/templates/reversepassproxy.registry-centos.conf @@ -0,0 +1,33 @@ +RewriteEngine on + +RewriteRule ^/v2/latest/(.*) /v2/f27/$1 [R,L] + +{% if env == "staging" %} +RewriteRule ^/v2/(.*) /v2/centos/$1 +{% endif %} + +RewriteRule ^/signatures/(.*) /srv/web/registry-signatures/$1 [L] + + + Require all granted + + +{% include './reversepassproxy.registry-generic.conf' %} + +# Write access to docker-deployer only +{% if env == "staging" %} + + + Require user docker-registry-internal-stg + + + + Require all denied + + +{% else %} + + + require valid-user + +{% endif %} From 87b919f769469d4aa10f14d2c6cb27db1d7f5042 Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Fri, 2 Feb 2018 10:06:34 +0000 Subject: [PATCH 090/242] Auto track rust-sig packages on Koschei --- roles/koschei/backend/files/koschei-track-group | 8 ++++++++ roles/koschei/backend/tasks/main.yml | 1 + roles/koschei/backend/templates/cron-refresh-groups.j2 | 1 + 3 files changed, 10 insertions(+) create mode 100755 roles/koschei/backend/files/koschei-track-group diff --git a/roles/koschei/backend/files/koschei-track-group b/roles/koschei/backend/files/koschei-track-group new file mode 100755 index 0000000000..3961b662ab --- /dev/null +++ b/roles/koschei/backend/files/koschei-track-group @@ -0,0 +1,8 @@ +#!/bin/sh +# Mark all packages in global group as tracked +# Usage: koschei-track-group + +set -e +test -n "$1" + +exec koschei-admin psql <<<"UPDATE package SET tracked = TRUE WHERE id IN (SELECT p.id FROM package p JOIN package_group_relation pgr ON p.base_id = pgr.base_id JOIN package_group g ON g.id = pgr.group_id WHERE NOT p.tracked AND g.name = '$1' AND g.namespace IS NULL)" diff --git a/roles/koschei/backend/tasks/main.yml b/roles/koschei/backend/tasks/main.yml index ce9b44744d..e49693f91d 100644 --- a/roles/koschei/backend/tasks/main.yml +++ b/roles/koschei/backend/tasks/main.yml @@ -122,6 +122,7 @@ - koschei-refresh-group - koschei-refresh-distgit-group - koschei-refresh-module + - koschei-track-group tags: - koschei - config diff --git a/roles/koschei/backend/templates/cron-refresh-groups.j2 b/roles/koschei/backend/templates/cron-refresh-groups.j2 index 7a607adb01..ff931f32b8 100644 --- a/roles/koschei/backend/templates/cron-refresh-groups.j2 +++ b/roles/koschei/backend/templates/cron-refresh-groups.j2 @@ -4,6 +4,7 @@ MAILTO=sysadmin-koschei-members@fedoraproject.org {% if env != 'stg' %}{# rust packages are not synced on stg yet #} 5 0-23/3 * * * koschei /usr/local/bin/koschei-refresh-distgit-group rust-sig rust-sig +15 0-23/3 * * * koschei /usr/local/bin/koschei-track-group rust-sig {%- endif %} # I'd use dnf clean, but it leaves stuff behind From d1276cf303aa0a23c93953bb88e856c0ce905884 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Fri, 2 Feb 2018 10:20:39 +0000 Subject: [PATCH 091/242] Add registry tag Signed-off-by: Patrick Uiterwijk --- playbooks/include/proxies-reverseproxy.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/playbooks/include/proxies-reverseproxy.yml b/playbooks/include/proxies-reverseproxy.yml index 187213d2c9..506889dfab 100644 --- a/playbooks/include/proxies-reverseproxy.yml +++ b/playbooks/include/proxies-reverseproxy.yml @@ -549,6 +549,8 @@ # proxyurl in this one is totally ignored, because Docker. # (turns out it uses PATCH requests that Varnish cannot deal with) proxyurl: "{{ varnish_url }}" + tags: + - registry - role: httpd/reverseproxy website: registry.centos.org @@ -556,6 +558,8 @@ # proxyurl in this one is totally ignored, because Docker. # (turns out it uses PATCH requests that Varnish cannot deal with) proxyurl: "{{ varnish_url }}" + tags: + - registry - role: httpd/reverseproxy website: candidate-registry.fedoraproject.org From ed5965b5053f7144eb53f656ad67ebd3bf70e127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kalu=C5=BEa?= Date: Fri, 2 Feb 2018 12:54:35 +0000 Subject: [PATCH 092/242] Default freshmaker_stg_oidc_client_id and freshmaker_stg_oidc_client_secret in defaults yaml to test role on staging --- roles/freshmaker/frontend/defaults/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/freshmaker/frontend/defaults/main.yml b/roles/freshmaker/frontend/defaults/main.yml index 2513ba4fc6..7b7a0eac08 100644 --- a/roles/freshmaker/frontend/defaults/main.yml +++ b/roles/freshmaker/frontend/defaults/main.yml @@ -6,3 +6,6 @@ freshmaker_endpoint: '' freshmaker_allowed_named_hosts: [] freshmaker_allowed_hosts: [] freshmaker_servername: localhost +freshmaker_stg_oidc_client_id: '' +freshmaker_stg_oidc_client_secret: '' + From 5caf19ea91c3c6d2ad482d97012f19f7c588bcf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kalu=C5=BEa?= Date: Fri, 2 Feb 2018 13:02:02 +0000 Subject: [PATCH 093/242] Default freshmaker_stg_oidc_client_id and freshmaker_stg_oidc_client_secret in defaults yaml to test role on staging --- roles/freshmaker/frontend/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/freshmaker/frontend/defaults/main.yml b/roles/freshmaker/frontend/defaults/main.yml index 7b7a0eac08..fde213d7fe 100644 --- a/roles/freshmaker/frontend/defaults/main.yml +++ b/roles/freshmaker/frontend/defaults/main.yml @@ -6,6 +6,6 @@ freshmaker_endpoint: '' freshmaker_allowed_named_hosts: [] freshmaker_allowed_hosts: [] freshmaker_servername: localhost -freshmaker_stg_oidc_client_id: '' -freshmaker_stg_oidc_client_secret: '' +freshmaker_stg_oidc_client_id: 'unset' +freshmaker_stg_oidc_client_secret: 'unset' From 1b0742d15552f8c09e3c96577eb55f339552b1be Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 2 Feb 2018 14:57:11 +0000 Subject: [PATCH 094/242] Add freshmaker to haproxy. --- roles/haproxy/templates/haproxy.cfg | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/roles/haproxy/templates/haproxy.cfg b/roles/haproxy/templates/haproxy.cfg index 762e3d9d71..4420927f87 100644 --- a/roles/haproxy/templates/haproxy.cfg +++ b/roles/haproxy/templates/haproxy.cfg @@ -642,6 +642,15 @@ backend odcs-backend server odcs-frontend01 odcs-frontend01:80 check inter 20s rise 2 fall 3 option httpchk GET /api/1/composes/ +frontend freshmaker-frontend + bind 0.0.0.0:10067 + default_backend freshmaker-backend + +backend freshmaker-backend + balance hdr(appserver) + server freshmaker-frontend01 freshmaker-frontend01:80 check inter 20s rise 2 fall 3 + option httpchk GET /api/1/builds/ + # Apache doesn't handle the initial connection here like the other proxy # entries. This proxy also doesn't use the http mode like the others. # stunnel should be sitting on port 9939 (public) and redirecting From 005cb81bd41754b8a1c24d44db1a61ba791a747e Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 2 Feb 2018 14:58:50 +0000 Subject: [PATCH 095/242] Add reverseproxy entry for freshmaker. --- playbooks/include/proxies-reverseproxy.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/playbooks/include/proxies-reverseproxy.yml b/playbooks/include/proxies-reverseproxy.yml index 506889dfab..a253e3851b 100644 --- a/playbooks/include/proxies-reverseproxy.yml +++ b/playbooks/include/proxies-reverseproxy.yml @@ -640,6 +640,13 @@ tags: - odcs + - role: httpd/reverseproxy + website: freshmaker.fedoraproject.org + destname: freshmaker + proxyurl: http://localhost:10067 + tags: + - freshmaker + - role: httpd/reverseproxy website: data-analysis.fedoraproject.org destname: awstats From 1d00065c41bec2a7fe4a32c1b4ed0e07fc7678dc Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 2 Feb 2018 14:59:20 +0000 Subject: [PATCH 096/242] Add httpd/website entry for freshmaker. --- playbooks/include/proxies-websites.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/playbooks/include/proxies-websites.yml b/playbooks/include/proxies-websites.yml index 7e062316b5..eb589714f1 100644 --- a/playbooks/include/proxies-websites.yml +++ b/playbooks/include/proxies-websites.yml @@ -791,6 +791,12 @@ server_aliases: [odcs.stg.fedoraproject.org] cert_name: "{{wildcard_cert_name}}" + - role: httpd/website + name: freshmaker.fedoraproject.org + sslonly: true + server_aliases: [freshmaker.stg.fedoraproject.org] + cert_name: "{{wildcard_cert_name}}" + # fedorahosted is retired. We have the site here so we can redirect it. - role: httpd/website From 42f2d39f922fde1e2148a9b02881d38c40959314 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 2 Feb 2018 15:01:57 +0000 Subject: [PATCH 097/242] Tag these roles, for convenience. --- playbooks/include/proxies-websites.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/playbooks/include/proxies-websites.yml b/playbooks/include/proxies-websites.yml index eb589714f1..2cf47850df 100644 --- a/playbooks/include/proxies-websites.yml +++ b/playbooks/include/proxies-websites.yml @@ -790,12 +790,14 @@ sslonly: true server_aliases: [odcs.stg.fedoraproject.org] cert_name: "{{wildcard_cert_name}}" + tags: odcs - role: httpd/website name: freshmaker.fedoraproject.org sslonly: true server_aliases: [freshmaker.stg.fedoraproject.org] cert_name: "{{wildcard_cert_name}}" + tags: freshmaker # fedorahosted is retired. We have the site here so we can redirect it. From 0a00c5a82c34861140a3dff3797cce459c8a0cdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Fri, 2 Feb 2018 15:10:12 +0000 Subject: [PATCH 098/242] Hubs: fix syntax in the fedmsg conf file --- roles/hubs/templates/fedmsg_config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/hubs/templates/fedmsg_config b/roles/hubs/templates/fedmsg_config index 26335faa4e..a5562dd8d1 100644 --- a/roles/hubs/templates/fedmsg_config +++ b/roles/hubs/templates/fedmsg_config @@ -21,7 +21,7 @@ config = { 'fas_credentials': { 'username': '{{ hubs_fas_username }}', 'password': '{{ hubs_fas_password }}', - } + }, {% endif %} # Use fedmsg-relay to publish messages From ead99e74419b43a9fe37310a230695c5d8b2f308 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 2 Feb 2018 15:15:26 +0000 Subject: [PATCH 099/242] Freshmaker should rely on tls termination at the proxy layer. --- playbooks/groups/freshmaker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/playbooks/groups/freshmaker.yml b/playbooks/groups/freshmaker.yml index 968eebc5a6..3d2ac7d730 100644 --- a/playbooks/groups/freshmaker.yml +++ b/playbooks/groups/freshmaker.yml @@ -58,7 +58,9 @@ roles: - mod_wsgi - - freshmaker/frontend + - role: freshmaker/frontend + # TLS is terminated for us at the proxy layer (like for every other app). + freshmaker_force_ssl: False handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml" From 0da5d836e2ad87c8b2dbb6036755df548a5a9b53 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 2 Feb 2018 15:29:46 +0000 Subject: [PATCH 100/242] Fix redirect and servername issue in freshmaker role. --- playbooks/groups/freshmaker.yml | 1 + .../templates/etc/httpd/conf.d/freshmaker.conf.j2 | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/playbooks/groups/freshmaker.yml b/playbooks/groups/freshmaker.yml index 3d2ac7d730..a92aa74b9f 100644 --- a/playbooks/groups/freshmaker.yml +++ b/playbooks/groups/freshmaker.yml @@ -61,6 +61,7 @@ - role: freshmaker/frontend # TLS is terminated for us at the proxy layer (like for every other app). freshmaker_force_ssl: False + freshmaker_servername: freshmaker{{env_suffix}}.fedoraproject.org handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml" diff --git a/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 b/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 index 213002c28e..4ae0fca4fb 100644 --- a/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 +++ b/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 @@ -8,13 +8,13 @@ RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI} WSGIDaemonProcess freshmaker user=fedmsg group=fedmsg threads=5 home=/usr/share/freshmaker WSGIScriptAlias /{{ freshmaker_endpoint }} /usr/share/freshmaker/freshmaker.wsgi -{% if freshmaker_servername != inventory_hostname %} -# Redirect from the hostname of this machine to user-visible hostname. -RewriteEngine On - -RewriteRule (.*) "%{REQUEST_SCHEME}://{{ freshmaker_servername }}%{REQUEST_URI}" [R,L] - -{% endif %} +##{% if freshmaker_servername != inventory_hostname %} +### Redirect from the hostname of this machine to user-visible hostname. +##RewriteEngine On +## +##RewriteRule (.*) "%{REQUEST_SCHEME}://{{ freshmaker_servername }}%{REQUEST_URI}" [R,L] +## +##{% endif %} {% if env == 'staging' %} OIDCOAuthClientID {{ freshmaker_stg_oidc_client_id }} From 3b21215d4d939db8251262ddc0389f575ffed45e Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 2 Feb 2018 17:04:05 +0000 Subject: [PATCH 101/242] Add OIDC scope for freshmaker (staging). --- roles/ipsilon/files/oidc_scopes/freshmaker.py | 14 ++++++++++++++ roles/ipsilon/tasks/main.yml | 3 ++- roles/ipsilon/templates/configuration.conf | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 roles/ipsilon/files/oidc_scopes/freshmaker.py diff --git a/roles/ipsilon/files/oidc_scopes/freshmaker.py b/roles/ipsilon/files/oidc_scopes/freshmaker.py new file mode 100644 index 0000000000..beb5d1108a --- /dev/null +++ b/roles/ipsilon/files/oidc_scopes/freshmaker.py @@ -0,0 +1,14 @@ +from __future__ import absolute_import + +from ipsilon.providers.openidc.plugins.common import OpenidCExtensionBase + + +class OpenidCExtension(OpenidCExtensionBase): + name = 'freshmaker' + display_name = 'Freshmaker Rebuilds' + scopes = { + 'https://pagure.io/freshmaker/manual-trigger': { + 'display_name': 'Permission to submit manual triggers of rebuilds', + 'claims': [], + }, + } diff --git a/roles/ipsilon/tasks/main.yml b/roles/ipsilon/tasks/main.yml index eefec73036..ede0c4205d 100644 --- a/roles/ipsilon/tasks/main.yml +++ b/roles/ipsilon/tasks/main.yml @@ -59,7 +59,8 @@ dest=/usr/lib/python2.7/site-packages/ipsilon/providers/openidc/plugins/{{item}}.py owner=root group=root mode=0644 when: env == "staging" - with_items: [] + with_items: + - freshmaker notify: - reload apache tags: diff --git a/roles/ipsilon/templates/configuration.conf b/roles/ipsilon/templates/configuration.conf index 3cc50dbc29..0ad1bb7bbd 100644 --- a/roles/ipsilon/templates/configuration.conf +++ b/roles/ipsilon/templates/configuration.conf @@ -25,7 +25,7 @@ global enabled=persona,openid,saml2,openidc {% if env == "production" %} openidc enabled extensions=fedora-account,mbs,beaker,waiverdb,odcs,wiki {% else %} -openidc enabled extensions=fedora-account,mbs,beaker,waiverdb,odcs,wiki +openidc enabled extensions=fedora-account,mbs,beaker,waiverdb,odcs,wiki,freshmaker {% endif %} {% if env == 'staging' %} From ef89fb2cbb7798a548f1a0f682d74fb8c7ac9278 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 2 Feb 2018 17:08:21 +0000 Subject: [PATCH 102/242] Use the correct scope name. --- roles/ipsilon/files/oidc_scopes/freshmaker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/ipsilon/files/oidc_scopes/freshmaker.py b/roles/ipsilon/files/oidc_scopes/freshmaker.py index beb5d1108a..ac1b44fef7 100644 --- a/roles/ipsilon/files/oidc_scopes/freshmaker.py +++ b/roles/ipsilon/files/oidc_scopes/freshmaker.py @@ -7,7 +7,7 @@ class OpenidCExtension(OpenidCExtensionBase): name = 'freshmaker' display_name = 'Freshmaker Rebuilds' scopes = { - 'https://pagure.io/freshmaker/manual-trigger': { + 'https://pagure.io/freshmaker/submit-build': { 'display_name': 'Permission to submit manual triggers of rebuilds', 'claims': [], }, From 193971bcf0c84f7932e140498f60afae28d92e21 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 2 Feb 2018 17:10:21 +0000 Subject: [PATCH 103/242] Enable freshmaker OIDC scopes in prod ipsilon. --- roles/ipsilon/tasks/main.yml | 4 ++-- roles/ipsilon/templates/configuration.conf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/ipsilon/tasks/main.yml b/roles/ipsilon/tasks/main.yml index ede0c4205d..2d24aebda8 100644 --- a/roles/ipsilon/tasks/main.yml +++ b/roles/ipsilon/tasks/main.yml @@ -48,6 +48,7 @@ - waiverdb - odcs - wiki + - freshmaker notify: - reload apache tags: @@ -59,8 +60,7 @@ dest=/usr/lib/python2.7/site-packages/ipsilon/providers/openidc/plugins/{{item}}.py owner=root group=root mode=0644 when: env == "staging" - with_items: - - freshmaker + with_items: [] notify: - reload apache tags: diff --git a/roles/ipsilon/templates/configuration.conf b/roles/ipsilon/templates/configuration.conf index 0ad1bb7bbd..a8cf050236 100644 --- a/roles/ipsilon/templates/configuration.conf +++ b/roles/ipsilon/templates/configuration.conf @@ -23,7 +23,7 @@ global enabled=allow global enabled=persona,openid,saml2,openidc {% if env == "production" %} -openidc enabled extensions=fedora-account,mbs,beaker,waiverdb,odcs,wiki +openidc enabled extensions=fedora-account,mbs,beaker,waiverdb,odcs,wiki,freshmaker {% else %} openidc enabled extensions=fedora-account,mbs,beaker,waiverdb,odcs,wiki,freshmaker {% endif %} From b2655e43b08a77f18450ec28ad43653a9d7e5851 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 2 Feb 2018 17:17:02 +0000 Subject: [PATCH 104/242] Freshmaker config should be owned by the fedmsg user. --- roles/freshmaker/base/tasks/main.yml | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/roles/freshmaker/base/tasks/main.yml b/roles/freshmaker/base/tasks/main.yml index 2922e77483..3b669fbced 100644 --- a/roles/freshmaker/base/tasks/main.yml +++ b/roles/freshmaker/base/tasks/main.yml @@ -25,20 +25,7 @@ tags: - freshmaker -- name: generate Freshmaker app config for frontend - template: - src: etc/freshmaker/config.py.j2 - dest: /etc/freshmaker/config.py - owner: apache - group: apache - mode: 0440 - notify: - - restart apache - when: inventory_hostname.startswith('freshmaker-frontend') - tags: - - freshmaker - -- name: generate Freshmaker app config for backend +- name: generate Freshmaker app config template: src: etc/freshmaker/config.py.j2 dest: /etc/freshmaker/config.py @@ -46,7 +33,7 @@ group: fedmsg mode: 0440 notify: + - restart apache - restart fedmsg-hub - when: inventory_hostname.startswith('freshmaker-backend') tags: - freshmaker From a907fb6d56fb6ce62332c24682abd07fc47290a0 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 2 Feb 2018 17:18:57 +0000 Subject: [PATCH 105/242] Use wsgi_procs and wsgi_threads for freshmaker apache config. --- .../frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 b/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 index 4ae0fca4fb..3885635a9c 100644 --- a/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 +++ b/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 @@ -5,7 +5,7 @@ RewriteCond %{HTTPS} off RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI} {% endif %} -WSGIDaemonProcess freshmaker user=fedmsg group=fedmsg threads=5 home=/usr/share/freshmaker +WSGIDaemonProcess freshmaker user=fedmsg group=fedmsg processes={{wsgi_procs}} threads={{wsgi_threads}} home=/usr/share/freshmaker WSGIScriptAlias /{{ freshmaker_endpoint }} /usr/share/freshmaker/freshmaker.wsgi ##{% if freshmaker_servername != inventory_hostname %} From b93bd375fbe0855b7739ba6e0617a05369c3d4e6 Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Fri, 2 Feb 2018 23:21:46 +0000 Subject: [PATCH 106/242] ok let us try to cut down http access ot the server from whatever backdoor is letting it in --- inventory/group_vars/pkgs | 11 ++++++----- inventory/group_vars/pkgs-stg | 10 ++++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/inventory/group_vars/pkgs b/inventory/group_vars/pkgs index 55434c1f15..8f81c83fea 100644 --- a/inventory/group_vars/pkgs +++ b/inventory/group_vars/pkgs @@ -3,12 +3,13 @@ lvm_size: 100000 mem_size: 4096 num_cpus: 4 -tcp_ports: [80, 443, - # These 16 ports are used by fedmsg. One for each wsgi thread. - 3000, 3001, 3002, 3003, 3004, 3005, 3006, 3007, - 3008, 3009, 3010, 3011, 3012, 3013, 3014, 3015] +tcp_ports: [ 9418 ] -custom_rules: [ '-A INPUT -p tcp -m tcp --dport 9418 -j ACCEPT'] +custom_rules: [ + '-A INPUT -p tcp -m tcp -s 10.0.0.0/8 --dport 80 -j ACCEPT', + '-A INPUT -p tcp -m tcp -s 10.0.0.0/8 --dport 443 -j ACCEPT', + '-A INPUT -p tcp -m tcp -s 192.168.0.0/16 --dport 80 -j ACCEPT', + '-A INPUT -p tcp -m tcp -s 192.168.0.0/16 --dport 443 -j ACCEPT' ] # We have both celery (pagure_worker) and web thread wanting to send out fedmsg's. # To make things easy on the listening side (so avoid contention of binding ports), let's set the pkgs boxes to active fedmsg. diff --git a/inventory/group_vars/pkgs-stg b/inventory/group_vars/pkgs-stg index ca75ead3c4..0172ca45ac 100644 --- a/inventory/group_vars/pkgs-stg +++ b/inventory/group_vars/pkgs-stg @@ -3,10 +3,12 @@ lvm_size: 100000 mem_size: 4096 num_cpus: 4 -tcp_ports: [80, 443, 9418, - # These 16 ports are used by fedmsg. One for each wsgi thread. - 3000, 3001, 3002, 3003, 3004, 3005, 3006, 3007, - 3008, 3009, 3010, 3011, 3012, 3013, 3014, 3015] +tcp_ports: [ 9418 ] +custom_rules: [ + '-A INPUT -p tcp -m tcp -s 10.0.0.0/8 --dport 80 -j ACCEPT', + '-A INPUT -p tcp -m tcp -s 10.0.0.0/8 --dport 443 -j ACCEPT', + '-A INPUT -p tcp -m tcp -s 192.168.0.0/16 --dport 80 -j ACCEPT', + '-A INPUT -p tcp -m tcp -s 192.168.0.0/16 --dport 443 -j ACCEPT' ] # Definining these vars has a number of effects # 1) mod_wsgi is configured to use the vars for its own setup From 5de10318d316b52c1340527a5b72307a1b4f0139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Zatloukal?= Date: Mon, 5 Feb 2018 10:35:47 +0100 Subject: [PATCH 107/242] Taskotron: Enable task python-versions on dev Ansiblized version of python-versions is ready! --- .../taskotron-trigger/templates/trigger_rules.yml.j2 | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2 b/roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2 index f59bb4b917..fa6d5cb93f 100644 --- a/roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2 +++ b/roles/taskotron/taskotron-trigger/templates/trigger_rules.yml.j2 @@ -3,6 +3,7 @@ message_type: KojiBuildPackageCompleted do: - tasks: + - python-versions - rpmgrill - rpmlint @@ -40,12 +41,6 @@ {% if deployment_type in ['stg', 'prod'] %} {# these tasks are not ansiblized yet #} -- when: - message_type: KojiBuildPackageCompleted - do: - - tasks: - - python-versions - - when: message_type: ModuleBuildComplete do: From f220d38189178e5c34336d4aee5342df97e11c57 Mon Sep 17 00:00:00 2001 From: Andrea Veri Date: Mon, 5 Feb 2018 11:06:30 +0000 Subject: [PATCH 108/242] GNOME Backups: s/bugzilla-new/bugzilla/ --- roles/gnome_backups/files/backup.sh | 2 +- roles/gnome_backups/tasks/main.yml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/roles/gnome_backups/files/backup.sh b/roles/gnome_backups/files/backup.sh index 6f91fb2a38..866638056f 100644 --- a/roles/gnome_backups/files/backup.sh +++ b/roles/gnome_backups/files/backup.sh @@ -25,7 +25,7 @@ MACHINES='signal.gnome.org range.gnome.org pentagon.gimp.org account.gnome.org - bugzilla-new.gnome.org + bugzilla.gnome.org socket.gnome.org odrs.gnome.org gnome-hispano.gnome.org diff --git a/roles/gnome_backups/tasks/main.yml b/roles/gnome_backups/tasks/main.yml index 0b234d3723..da97d95137 100644 --- a/roles/gnome_backups/tasks/main.yml +++ b/roles/gnome_backups/tasks/main.yml @@ -35,7 +35,7 @@ - git.gnome.org - webapps.gnome.org - socket.gnome.org - - bugzilla-web.gnome.org + - bugzilla.gnome.org - progress.gnome.org - cloud.gnome.org - bastion.gnome.org @@ -48,7 +48,6 @@ - range.gnome.org - pentagon.gimp.org - account.gnome.org - - bugzilla-new.gnome.org - odrs.gnome.org - gnome-hispano.gnome.org - scale.gnome.org From 321ad82c19b405e66e9ec5ed9c8166bcdafc9b4d Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Mon, 5 Feb 2018 13:17:54 +0000 Subject: [PATCH 109/242] Unblock pkgs and do redirect instead Signed-off-by: Patrick Uiterwijk --- inventory/group_vars/pkgs | 6 ------ inventory/group_vars/pkgs-stg | 6 ------ roles/distgit/templates/lookaside-upload.conf | 5 +++-- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/inventory/group_vars/pkgs b/inventory/group_vars/pkgs index 8f81c83fea..023f8952a2 100644 --- a/inventory/group_vars/pkgs +++ b/inventory/group_vars/pkgs @@ -5,12 +5,6 @@ num_cpus: 4 tcp_ports: [ 9418 ] -custom_rules: [ - '-A INPUT -p tcp -m tcp -s 10.0.0.0/8 --dport 80 -j ACCEPT', - '-A INPUT -p tcp -m tcp -s 10.0.0.0/8 --dport 443 -j ACCEPT', - '-A INPUT -p tcp -m tcp -s 192.168.0.0/16 --dport 80 -j ACCEPT', - '-A INPUT -p tcp -m tcp -s 192.168.0.0/16 --dport 443 -j ACCEPT' ] - # We have both celery (pagure_worker) and web thread wanting to send out fedmsg's. # To make things easy on the listening side (so avoid contention of binding ports), let's set the pkgs boxes to active fedmsg. fedmsg_active: True diff --git a/inventory/group_vars/pkgs-stg b/inventory/group_vars/pkgs-stg index 0172ca45ac..708d8b4406 100644 --- a/inventory/group_vars/pkgs-stg +++ b/inventory/group_vars/pkgs-stg @@ -4,12 +4,6 @@ mem_size: 4096 num_cpus: 4 tcp_ports: [ 9418 ] -custom_rules: [ - '-A INPUT -p tcp -m tcp -s 10.0.0.0/8 --dport 80 -j ACCEPT', - '-A INPUT -p tcp -m tcp -s 10.0.0.0/8 --dport 443 -j ACCEPT', - '-A INPUT -p tcp -m tcp -s 192.168.0.0/16 --dport 80 -j ACCEPT', - '-A INPUT -p tcp -m tcp -s 192.168.0.0/16 --dport 443 -j ACCEPT' ] - # Definining these vars has a number of effects # 1) mod_wsgi is configured to use the vars for its own setup # 2) iptables opens enough ports for all threads for fedmsg diff --git a/roles/distgit/templates/lookaside-upload.conf b/roles/distgit/templates/lookaside-upload.conf index eab5f169f1..ec9afabc35 100644 --- a/roles/distgit/templates/lookaside-upload.conf +++ b/roles/distgit/templates/lookaside-upload.conf @@ -14,8 +14,9 @@ SSLCryptoDevice builtin ServerName pkgs{{ env_suffix }}.fedoraproject.org - #Redirect "/" "https://src{{ env_suffix }}.fedoraproject.org/" - # This is temporary for fixing Kojid because of firewall rules + RewriteCond expr "! -R '192.168.0.0/16'" + RewriteCond expr "! -R '10.0.0.0/8'" + RewriteRule ^(.*)$ https://src.fedoraproject.org/$1 [L,R] Alias /repo/ /srv/cache/lookaside/ RewriteEngine on From d70b3eb3b498804fa7b1c54b35046ec4483c6063 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 5 Feb 2018 19:25:20 +0000 Subject: [PATCH 110/242] move to f27 mirrorlist container, will roll out slowly --- roles/mirrormanager/mirrorlist_proxy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/mirrormanager/mirrorlist_proxy/defaults/main.yml b/roles/mirrormanager/mirrorlist_proxy/defaults/main.yml index b845b9b98d..dc1238dd34 100644 --- a/roles/mirrormanager/mirrorlist_proxy/defaults/main.yml +++ b/roles/mirrormanager/mirrorlist_proxy/defaults/main.yml @@ -1 +1 @@ -mirrorlist_container_image: "candidate-registry.fedoraproject.org/f25/mirrormanager2-mirrorlist:f25-docker-candidate-20170426172654" +mirrorlist_container_image: "candidate-registry.fedoraproject.org/f27/mirrormanager2-mirrorlist:0.8.3-1" From a9fe75e33095067c1dba2144df8967bf1bab0663 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Tue, 6 Feb 2018 01:32:29 +0100 Subject: [PATCH 111/242] Instead of mailing, log to syslog Signed-off-by: Patrick Uiterwijk --- roles/bodhi2/backend/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/bodhi2/backend/tasks/main.yml b/roles/bodhi2/backend/tasks/main.yml index 53a76ad685..2a16d9b498 100644 --- a/roles/bodhi2/backend/tasks/main.yml +++ b/roles/bodhi2/backend/tasks/main.yml @@ -330,7 +330,7 @@ - name: bodhi-expire-overrides cron job. cron: name="bodhi-expire-overrides" hour="*" minute=0 user="apache" - job="/usr/bin/bodhi-expire-overrides /etc/bodhi/production.ini" + job="/usr/bin/bodhi-expire-overrides /etc/bodhi/production.ini 2>&1 | logger -t bodhi-expire-overrides" cron_file=bodhi-expire-overrides-job when: inventory_hostname.startswith('bodhi-backend02') and env == "production" tags: From c5f1b83e49cc49526a3b64c35b92d7ad92dfb5f0 Mon Sep 17 00:00:00 2001 From: Andrea Veri Date: Tue, 6 Feb 2018 10:17:22 +0000 Subject: [PATCH 112/242] GNOME Backups: puppet 2.0 has been decommissioned, new master FQDN is puppetmaster01.gnome.org --- roles/gnome_backups/files/backup.sh | 2 +- roles/gnome_backups/files/ssh_config | 2 +- roles/gnome_backups/tasks/main.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/gnome_backups/files/backup.sh b/roles/gnome_backups/files/backup.sh index 866638056f..a70372272f 100644 --- a/roles/gnome_backups/files/backup.sh +++ b/roles/gnome_backups/files/backup.sh @@ -20,7 +20,7 @@ MACHINES='signal.gnome.org expander.gnome.org live.gnome.org view.gnome.org - puppet.gnome.org + puppetmaster01.gnome.org accelerator.gnome.org range.gnome.org pentagon.gimp.org diff --git a/roles/gnome_backups/files/ssh_config b/roles/gnome_backups/files/ssh_config index 74684aa5ad..5f97b352bb 100644 --- a/roles/gnome_backups/files/ssh_config +++ b/roles/gnome_backups/files/ssh_config @@ -1,4 +1,4 @@ -Host live.gnome.org puppet.gnome.org cloud.gnome.org webapps3.gnome.org +Host live.gnome.org puppetmaster01.gnome.org cloud.gnome.org webapps3.gnome.org User root IdentityFile /usr/local/etc/gnome_backup_id.rsa ProxyCommand ssh -W %h:%p bastion.gnome.org -F /usr/local/etc/gnome_ssh_config diff --git a/roles/gnome_backups/tasks/main.yml b/roles/gnome_backups/tasks/main.yml index da97d95137..9bb121174f 100644 --- a/roles/gnome_backups/tasks/main.yml +++ b/roles/gnome_backups/tasks/main.yml @@ -30,7 +30,7 @@ - clutter.gnome.org - blogs.gnome.org - view.gnome.org - - puppet.gnome.org + - puppetmaster01.gnome.org - palette.gnome.org - git.gnome.org - webapps.gnome.org From 7adc884a8e910e3dcc2667c57913eb825c7063e8 Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Tue, 6 Feb 2018 13:37:05 +0000 Subject: [PATCH 113/242] running backups for prod resultsdb db would be a good thing --- inventory/backups | 1 + 1 file changed, 1 insertion(+) diff --git a/inventory/backups b/inventory/backups index 21d4790e40..b05a334433 100644 --- a/inventory/backups +++ b/inventory/backups @@ -13,6 +13,7 @@ people02.fedoraproject.org pkgs02.phx2.fedoraproject.org log01.phx2.fedoraproject.org db-qa01.qa.fedoraproject.org +db-qa02.qa.fedoraproject.org db-koji01.phx2.fedoraproject.org #copr-be.cloud.fedoraproject.org copr-fe.cloud.fedoraproject.org From 81c59788900359650121787819f745839af22906 Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Tue, 6 Feb 2018 16:42:36 +0000 Subject: [PATCH 114/242] try to make awstats file a bit more resilient --- .../files/run-daily-awstats.sh | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/roles/web-data-analysis/files/run-daily-awstats.sh b/roles/web-data-analysis/files/run-daily-awstats.sh index 8b559d2051..12f16f8fad 100644 --- a/roles/web-data-analysis/files/run-daily-awstats.sh +++ b/roles/web-data-analysis/files/run-daily-awstats.sh @@ -33,6 +33,7 @@ DAY=$(/bin/date -d "-${NUMDAYS} days" +%d) LOGDIR=/mnt/fedora_stats/combined-http/ CONFDIR=/mnt/fedora_stats/awstats/conf +STORDIR=/mnt/fedora_stats/awstats/storage OUTDIR=/var/www/html/awstats-reports TREEDIR=${LOGDIR}/${YEAR}/${MONTH}/${DAY} @@ -43,11 +44,21 @@ HTMLDOC=/usr/bin/htmldoc #SITES="apps.fedoraproject.org codecs.fedoraproject.org communityblog.fedoraproject.org docs.fedoraproject.org download.fedoraproject.org fedoramagazine.org fedoraproject.org geoip.fedoraproject.org get.fedoraproject.org getfedora.org labs.fedoraproject.org mirrors.fedoraproject.org spins.fedoraproject.org start.fedoraproject.org" -SITES="admin.fedoraproject.org apps.fedoraproject.org arm.fedoraproject.org ask.fedoraproject.org badges.fedoraproject.org bodhi.fedoraproject.org boot.fedoraproject.org budget.fedoraproject.org bugz.fedoraproject.org cloud.fedoraproject.org codecs.fedoraproject.org communityblog.fedoraproject.org copr.fedoraproject.org darkserver.fedoraproject.org developer.fedoraproject.org developers.fedoraproject.org dl.fedoraproject.org docs.fedoraproject.org download.fedoraproject.org fas.fedoraproject.org fedora.my fedoracommunity.org fedoramagazine.org fedoraproject.com fedoraproject.org flocktofedora.net flocktofedora.org fonts.fedoraproject.org fpaste.org fudcon.fedoraproject.org geoip.fedoraproject.org get.fedoraproject.org getfedora.org help.fedoraproject.org id.fedoraproject.org it.fedoracommunity.org join.fedoraproject.org k12linux.org kde.fedoraproject.org l10n.fedoraproject.org labs.fedoraproject.org lists.fedorahosted.org lists.fedoraproject.org meetbot-raw.fedoraproject.org meetbot.fedoraproject.org mirrors.fedoraproject.org nightly.fedoraproject.org osbs.fedoraproject.org paste.fedoraproject.org pdc.fedoraproject.org people.fedoraproject.org port389.org qa.fedoraproject.org redirect.fedoraproject.org registry.fedoraproject.org smolts.org spins.fedoraproject.org src.fedoraproject.org start.fedoraproject.org store.fedoraproject.org taskotron.fedoraproject.org translate.fedoraproject.org uk.fedoracommunity.org " +SITES="admin.fedoraproject.org apps.fedoraproject.org arm.fedoraproject.org ask.fedoraproject.org badges.fedoraproject.org bodhi.fedoraproject.org boot.fedoraproject.org budget.fedoraproject.org bugz.fedoraproject.org cloud.fedoraproject.org codecs.fedoraproject.org communityblog.fedoraproject.org copr.fedoraproject.org darkserver.fedoraproject.org developer.fedoraproject.org developers.fedoraproject.org dl.fedoraproject.org docs.fedoraproject.org docs-old.fedoraproject.org download.fedoraproject.org fas.fedoraproject.org fedora.my fedoracommunity.org fedoramagazine.org fedoraproject.com fedoraproject.org flocktofedora.net flocktofedora.org fonts.fedoraproject.org fpaste.org fudcon.fedoraproject.org geoip.fedoraproject.org get.fedoraproject.org getfedora.org help.fedoraproject.org id.fedoraproject.org it.fedoracommunity.org join.fedoraproject.org k12linux.org kde.fedoraproject.org l10n.fedoraproject.org labs.fedoraproject.org lists.fedorahosted.org lists.fedoraproject.org meetbot-raw.fedoraproject.org meetbot.fedoraproject.org mirrors.fedoraproject.org nightly.fedoraproject.org osbs.fedoraproject.org paste.fedoraproject.org pdc.fedoraproject.org people.fedoraproject.org port389.org qa.fedoraproject.org redirect.fedoraproject.org registry.fedoraproject.org smolts.org spins.fedoraproject.org src.fedoraproject.org start.fedoraproject.org store.fedoraproject.org taskotron.fedoraproject.org translate.fedoraproject.org uk.fedoracommunity.org " pushd ${CONFDIR} for SITE in ${SITES}; do - perl /usr/share/awstats/wwwroot/cgi-bin/awstats.pl -config=${CONFDIR}/${SITE} -update -Logfile=${TREEDIR}/${SITE}-access.log - perl /mnt/fedora_stats/awstats/conf/awstats_buildstaticpages.pl -awstatsprog=${AWSTATS} -config=${SITE} -month=all -year=${YEAR} -dir=${OUTDIR}/${YEAR} ; + if [[ -f ${CONFDIR/${SITE} ]]; then + if [[ -d ${STORDIR}/${SITE} ]]; then + mkdir -p ${STORDIR}/${SITE} + fi + if [[ -d ${OUTDIR}/${YEAR} ]]; then + mkdir -p ${OUTDIR}/${YEAR} + fi + perl /usr/share/awstats/wwwroot/cgi-bin/awstats.pl -config=${CONFDIR}/${SITE} -update -Logfile=${TREEDIR}/${SITE}-access.log + perl /mnt/fedora_stats/awstats/conf/awstats_buildstaticpages.pl -awstatsprog=${AWSTATS} -config=${SITE} -month=all -year=${YEAR} -dir=${OUTDIR}/${YEAR} ; + else + echo "Site ${SITE} does not have config file" + fi done popd From ef09bafb79e88bec8466161d4ab1c59658265a8c Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Tue, 6 Feb 2018 18:16:29 +0000 Subject: [PATCH 115/242] oh did you remember the mac address? --- roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org index 8d57ea802b..6006682eba 100644 --- a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org +++ b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org @@ -359,7 +359,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { } host aarch64-c17n1 { - hardware ethernet 14:58:D0:58:C5:F2; + hardware ethernet 14:58:d0:58:e5:32; fixed-address 10.5.129.117; next-server 10.5.126.41; option host-name "aarch64-c17n1"; From 619dd465e46bf9629c9f49597867e5fbb7e12ba3 Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Tue, 6 Feb 2018 18:41:57 +0000 Subject: [PATCH 116/242] and we try to make this work for the 10.5.129 --- .../files/dhcpd.conf.noc01.phx2.fedoraproject.org | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org index 6006682eba..c2579d8e1a 100644 --- a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org +++ b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org @@ -202,6 +202,11 @@ subnet 10.5.129.0 netmask 255.255.255.0 { option routers 10.5.129.254; option log-servers 10.5.126.29; + range 10.5.129.200 10.5.129.209 + next-server 10.5.126.41; + filename "/uefi/grubaa64.efi"; + + host ppc8-01 { hardware ethernet 40:f2:e9:5d:39:43; fixed-address 10.5.129.20; From 4d411ec36dc33810576195133845678019adfd3a Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Tue, 6 Feb 2018 19:30:26 +0000 Subject: [PATCH 117/242] you must have a ; at the end of every line stupid --- roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org index c2579d8e1a..6253bcb375 100644 --- a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org +++ b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org @@ -202,7 +202,7 @@ subnet 10.5.129.0 netmask 255.255.255.0 { option routers 10.5.129.254; option log-servers 10.5.126.29; - range 10.5.129.200 10.5.129.209 + range 10.5.129.200 10.5.129.209; next-server 10.5.126.41; filename "/uefi/grubaa64.efi"; From 1cb897a769d816c37e42971c7afa038b783d0b12 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 6 Feb 2018 23:12:14 +0000 Subject: [PATCH 118/242] Clean up nagios client for old stuff that no longer matters. Add a mailman api check. It gets a 401 now, but at least that tells us it's working. --- roles/nagios_client/tasks/main.yml | 28 +------------------ .../files/nagios/services/mailman.cfg | 7 +++++ roles/nagios_server/files/nrpe/nrpe.cfg | 1 + 3 files changed, 9 insertions(+), 27 deletions(-) create mode 100644 roles/nagios_server/files/nagios/services/mailman.cfg diff --git a/roles/nagios_client/tasks/main.yml b/roles/nagios_client/tasks/main.yml index 9699e09549..f6b603253c 100644 --- a/roles/nagios_client/tasks/main.yml +++ b/roles/nagios_client/tasks/main.yml @@ -17,39 +17,13 @@ tags: - packages - nagios_client - when: ansible_distribution_major_version|int < 22 - -# install pkgs: -- name: install nagios client pkgs - dnf: name={{ item }} state=present - with_items: - - nrpe - - nagios-plugins - - nagios-plugins-disk - - nagios-plugins-file_age - - nagios-plugins-users - - nagios-plugins-procs - - nagios-plugins-swap - - nagios-plugins-load - - nagios-plugins-ping - tags: - - packages - - nagios_client - when: ansible_distribution_major_version|int > 21 - name: install nagios tcp check for mirrorlist proxies package: name=nagios-plugins-tcp state=present tags: - packages - nagios_client - when: ansible_distribution_major_version|int < 22 and 'mirrorlist-proxies' in group_names - -- name: install nagios tcp check for mirrorlist proxies - dnf: name=nagios-plugins-tcp state=present - tags: - - packages - - nagios_client - when: ansible_distribution_major_version|int > 21 and 'mirrorlist-proxies' in group_names + when: 'mailman' in group_names or 'mirrorlist-proxies' in group_names - name: install local nrpe check scripts that are not packaged copy: src="scripts/{{ item }}" dest="{{ libdir }}/nagios/plugins/{{ item }}" mode=0755 owner=nagios group=nagios diff --git a/roles/nagios_server/files/nagios/services/mailman.cfg b/roles/nagios_server/files/nagios/services/mailman.cfg new file mode 100644 index 0000000000..a8659cb108 --- /dev/null +++ b/roles/nagios_server/files/nagios/services/mailman.cfg @@ -0,0 +1,7 @@ +define service { + host_name mailman01.phx2.fedoraproject.org + service_description check mailman api + check_command check_mailman_api + max_check_attempts 5 + use defaulttemplate +} diff --git a/roles/nagios_server/files/nrpe/nrpe.cfg b/roles/nagios_server/files/nrpe/nrpe.cfg index dcec90b25e..39235b7771 100644 --- a/roles/nagios_server/files/nrpe/nrpe.cfg +++ b/roles/nagios_server/files/nrpe/nrpe.cfg @@ -344,6 +344,7 @@ command[check_koschei_repo_resolver_proc]=/usr/lib64/nagios/plugins/check_procs command[check_koschei_scheduler_proc]=/usr/lib64/nagios/plugins/check_procs -s RSD -u koschei -C koschei-schedul -c 1:1 command[check_koschei_watcher_proc]=/usr/lib64/nagios/plugins/check_procs -s RSD -u koschei -C koschei-watcher -c 1:1 command[check_mirrorlist_docker_proxy]=/usr/lib64/nagios/plugins/check_tcp -H localhost -p 18081 +command[check_mailman_api]=/usr/lib64/nagios/plugins/check_http -H localhost -p 8001 -u /3.0 command[check_odcs_backend_proc]=/usr/lib64/nagios/plugins/check_procs -c 1:1 -C 'odcs-bakend' -u odcs # The following are fedmsg/datanommer checks to be run on busgateway01. From 3cea53b5b5000208a14f9a181c388b9a71ca4847 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 6 Feb 2018 23:16:37 +0000 Subject: [PATCH 119/242] perhaps this needs quoted --- roles/nagios_client/tasks/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/nagios_client/tasks/main.yml b/roles/nagios_client/tasks/main.yml index f6b603253c..91e1f27fc1 100644 --- a/roles/nagios_client/tasks/main.yml +++ b/roles/nagios_client/tasks/main.yml @@ -23,7 +23,7 @@ tags: - packages - nagios_client - when: 'mailman' in group_names or 'mirrorlist-proxies' in group_names + when: "'mailman' in group_names or 'mirrorlist-proxies' in group_names" - name: install local nrpe check scripts that are not packaged copy: src="scripts/{{ item }}" dest="{{ libdir }}/nagios/plugins/{{ item }}" mode=0755 owner=nagios group=nagios @@ -98,7 +98,6 @@ - nagios_client - selinux - # Set up our base config. - name: /etc/nagios/nrpe.cfg template: src=nrpe.cfg.j2 dest=/etc/nagios/nrpe.cfg From 4a6402394a859a4a70751f17b7ba3f6f5b20213b Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Tue, 6 Feb 2018 23:17:07 +0000 Subject: [PATCH 120/242] Make releng-team able to use releng keytab Signed-off-by: Patrick Uiterwijk --- playbooks/groups/releng-compose.yml | 1 + roles/keytab/service/tasks/main.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/playbooks/groups/releng-compose.yml b/playbooks/groups/releng-compose.yml index 8474051fdd..90879c4dc4 100644 --- a/playbooks/groups/releng-compose.yml +++ b/playbooks/groups/releng-compose.yml @@ -36,6 +36,7 @@ - role: keytab/service service: compose host: "koji{{env_suffix}}.fedoraproject.org" + owner_group: releng-team - role: keytab/service service: mash host: "koji{{env_suffix}}.fedoraproject.org" diff --git a/roles/keytab/service/tasks/main.yml b/roles/keytab/service/tasks/main.yml index a98c89604f..6ccc61e703 100644 --- a/roles/keytab/service/tasks/main.yml +++ b/roles/keytab/service/tasks/main.yml @@ -149,7 +149,7 @@ - krb5 - name: Set keytab permissions - file: path={{kt_location}} owner={{owner_user}} group={{owner_group}} mode=0600 state=file + file: path={{kt_location}} owner={{owner_user}} group={{owner_group}} mode=0640 state=file tags: - keytab - config From 68a56482fab89c80eb2e4faba23ead582a699b9f Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 6 Feb 2018 23:47:17 +0000 Subject: [PATCH 121/242] need to add new file to the loop to actually install --- roles/nagios_server/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/nagios_server/tasks/main.yml b/roles/nagios_server/tasks/main.yml index d1779248b4..d0f8e90824 100644 --- a/roles/nagios_server/tasks/main.yml +++ b/roles/nagios_server/tasks/main.yml @@ -137,6 +137,7 @@ - httpd.cfg - koji.cfg - local.cfg + - mailman.cfg - misc.cfg - notify.cfg - nrpe.cfg From 407efad1f787f7fbe67d069fee404d37d47d4bfd Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 6 Feb 2018 23:55:01 +0000 Subject: [PATCH 122/242] and put it in the right place --- roles/nagios_server/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/nagios_server/tasks/main.yml b/roles/nagios_server/tasks/main.yml index d0f8e90824..6fb8f87722 100644 --- a/roles/nagios_server/tasks/main.yml +++ b/roles/nagios_server/tasks/main.yml @@ -137,7 +137,6 @@ - httpd.cfg - koji.cfg - local.cfg - - mailman.cfg - misc.cfg - notify.cfg - nrpe.cfg @@ -198,6 +197,7 @@ - koschei.cfg - locking.cfg - mail_queue.cfg + - mailman.cfg - memcached.cfg - nagios.cfg - nrpe.cfg From dec23cd1bc8b4a826e8669cf602f680872d49095 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 7 Feb 2018 00:31:15 +0000 Subject: [PATCH 123/242] try this --- .../files/nagios/commands/mailman.cfg | 29 +++++++++++++++++++ roles/nagios_server/tasks/main.yml | 1 + 2 files changed, 30 insertions(+) create mode 100644 roles/nagios_server/files/nagios/commands/mailman.cfg diff --git a/roles/nagios_server/files/nagios/commands/mailman.cfg b/roles/nagios_server/files/nagios/commands/mailman.cfg new file mode 100644 index 0000000000..8d31c1b601 --- /dev/null +++ b/roles/nagios_server/files/nagios/commands/mailman.cfg @@ -0,0 +1,29 @@ +################################################################################ +# COMMAND DEFINITIONS +# +# SYNTAX: +# +# define command{ +# template +# name +# command_name +# command_line +# } +# +# WHERE: +# +# = object name of another command definition that should be +# used as a template for this definition (optional) +# = object name of command definition, referenced by other +# command definitions that use it as a template (optional) +# = name of the command, as recognized/used by Nagios +# = command line +# +################################################################################ + +# 'check_mailman_api' +define command{ + command_name check_mailman_api + command_line $USER1$/check_http -H localhost -p 8001 -u /3.0 +} + diff --git a/roles/nagios_server/tasks/main.yml b/roles/nagios_server/tasks/main.yml index 6fb8f87722..e357d628dc 100644 --- a/roles/nagios_server/tasks/main.yml +++ b/roles/nagios_server/tasks/main.yml @@ -137,6 +137,7 @@ - httpd.cfg - koji.cfg - local.cfg + - mailman.cfg - misc.cfg - notify.cfg - nrpe.cfg From 326c39d1b0157abc25806ab664da79f331d89383 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 7 Feb 2018 01:06:04 +0000 Subject: [PATCH 124/242] perhaps this? --- roles/nagios_server/files/nagios/services/mailman.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/nagios_server/files/nagios/services/mailman.cfg b/roles/nagios_server/files/nagios/services/mailman.cfg index a8659cb108..27dc5768b1 100644 --- a/roles/nagios_server/files/nagios/services/mailman.cfg +++ b/roles/nagios_server/files/nagios/services/mailman.cfg @@ -1,7 +1,7 @@ define service { host_name mailman01.phx2.fedoraproject.org service_description check mailman api - check_command check_mailman_api + check_command check_by_nrpe!check_mailman_api max_check_attempts 5 use defaulttemplate } From 8a0c1f266f5672619556fe9cd746f17f4ced6d4b Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 7 Feb 2018 01:24:52 +0000 Subject: [PATCH 125/242] and setup check on mailman01 nrpe side --- roles/nagios_client/tasks/main.yml | 10 ++++++++++ roles/nagios_client/templates/check_mailman_api.cfg.j2 | 1 + 2 files changed, 11 insertions(+) create mode 100644 roles/nagios_client/templates/check_mailman_api.cfg.j2 diff --git a/roles/nagios_client/tasks/main.yml b/roles/nagios_client/tasks/main.yml index 91e1f27fc1..ed0c119999 100644 --- a/roles/nagios_client/tasks/main.yml +++ b/roles/nagios_client/tasks/main.yml @@ -211,6 +211,16 @@ tags: - nagios_client +- name: install nrpe checks for mailman01 + template: src={{ item }}.j2 dest=/etc/nrpe.d/{{ item }} + with_items: + - check_mailman_api.cfg + when: inventory_hostname.startswith('mailman01') + notify: + - restart nrpe + tags: + - nagios_client + - name: install nrpe checks for proxies template: src={{ item }}.j2 dest=/etc/nrpe.d/{{ item }} with_items: diff --git a/roles/nagios_client/templates/check_mailman_api.cfg.j2 b/roles/nagios_client/templates/check_mailman_api.cfg.j2 new file mode 100644 index 0000000000..132e014018 --- /dev/null +++ b/roles/nagios_client/templates/check_mailman_api.cfg.j2 @@ -0,0 +1 @@ +command[check_mailman_api]=/usr/lib64/nagios/plugins/check_http -H localhost -p 8001 -u /3.0 From c8e5316fb7d552076664188afb6524a99c47a170 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 7 Feb 2018 01:32:17 +0000 Subject: [PATCH 126/242] adjust more --- roles/nagios_client/templates/check_mailman_api.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/nagios_client/templates/check_mailman_api.cfg.j2 b/roles/nagios_client/templates/check_mailman_api.cfg.j2 index 132e014018..95213f335f 100644 --- a/roles/nagios_client/templates/check_mailman_api.cfg.j2 +++ b/roles/nagios_client/templates/check_mailman_api.cfg.j2 @@ -1 +1 @@ -command[check_mailman_api]=/usr/lib64/nagios/plugins/check_http -H localhost -p 8001 -u /3.0 +command[check_mailman_api]=/usr/lib64/nagios/plugins/check_http -H localhost -p 8001 -u /3.0 -e 'HTTP/1.0 401 Unauthorized' From 808daa537707b000b76e2f4433dedf6bddec8e6d Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Wed, 7 Feb 2018 12:14:28 +0800 Subject: [PATCH 127/242] freshmaker: create kerberos keytab on backend nodes --- playbooks/groups/freshmaker.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/playbooks/groups/freshmaker.yml b/playbooks/groups/freshmaker.yml index a92aa74b9f..d9b021be00 100644 --- a/playbooks/groups/freshmaker.yml +++ b/playbooks/groups/freshmaker.yml @@ -80,5 +80,11 @@ - fedmsg/base - freshmaker/backend + - role: keytab/service + service: freshmaker + owner_user: fedmsg + owner_group: fedmsg + host: "freshmaker{{env_suffix}}.fedoraproject.org" + handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml" From 7edc8430dc7d4b816278544be23bc595fa9313fc Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Wed, 7 Feb 2018 12:52:25 +0800 Subject: [PATCH 128/242] freshmaker: fix openidc userinfo uri in template --- roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 b/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 index 8a03a4aaa5..2649a6f1bd 100644 --- a/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 +++ b/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 @@ -186,7 +186,7 @@ class ProdConfiguration(BaseConfiguration): {% else %} SECRET_KEY = "{{ freshmaker_prod_secret_key }}" - AUTH_OPENIDC_USERINFO_URI = "{{ freshmaker_prod_auth_openidc_userinfo_uri }}" + AUTH_OPENIDC_USERINFO_URI = 'https://id.fedoraproject.org/openidc/UserInfo' SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://freshmaker:{{freshmaker_prod_db_password}}@db-freshmaker/freshmaker{{ '?sslmode=require' if freshmaker_force_postgres_ssl else '' }}' From 8f7acb0dde5f4242147292f8cedc360145abb9e4 Mon Sep 17 00:00:00 2001 From: Till Maas Date: Wed, 7 Feb 2018 12:42:36 +0100 Subject: [PATCH 129/242] Increase HSTS max age to one year The HSTS preload list requires this now: https://hstspreload.org/ --- files/fedora-cloud/haproxy.cfg | 28 +++++++++---------- .../templates/0_releasemonitoring.conf | 2 +- .../infrastructure.fedoraproject.org.conf.j2 | 2 +- .../templates/httpd/coprs_ssl.conf.j2 | 4 +-- roles/distgit/pagure/templates/z_pagure.conf | 2 +- .../templates/reversepassproxy.id.conf | 2 +- roles/httpd/website/templates/website.conf | 2 +- .../templates/infinote.fedoraproject.org.conf | 2 +- roles/keyserver/templates/sks.conf | 4 +-- roles/kojipkgs/templates/kojipkgs.conf | 2 +- .../templates/httpd/0_nagios-external.conf.j2 | 2 +- roles/nginx/templates/example_ssl.conf.2 | 4 +-- roles/pagure/frontend/templates/0_pagure.conf | 6 ++-- .../templates/0_pagure.conf | 4 +-- roles/people/templates/people.conf | 2 +- roles/ufmonitor/templates/ufmonitor.conf.j2 | 2 +- 16 files changed, 35 insertions(+), 35 deletions(-) diff --git a/files/fedora-cloud/haproxy.cfg b/files/fedora-cloud/haproxy.cfg index 8548645e9a..ee0938c303 100644 --- a/files/fedora-cloud/haproxy.cfg +++ b/files/fedora-cloud/haproxy.cfg @@ -68,44 +68,44 @@ defaults frontend neutron bind 0.0.0.0:9696 ssl no-sslv3 no-tlsv10 crt /etc/haproxy/fedorainfracloud.org.combined default_backend neutron - # HSTS (15768000 seconds = 6 months) - rspadd Strict-Transport-Security:\ max-age=15768000 + # HSTS (31536000 seconds = 365 days) + rspadd Strict-Transport-Security:\ max-age=31536000 frontend cinder bind 0.0.0.0:8776 ssl no-sslv3 no-tlsv10 crt /etc/haproxy/fedorainfracloud.org.combined default_backend cinder - # HSTS (15768000 seconds = 6 months) - rspadd Strict-Transport-Security:\ max-age=15768000 + # HSTS (31536000 seconds = 365 days) + rspadd Strict-Transport-Security:\ max-age=31536000 frontend swift bind 0.0.0.0:8080 ssl no-sslv3 no-tlsv10 crt /etc/haproxy/fedorainfracloud.org.combined default_backend swift - # HSTS (15768000 seconds = 6 months) - rspadd Strict-Transport-Security:\ max-age=15768000 + # HSTS (31536000 seconds = 365 days) + rspadd Strict-Transport-Security:\ max-age=31536000 frontend nova bind 0.0.0.0:8774 ssl no-sslv3 no-tlsv10 crt /etc/haproxy/fedorainfracloud.org.combined default_backend nova - # HSTS (15768000 seconds = 6 months) - rspadd Strict-Transport-Security:\ max-age=15768000 + # HSTS (31536000 seconds = 365 days) + rspadd Strict-Transport-Security:\ max-age=31536000 frontend ceilometer bind 0.0.0.0:8777 ssl no-sslv3 no-tlsv10 crt /etc/haproxy/fedorainfracloud.org.combined default_backend ceilometer - # HSTS (15768000 seconds = 6 months) - rspadd Strict-Transport-Security:\ max-age=15768000 + # HSTS (31536000 seconds = 365 days) + rspadd Strict-Transport-Security:\ max-age=31536000 frontend ec2 bind 0.0.0.0:8773 ssl no-sslv3 no-tlsv10 crt /etc/haproxy/fedorainfracloud.org.combined default_backend ec2 - # HSTS (15768000 seconds = 6 months) - rspadd Strict-Transport-Security:\ max-age=15768000 + # HSTS (31536000 seconds = 365 days) + rspadd Strict-Transport-Security:\ max-age=31536000 frontend glance bind 0.0.0.0:9292 ssl no-sslv3 no-tlsv10 crt /etc/haproxy/fedorainfracloud.org.combined default_backend glance - # HSTS (15768000 seconds = 6 months) - rspadd Strict-Transport-Security:\ max-age=15768000 + # HSTS (31536000 seconds = 365 days) + rspadd Strict-Transport-Security:\ max-age=31536000 backend neutron server neutron 127.0.0.1:8696 check diff --git a/roles/anitya/frontend/templates/0_releasemonitoring.conf b/roles/anitya/frontend/templates/0_releasemonitoring.conf index e05414777f..047aacf35e 100644 --- a/roles/anitya/frontend/templates/0_releasemonitoring.conf +++ b/roles/anitya/frontend/templates/0_releasemonitoring.conf @@ -9,7 +9,7 @@ SSLEngine on SSLProtocol {{ ssl_protocols }} SSLCipherSuite {{ ssl_ciphers }} - Header always add Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" + Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" SSLCertificateFile /etc/pki/tls/certs/release-monitoring.org.cert SSLCertificateChainFile /etc/pki/tls/certs/release-monitoring.org.intermediate.cert diff --git a/roles/batcave/templates/infrastructure.fedoraproject.org.conf.j2 b/roles/batcave/templates/infrastructure.fedoraproject.org.conf.j2 index 72b5c6b9ec..0f4d2334f7 100644 --- a/roles/batcave/templates/infrastructure.fedoraproject.org.conf.j2 +++ b/roles/batcave/templates/infrastructure.fedoraproject.org.conf.j2 @@ -114,7 +114,7 @@ ScriptAlias /cgi-bin/ "/var/www/cgi-bin/" SSLCertificateKeyFile /etc/pki/tls/private/{{ wildcard_key_file }} SSLCertificateChainFile /etc/pki/tls/certs/{{ wildcard_int_file }} - Header always add Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" + Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" SSLHonorCipherOrder On diff --git a/roles/copr/frontend/templates/httpd/coprs_ssl.conf.j2 b/roles/copr/frontend/templates/httpd/coprs_ssl.conf.j2 index 4b79dc48ad..18643d9c97 100644 --- a/roles/copr/frontend/templates/httpd/coprs_ssl.conf.j2 +++ b/roles/copr/frontend/templates/httpd/coprs_ssl.conf.j2 @@ -4,7 +4,7 @@ # Use secure TLSv1.1 and TLSv1.2 ciphers SSLCipherSuite {{ ssl_ciphers }} SSLHonorCipherOrder on - Header always add Strict-Transport-Security "max-age=15768000; preload" + Header always add Strict-Transport-Security "max-age=31536000; preload" SSLCertificateFile /etc/pki/tls/certs/copr.fedorainfracloud.org.crt SSLCertificateKeyFile /etc/pki/tls/private/copr.fedorainfracloud.org.key @@ -48,7 +48,7 @@ # Use secure TLSv1.1 and TLSv1.2 ciphers SSLCipherSuite {{ ssl_ciphers }} SSLHonorCipherOrder on - Header always add Strict-Transport-Security "max-age=15768000; preload" + Header always add Strict-Transport-Security "max-age=31536000; preload" SSLCertificateFile /etc/pki/tls/certs/copr.fedorainfracloud.org.crt SSLCertificateKeyFile /etc/pki/tls/private/copr.fedorainfracloud.org.key diff --git a/roles/distgit/pagure/templates/z_pagure.conf b/roles/distgit/pagure/templates/z_pagure.conf index 64ae6d4571..4c390130ad 100644 --- a/roles/distgit/pagure/templates/z_pagure.conf +++ b/roles/distgit/pagure/templates/z_pagure.conf @@ -11,7 +11,7 @@ WSGIDaemonProcess pagureproc user=pagure group=packager maximum-requests=1000 di # SSLEngine on # SSLProtocol all -SSLv2 -SSLv3 # # Use secure TLSv1.1 and TLSv1.2 ciphers -# Header always add Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" +# Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" # SSLCertificateFile /etc/pki/tls/certs/pagure.io.cert # SSLCertificateChainFile /etc/pki/tls/certs/pagure.io.intermediate.cert diff --git a/roles/httpd/reverseproxy/templates/reversepassproxy.id.conf b/roles/httpd/reverseproxy/templates/reversepassproxy.id.conf index a319b7baed..115fed5d03 100644 --- a/roles/httpd/reverseproxy/templates/reversepassproxy.id.conf +++ b/roles/httpd/reverseproxy/templates/reversepassproxy.id.conf @@ -5,7 +5,7 @@ RequestHeader set X-Forwarded-Proto https early # Cannot redirect to HTTPS for *.id.fedoraproject.org or set # "includeSubdomains", because relying parties need to be able to access # username.id.fedoraproject.org via plain HTTP -Header always add Strict-Transport-Security "max-age=15768000; preload" +Header always add Strict-Transport-Security "max-age=31536000; preload" RewriteEngine on diff --git a/roles/httpd/website/templates/website.conf b/roles/httpd/website/templates/website.conf index ff5bcae024..1c747fbc6a 100644 --- a/roles/httpd/website/templates/website.conf +++ b/roles/httpd/website/templates/website.conf @@ -55,7 +55,7 @@ SSLCipherSuite {{ ssl_ciphers }} {% if sslonly %} - Header always add Strict-Transport-Security "max-age=15768000; {% if stssubdomains %}includeSubDomains; {% endif %}preload" + Header always add Strict-Transport-Security "max-age=31536000; {% if stssubdomains %}includeSubDomains; {% endif %}preload" {% endif %} Include "conf.d/{{ name }}/*.conf" diff --git a/roles/infinote/templates/infinote.fedoraproject.org.conf b/roles/infinote/templates/infinote.fedoraproject.org.conf index da6f179302..c09cd3016d 100644 --- a/roles/infinote/templates/infinote.fedoraproject.org.conf +++ b/roles/infinote/templates/infinote.fedoraproject.org.conf @@ -75,7 +75,7 @@ ScriptAlias /cgi-bin/ "/var/www/cgi-bin/" SSLCertificateKeyFile /etc/pki/tls/private/infinote.fedoraproject.org.key SSLCertificateChainFile /etc/pki/tls/certs/wildcard-2014.fedoraproject.org.intermediate.cert - Header always add Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" + Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" SSLHonorCipherOrder On diff --git a/roles/keyserver/templates/sks.conf b/roles/keyserver/templates/sks.conf index a64adf5a7a..14ebe838fb 100644 --- a/roles/keyserver/templates/sks.conf +++ b/roles/keyserver/templates/sks.conf @@ -49,14 +49,14 @@ NameVirtualHost *:443 RewriteCond %{HTTPS} off RewriteRule ^/\.well-known/(.*) /srv/web/acme-challenge/.well-known/$1 [L] RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI} [NE] - Header always add Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" + Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" ServerAdmin sysadmin-keys-members@fedoraproject.org ServerName keys.fedoraproject.org ServerAlias keys02.fedoraproject.org - Header always add Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" + Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" SSLEngine on SSLCertificateFile /etc/letsencrypt/live/keys.fedoraproject.org/cert.pem diff --git a/roles/kojipkgs/templates/kojipkgs.conf b/roles/kojipkgs/templates/kojipkgs.conf index 434d9adf8a..57f62bdd2c 100644 --- a/roles/kojipkgs/templates/kojipkgs.conf +++ b/roles/kojipkgs/templates/kojipkgs.conf @@ -129,4 +129,4 @@ RewriteCond %{HTTP:X-Forwarded-For} !10.5.125.71 RewriteRule ".*/.*openh264.*.(x86_64|armv7hl|i686|ppc64|ppc64le|aarch64|s390x).rpm$" "https://fedoraproject.org/wiki/non-distributable-rpms" [R=302,L] # Set HSTS header via HTTP since it cannot be easily set in squid, which terminates HTTPS -Header always add Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" +Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" diff --git a/roles/nagios_server/templates/httpd/0_nagios-external.conf.j2 b/roles/nagios_server/templates/httpd/0_nagios-external.conf.j2 index b868f5a449..c9c3124792 100644 --- a/roles/nagios_server/templates/httpd/0_nagios-external.conf.j2 +++ b/roles/nagios_server/templates/httpd/0_nagios-external.conf.j2 @@ -9,7 +9,7 @@ SSLEngine on SSLProtocol {{ ssl_protocols }} SSLCipherSuite {{ ssl_ciphers }} - Header always add Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" + Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" SSLCertificateFile /etc/pki/tls/certs/noc02.fedoraproject.org.cert SSLCertificateChainFile /etc/pki/tls/certs/noc02.fedoraproject.org.intermediate.cert diff --git a/roles/nginx/templates/example_ssl.conf.2 b/roles/nginx/templates/example_ssl.conf.2 index 42bc897225..e4c3a703d2 100644 --- a/roles/nginx/templates/example_ssl.conf.2 +++ b/roles/nginx/templates/example_ssl.conf.2 @@ -19,8 +19,8 @@ # ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!3DES:!MD5:!PSK'; # ssl_prefer_server_ciphers on; # -# # HSTS (ngx_http_headers_module is required) (15768000 seconds = 6 months) -# add_header Strict-Transport-Security max-age=15768000; +# # HSTS (ngx_http_headers_module is required) (31536000 seconds = 365 days) +# add_header Strict-Transport-Security max-age=31536000; # location / { # root /usr/share/nginx/html; diff --git a/roles/pagure/frontend/templates/0_pagure.conf b/roles/pagure/frontend/templates/0_pagure.conf index 8b3cda870d..b51d77cba2 100644 --- a/roles/pagure/frontend/templates/0_pagure.conf +++ b/roles/pagure/frontend/templates/0_pagure.conf @@ -69,7 +69,7 @@ WSGIDaemonProcess paguredocs user=git group=git maximum-requests=1000 display-na SSLProtocol {{ ssl_protocols }} SSLCipherSuite {{ ssl_ciphers }} # Use secure TLSv1.1 and TLSv1.2 ciphers - Header always add Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" + Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" SSLCertificateFile /etc/pki/tls/certs/pagure.io.cert SSLCertificateChainFile /etc/pki/tls/certs/pagure.io.intermediate.cert @@ -119,7 +119,7 @@ WSGIDaemonProcess paguredocs user=git group=git maximum-requests=1000 display-na SSLProtocol {{ ssl_protocols }} SSLCipherSuite {{ ssl_ciphers }} # Use secure TLSv1.1 and TLSv1.2 ciphers - Header always add Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" + Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" SSLCertificateFile /etc/pki/tls/certs/docs.pagure.org.crt SSLCertificateChainFile /etc/pki/tls/certs/docs.pagure.org.intermediate.crt @@ -145,7 +145,7 @@ WSGIDaemonProcess paguredocs user=git group=git maximum-requests=1000 display-na SSLProtocol {{ ssl_protocols }} SSLCipherSuite {{ ssl_ciphers }} # Use secure TLSv1.1 and TLSv1.2 ciphers - Header always add Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" + Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" SSLCertificateFile /etc/pki/tls/certs/pagure.io.cert SSLCertificateChainFile /etc/pki/tls/certs/pagure.io.intermediate.cert diff --git a/roles/pagure/upstreamfirst-frontend/templates/0_pagure.conf b/roles/pagure/upstreamfirst-frontend/templates/0_pagure.conf index dc1dbefb4b..0d61a08504 100644 --- a/roles/pagure/upstreamfirst-frontend/templates/0_pagure.conf +++ b/roles/pagure/upstreamfirst-frontend/templates/0_pagure.conf @@ -56,7 +56,7 @@ WSGIDaemonProcess paguredocs user=git group=git maximum-requests=1000 display-na SSLProtocol {{ ssl_protocols }} SSLCipherSuite {{ ssl_ciphers }} # Use secure TLSv1.1 and TLSv1.2 ciphers - Header always add Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" + Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" SSLCertificateFile /etc/letsencrypt/live/{{ external_hostname }}/cert.pem SSLCertificateKeyFile /etc/letsencrypt/live/{{ external_hostname }}/privkey.pem @@ -118,7 +118,7 @@ WSGIDaemonProcess paguredocs user=git group=git maximum-requests=1000 display-na SSLProtocol {{ ssl_protocols }} SSLCipherSuite {{ ssl_ciphers }} # Use secure TLSv1.1 and TLSv1.2 ciphers - Header always add Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" + Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" SSLCertificateFile /etc/letsencrypt/live/{{ external_hostname }}/cert.pem diff --git a/roles/people/templates/people.conf b/roles/people/templates/people.conf index 6df7672854..ca57fa3a8c 100644 --- a/roles/people/templates/people.conf +++ b/roles/people/templates/people.conf @@ -34,7 +34,7 @@ NameVirtualHost *:80 SSLCipherSuite {{ ssl_ciphers }} SSLProtocol {{ ssl_protocols }} - Header always add Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" + Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" LogFormat "%V %h %l %u %t \"%r\" %s %b" vcommon # ErrorLog "| /usr/sbin/rotatelogs /var/log/httpd/fedorapeople.org-error.log-%Y%m%d 86400 -l" diff --git a/roles/ufmonitor/templates/ufmonitor.conf.j2 b/roles/ufmonitor/templates/ufmonitor.conf.j2 index 265e5faf8d..3e56041ac8 100644 --- a/roles/ufmonitor/templates/ufmonitor.conf.j2 +++ b/roles/ufmonitor/templates/ufmonitor.conf.j2 @@ -19,7 +19,7 @@ SSLProtocol {{ ssl_protocols }} SSLCipherSuite {{ ssl_ciphers }} # Use secure TLSv1.1 and TLSv1.2 ciphers - Header always add Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" + Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" SSLCertificateFile /etc/letsencrypt/live/{{ external_hostname }}/cert.pem SSLCertificateKeyFile /etc/letsencrypt/live/{{ external_hostname }}/privkey.pem From 138c94c891b4ecf8016617ee022a0d0366fa6fa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Wed, 7 Feb 2018 13:43:38 +0100 Subject: [PATCH 130/242] do not sync f25 on retrace --- inventory/host_vars/retrace01.qa.fedoraproject.org | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inventory/host_vars/retrace01.qa.fedoraproject.org b/inventory/host_vars/retrace01.qa.fedoraproject.org index b9cb203684..526c1e3621 100644 --- a/inventory/host_vars/retrace01.qa.fedoraproject.org +++ b/inventory/host_vars/retrace01.qa.fedoraproject.org @@ -3,8 +3,8 @@ faf_server_name: retrace.fedoraproject.org/faf rs_use_faf_packages: true # we do not have enough storage on stg -rs_internal_fedora_vers: [25, 26, 27, rawhide] -rs_internal_fedora_vers_removed: [24] +rs_internal_fedora_vers: [26, 27, rawhide] +rs_internal_fedora_vers_removed: [24, 25] rs_internal_arch_list: [source, x86_64, i386] nagios_Check_Services: From 2a890f180b6aa44858e5111db444910eb214a3ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Wed, 7 Feb 2018 14:00:43 +0100 Subject: [PATCH 131/242] add retrace02.qa.fedoraproject.org host vars --- .../host_vars/retrace02.qa.fedoraproject.org | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 inventory/host_vars/retrace02.qa.fedoraproject.org diff --git a/inventory/host_vars/retrace02.qa.fedoraproject.org b/inventory/host_vars/retrace02.qa.fedoraproject.org new file mode 100644 index 0000000000..c20bf0226a --- /dev/null +++ b/inventory/host_vars/retrace02.qa.fedoraproject.org @@ -0,0 +1,18 @@ +--- +faf_server_name: retrace.fedoraproject.org/faf +rs_use_faf_packages: true + +# we do not have enough storage on stg +rs_internal_fedora_vers: [rawhide] +#rs_internal_fedora_vers_removed: [24, 25, 26, 27] +rs_internal_arch_list: [source, x86_64, i386] + +nagios_Check_Services: + nrpe: true + sshd: true + named: false + dhcpd: false + httpd: false + swap: false + +faf_repos: [] From a554ac30feda8819aec4a13f140d2a4e501b9e9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Wed, 7 Feb 2018 14:31:53 +0100 Subject: [PATCH 132/242] retrace: update faf role --- roles/abrt/faf/defaults/main.yml | 1 + roles/abrt/faf/meta/.galaxy_install_info | 2 +- roles/abrt/faf/meta/main.yml | 4 ++-- roles/abrt/faf/tasks/celery.yml | 2 +- roles/abrt/faf/tasks/web.yml | 2 +- roles/abrt/faf/templates/etc-faf-faf.conf.j2 | 3 ++- roles/abrt/retrace/meta/.galaxy_install_info | 2 +- roles/abrt/retrace/meta/main.yml | 6 +++--- roles/abrt/retrace/tasks/install.yml | 2 +- 9 files changed, 13 insertions(+), 11 deletions(-) diff --git a/roles/abrt/faf/defaults/main.yml b/roles/abrt/faf/defaults/main.yml index 483de93023..969f3054b7 100644 --- a/roles/abrt/faf/defaults/main.yml +++ b/roles/abrt/faf/defaults/main.yml @@ -30,6 +30,7 @@ faf_migrate_db: true faf_cron_jobs: true faf_admin_mail: root@localhost +faf_from: no-reply@localhost faf_spool_dir: /var/spool/faf diff --git a/roles/abrt/faf/meta/.galaxy_install_info b/roles/abrt/faf/meta/.galaxy_install_info index 07e2295663..6d9514b460 100644 --- a/roles/abrt/faf/meta/.galaxy_install_info +++ b/roles/abrt/faf/meta/.galaxy_install_info @@ -1 +1 @@ -{install_date: 'Tue Jul 4 08:35:09 2017', version: ''} +{install_date: 'Wed Feb 7 13:30:30 2018', version: ''} diff --git a/roles/abrt/faf/meta/main.yml b/roles/abrt/faf/meta/main.yml index 18b616d6d8..865e1230d0 100644 --- a/roles/abrt/faf/meta/main.yml +++ b/roles/abrt/faf/meta/main.yml @@ -12,8 +12,8 @@ galaxy_info: - name: Fedora versions: - 25 - - 24 - - 23 + - 26 + - 27 categories: - web dependencies: [] diff --git a/roles/abrt/faf/tasks/celery.yml b/roles/abrt/faf/tasks/celery.yml index 7c729a5512..c86cf84386 100644 --- a/roles/abrt/faf/tasks/celery.yml +++ b/roles/abrt/faf/tasks/celery.yml @@ -5,7 +5,7 @@ - packages - name: install redis package - yum : name={{ item }} state=present + package: name={{ item }} state=present with_items: - redis - python-redis diff --git a/roles/abrt/faf/tasks/web.yml b/roles/abrt/faf/tasks/web.yml index 5a42615a51..b57497277c 100644 --- a/roles/abrt/faf/tasks/web.yml +++ b/roles/abrt/faf/tasks/web.yml @@ -8,7 +8,7 @@ when: not faf_web_on_root - name: install faf-webui packages - yum : name={{ item }} state=latest + package : name={{ item }} state=latest with_items: "{{ faf_web_packages }}" - import_tasks: celery.yml diff --git a/roles/abrt/faf/templates/etc-faf-faf.conf.j2 b/roles/abrt/faf/templates/etc-faf-faf.conf.j2 index 9dc8cfd8eb..e62012e2f4 100644 --- a/roles/abrt/faf/templates/etc-faf-faf.conf.j2 +++ b/roles/abrt/faf/templates/etc-faf-faf.conf.j2 @@ -20,7 +20,8 @@ Server = {{ smtp_server }} Port = {{ smtp_port }} Username = {{ smtp_username|default("", true) }} Password = {{ smtp_password|default("", true) }} -From = {{ faf_admin_mail }} +From = {{ faf_from }} + [uReport] # The directory that holds 'reports' and 'attachments' subdirectories Directory = {{ faf_spool_dir }} diff --git a/roles/abrt/retrace/meta/.galaxy_install_info b/roles/abrt/retrace/meta/.galaxy_install_info index 4e5f81968e..c754620b93 100644 --- a/roles/abrt/retrace/meta/.galaxy_install_info +++ b/roles/abrt/retrace/meta/.galaxy_install_info @@ -1 +1 @@ -{install_date: 'Tue Jul 4 08:34:40 2017', version: ''} +{install_date: 'Wed Feb 7 13:30:31 2018', version: ''} diff --git a/roles/abrt/retrace/meta/main.yml b/roles/abrt/retrace/meta/main.yml index dc9c449d10..a9ce491470 100644 --- a/roles/abrt/retrace/meta/main.yml +++ b/roles/abrt/retrace/meta/main.yml @@ -10,9 +10,9 @@ galaxy_info: - 7 - name: Fedora versions: - - 21 - - 22 - - 23 + - 26 + - 27 + - 25 categories: - system #dependencies: diff --git a/roles/abrt/retrace/tasks/install.yml b/roles/abrt/retrace/tasks/install.yml index 7acb10f471..44afdc5526 100644 --- a/roles/abrt/retrace/tasks/install.yml +++ b/roles/abrt/retrace/tasks/install.yml @@ -4,4 +4,4 @@ when: rs_force_reinstall - name: install retrace-server package - yum : name=retrace-server state=present + package: name=retrace-server state=present From 3c27e5a2505991268d7c7f03020749eb462c88de Mon Sep 17 00:00:00 2001 From: Andrea Veri Date: Wed, 7 Feb 2018 14:31:49 +0000 Subject: [PATCH 133/242] GNOME Backups: live.g.o has been decomm and replaced by wiki.g.o --- roles/gnome_backups/files/backup.sh | 2 +- roles/gnome_backups/files/ssh_config | 2 +- roles/gnome_backups/tasks/main.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/gnome_backups/files/backup.sh b/roles/gnome_backups/files/backup.sh index a70372272f..4931806d3a 100644 --- a/roles/gnome_backups/files/backup.sh +++ b/roles/gnome_backups/files/backup.sh @@ -18,7 +18,7 @@ MACHINES='signal.gnome.org master.gnome.org restaurant.gnome.org expander.gnome.org - live.gnome.org + wiki.gnome.org view.gnome.org puppetmaster01.gnome.org accelerator.gnome.org diff --git a/roles/gnome_backups/files/ssh_config b/roles/gnome_backups/files/ssh_config index 5f97b352bb..e6ed7f9d59 100644 --- a/roles/gnome_backups/files/ssh_config +++ b/roles/gnome_backups/files/ssh_config @@ -1,4 +1,4 @@ -Host live.gnome.org puppetmaster01.gnome.org cloud.gnome.org webapps3.gnome.org +Host puppetmaster01.gnome.org cloud.gnome.org webapps3.gnome.org User root IdentityFile /usr/local/etc/gnome_backup_id.rsa ProxyCommand ssh -W %h:%p bastion.gnome.org -F /usr/local/etc/gnome_ssh_config diff --git a/roles/gnome_backups/tasks/main.yml b/roles/gnome_backups/tasks/main.yml index 9bb121174f..d95aa7a02a 100644 --- a/roles/gnome_backups/tasks/main.yml +++ b/roles/gnome_backups/tasks/main.yml @@ -41,7 +41,7 @@ - bastion.gnome.org - spinner.gnome.org - master.gnome.org - - live.gnome.org + - wiki.gnome.org - restaurant.gnome.org - expander.gnome.org - accelerator.gnome.org From de9af8dedabfd5960dacbe76331a04558f9d710d Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Wed, 7 Feb 2018 19:48:49 +0100 Subject: [PATCH 134/242] Add src.fp.o OIDC push OIDC scope Signed-off-by: Patrick Uiterwijk --- roles/ipsilon/files/oidc_scopes/src.py | 14 ++++++++++++++ roles/ipsilon/tasks/main.yml | 1 + roles/ipsilon/templates/configuration.conf | 4 ++-- 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 roles/ipsilon/files/oidc_scopes/src.py diff --git a/roles/ipsilon/files/oidc_scopes/src.py b/roles/ipsilon/files/oidc_scopes/src.py new file mode 100644 index 0000000000..eed4eaca4f --- /dev/null +++ b/roles/ipsilon/files/oidc_scopes/src.py @@ -0,0 +1,14 @@ +from __future__ import absolute_import + +from ipsilon.providers.openidc.plugins.common import OpenidCExtensionBase + + +class OpenidCExtension(OpenidCExtensionBase): + name = 'src' + display_name = 'Dist-Git' + scopes = { + 'https://src.fedoraproject.org/push': { + 'display_name': 'Push to Fedora Dist-Git', + 'claims': [], + }, + } diff --git a/roles/ipsilon/tasks/main.yml b/roles/ipsilon/tasks/main.yml index 2d24aebda8..0aeb5d8805 100644 --- a/roles/ipsilon/tasks/main.yml +++ b/roles/ipsilon/tasks/main.yml @@ -49,6 +49,7 @@ - odcs - wiki - freshmaker + - src notify: - reload apache tags: diff --git a/roles/ipsilon/templates/configuration.conf b/roles/ipsilon/templates/configuration.conf index a8cf050236..fd41a8d732 100644 --- a/roles/ipsilon/templates/configuration.conf +++ b/roles/ipsilon/templates/configuration.conf @@ -23,9 +23,9 @@ global enabled=allow global enabled=persona,openid,saml2,openidc {% if env == "production" %} -openidc enabled extensions=fedora-account,mbs,beaker,waiverdb,odcs,wiki,freshmaker +openidc enabled extensions=fedora-account,mbs,beaker,waiverdb,odcs,wiki,freshmaker,src {% else %} -openidc enabled extensions=fedora-account,mbs,beaker,waiverdb,odcs,wiki,freshmaker +openidc enabled extensions=fedora-account,mbs,beaker,waiverdb,odcs,wiki,freshmaker,src {% endif %} {% if env == 'staging' %} From b9b720043bc537b2d598f0d2b02e4a5ac7df6f37 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Wed, 7 Feb 2018 20:12:22 +0100 Subject: [PATCH 135/242] Mark the username 'git' as blacklisted Signed-off-by: Patrick Uiterwijk --- roles/fas_server/templates/fas.cfg.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/fas_server/templates/fas.cfg.j2 b/roles/fas_server/templates/fas.cfg.j2 index bd8801ad5c..f646346db3 100644 --- a/roles/fas_server/templates/fas.cfg.j2 +++ b/roles/fas_server/templates/fas.cfg.j2 @@ -76,9 +76,9 @@ ipa_sync_certfile = '/etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt' # Usernames that are unavailable for fas allocation {% if env == "staging" %} -username_blacklist = "abuse,accounts,adm,admin,amanda,apache,askfedora,asterisk,axk4545,bin,board,bodhi,bodhi2,canna,census,chair,chairman,containerbuild,cvsdirsec,cvsdocs,cvseclipse,cvsextras,cvsfont,daemon,dbus,decode,desktop,dgilmore,directors,dovecot,dumper,fama,famsco,fas,fas_sync,fax,fedora,fedorarewards,fesco,freemedia,freshmaker,ftbfs,ftp,ftpadm,ftpadmin,ftpsync,games,gdm,gnomebackup,gopher,gregdek,grokmirror,halt,hostmaster,hotness,ident,info,ingres,jaboutboul,jan,jwf,keys,kojiadmin,ldap,legal,logo,lp,m8y,mail,mailnull,manager,marketing,masher,masta,mirrormanager,mysql,nagios,named,netdump,news,newsadm,newsadmin,nfsnobody,nobody,noc,notifications,nrpe,nscd,ntp,nut,openvideo,operator,packager,patrick,pcap,pkgdb,pkgsigner,postfix,postgres,postmaster,press,privoxy,pvm,quagga,radiusd,radvd,relnotes,relrod,rel-eng,root,rpc,rpcuser,rpm,rsc,s3-mirror,sales,scholarship,secalert,secondary-signer,security,server-wg,shutdown,smmsp,spevack,squid,sshd,support,sync,system,tickets,toor,updates,usenet,uucp,vcsa,vendors,vendor-support,voting,webalizer,webmaster,wikiadmin,wnn,www,xfs,zabbix" +username_blacklist = "abuse,accounts,adm,admin,amanda,apache,askfedora,asterisk,axk4545,bin,board,bodhi,bodhi2,canna,census,chair,chairman,containerbuild,cvsdirsec,cvsdocs,cvseclipse,cvsextras,cvsfont,daemon,dbus,decode,desktop,dgilmore,directors,dovecot,dumper,fama,famsco,fas,fas_sync,fax,fedora,fedorarewards,fesco,freemedia,freshmaker,ftbfs,ftp,ftpadm,ftpadmin,ftpsync,games,gdm,git,gnomebackup,gopher,gregdek,grokmirror,halt,hostmaster,hotness,ident,info,ingres,jaboutboul,jan,jwf,keys,kojiadmin,ldap,legal,logo,lp,m8y,mail,mailnull,manager,marketing,masher,masta,mirrormanager,mysql,nagios,named,netdump,news,newsadm,newsadmin,nfsnobody,nobody,noc,notifications,nrpe,nscd,ntp,nut,openvideo,operator,packager,patrick,pcap,pkgdb,pkgsigner,postfix,postgres,postmaster,press,privoxy,pvm,quagga,radiusd,radvd,relnotes,relrod,rel-eng,root,rpc,rpcuser,rpm,rsc,s3-mirror,sales,scholarship,secalert,secondary-signer,security,server-wg,shutdown,smmsp,spevack,squid,sshd,support,sync,system,tickets,toor,updates,usenet,uucp,vcsa,vendors,vendor-support,voting,webalizer,webmaster,wikiadmin,wnn,www,xfs,zabbix" {% else %} -username_blacklist = "abuse,accounts,adm,admin,amanda,apache,askfedora,asterisk,axk4545,bin,board,bodhi,bodhi2,canna,census,chair,chairman,containerbuild,cvsdirsec,cvsdocs,cvseclipse,cvsextras,cvsfont,daemon,dbus,decode,desktop,dgilmore,directors,dovecot,dumper,fama,famsco,fas,fax,fedora,fedorarewards,fesco,freemedia,freshmaker,ftbfs,ftp,ftpadm,ftpadmin,ftpsync,games,gdm,gnomebackup,gopher,gregdek,grokmirror,halt,hostmaster,hotness,ident,info,ingres,jaboutboul,jan,jwf,keys,kojiadmin,ldap,legal,logo,lp,m8y,mail,mailnull,manager,marketing,masher,masta,mirrormanager,mysql,nagios,named,netdump,news,newsadm,newsadmin,nfsnobody,nobody,noc,notifications,nrpe,nscd,ntp,nut,openvideo,operator,packager,patrick,pcap,pkgdb,pkgsigner,postfix,postgres,postmaster,press,privoxy,pvm,quagga,radiusd,radvd,relnotes,relrod,rel-eng,root,rpc,rpcuser,rpm,rsc,s3-mirror,sales,scholarship,secalert,secondary-signer,security,server-wg,shutdown,smmsp,spevack,squid,sshd,support,sync,system,tickets,toor,updates,usenet,uucp,vcsa,vendors,vendor-support,voting,webalizer,webmaster,wikiadmin,wnn,www,xfs,zabbix" +username_blacklist = "abuse,accounts,adm,admin,amanda,apache,askfedora,asterisk,axk4545,bin,board,bodhi,bodhi2,canna,census,chair,chairman,containerbuild,cvsdirsec,cvsdocs,cvseclipse,cvsextras,cvsfont,daemon,dbus,decode,desktop,dgilmore,directors,dovecot,dumper,fama,famsco,fas,fax,fedora,fedorarewards,fesco,freemedia,freshmaker,ftbfs,ftp,ftpadm,ftpadmin,ftpsync,games,gdm,git,gnomebackup,gopher,gregdek,grokmirror,halt,hostmaster,hotness,ident,info,ingres,jaboutboul,jan,jwf,keys,kojiadmin,ldap,legal,logo,lp,m8y,mail,mailnull,manager,marketing,masher,masta,mirrormanager,mysql,nagios,named,netdump,news,newsadm,newsadmin,nfsnobody,nobody,noc,notifications,nrpe,nscd,ntp,nut,openvideo,operator,packager,patrick,pcap,pkgdb,pkgsigner,postfix,postgres,postmaster,press,privoxy,pvm,quagga,radiusd,radvd,relnotes,relrod,rel-eng,root,rpc,rpcuser,rpm,rsc,s3-mirror,sales,scholarship,secalert,secondary-signer,security,server-wg,shutdown,smmsp,spevack,squid,sshd,support,sync,system,tickets,toor,updates,usenet,uucp,vcsa,vendors,vendor-support,voting,webalizer,webmaster,wikiadmin,wnn,www,xfs,zabbix" {% endif %} email_domain_blacklist = "{{ fas_blocked_emails }}" From b2d8343b3f3b13cdb7dc946b9fdca3f8056c19cf Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Wed, 7 Feb 2018 19:41:36 +0000 Subject: [PATCH 136/242] and the rhn files need to be added someday --- roles/batcave/files/sync-rhn | 1 - roles/batcave/tasks/main.yml | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/batcave/files/sync-rhn b/roles/batcave/files/sync-rhn index a450aa9f7a..5ecececcde 100644 --- a/roles/batcave/files/sync-rhn +++ b/roles/batcave/files/sync-rhn @@ -1,3 +1,2 @@ -30 1 * * * root /mnt/fedora/app/fi-repo/rhel/rhel5/rhel5-sync > /dev/null 30 2 * * * root /mnt/fedora/app/fi-repo/rhel/rhel6/rhel6-sync > /dev/null 30 3 * * * root /mnt/fedora/app/fi-repo/rhel/rhel7/rhel7-sync > /dev/null diff --git a/roles/batcave/tasks/main.yml b/roles/batcave/tasks/main.yml index 2c220d9363..21efd7ab76 100644 --- a/roles/batcave/tasks/main.yml +++ b/roles/batcave/tasks/main.yml @@ -339,7 +339,8 @@ - config when: inventory_hostname.startswith('batcave01') # -# Monday morning run a script to show all the packages we have in infra tags in koji. +# Monday morning run a script to show all the packages we have in infra +# tags in koji. # - name: Install infra-tags-report script From e4baec55e4dd7dada210c88ab84a659064f83135 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 8 Feb 2018 00:33:58 +0100 Subject: [PATCH 137/242] Blacklist username 'pagure' Signed-off-by: Patrick Uiterwijk --- roles/fas_server/templates/fas.cfg.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/fas_server/templates/fas.cfg.j2 b/roles/fas_server/templates/fas.cfg.j2 index f646346db3..c0853eb9ae 100644 --- a/roles/fas_server/templates/fas.cfg.j2 +++ b/roles/fas_server/templates/fas.cfg.j2 @@ -76,9 +76,9 @@ ipa_sync_certfile = '/etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt' # Usernames that are unavailable for fas allocation {% if env == "staging" %} -username_blacklist = "abuse,accounts,adm,admin,amanda,apache,askfedora,asterisk,axk4545,bin,board,bodhi,bodhi2,canna,census,chair,chairman,containerbuild,cvsdirsec,cvsdocs,cvseclipse,cvsextras,cvsfont,daemon,dbus,decode,desktop,dgilmore,directors,dovecot,dumper,fama,famsco,fas,fas_sync,fax,fedora,fedorarewards,fesco,freemedia,freshmaker,ftbfs,ftp,ftpadm,ftpadmin,ftpsync,games,gdm,git,gnomebackup,gopher,gregdek,grokmirror,halt,hostmaster,hotness,ident,info,ingres,jaboutboul,jan,jwf,keys,kojiadmin,ldap,legal,logo,lp,m8y,mail,mailnull,manager,marketing,masher,masta,mirrormanager,mysql,nagios,named,netdump,news,newsadm,newsadmin,nfsnobody,nobody,noc,notifications,nrpe,nscd,ntp,nut,openvideo,operator,packager,patrick,pcap,pkgdb,pkgsigner,postfix,postgres,postmaster,press,privoxy,pvm,quagga,radiusd,radvd,relnotes,relrod,rel-eng,root,rpc,rpcuser,rpm,rsc,s3-mirror,sales,scholarship,secalert,secondary-signer,security,server-wg,shutdown,smmsp,spevack,squid,sshd,support,sync,system,tickets,toor,updates,usenet,uucp,vcsa,vendors,vendor-support,voting,webalizer,webmaster,wikiadmin,wnn,www,xfs,zabbix" +username_blacklist = "abuse,accounts,adm,admin,amanda,apache,askfedora,asterisk,axk4545,bin,board,bodhi,bodhi2,canna,census,chair,chairman,containerbuild,cvsdirsec,cvsdocs,cvseclipse,cvsextras,cvsfont,daemon,dbus,decode,desktop,dgilmore,directors,dovecot,dumper,fama,famsco,fas,fas_sync,fax,fedora,fedorarewards,fesco,freemedia,freshmaker,ftbfs,ftp,ftpadm,ftpadmin,ftpsync,games,gdm,git,gnomebackup,gopher,gregdek,grokmirror,halt,hostmaster,hotness,ident,info,ingres,jaboutboul,jan,jwf,keys,kojiadmin,ldap,legal,logo,lp,m8y,mail,mailnull,manager,marketing,masher,masta,mirrormanager,mysql,nagios,named,netdump,news,newsadm,newsadmin,nfsnobody,nobody,noc,notifications,nrpe,nscd,ntp,nut,openvideo,operator,packager,pagure,patrick,pcap,pkgdb,pkgsigner,postfix,postgres,postmaster,press,privoxy,pvm,quagga,radiusd,radvd,relnotes,relrod,rel-eng,root,rpc,rpcuser,rpm,rsc,s3-mirror,sales,scholarship,secalert,secondary-signer,security,server-wg,shutdown,smmsp,spevack,squid,sshd,support,sync,system,tickets,toor,updates,usenet,uucp,vcsa,vendors,vendor-support,voting,webalizer,webmaster,wikiadmin,wnn,www,xfs,zabbix" {% else %} -username_blacklist = "abuse,accounts,adm,admin,amanda,apache,askfedora,asterisk,axk4545,bin,board,bodhi,bodhi2,canna,census,chair,chairman,containerbuild,cvsdirsec,cvsdocs,cvseclipse,cvsextras,cvsfont,daemon,dbus,decode,desktop,dgilmore,directors,dovecot,dumper,fama,famsco,fas,fax,fedora,fedorarewards,fesco,freemedia,freshmaker,ftbfs,ftp,ftpadm,ftpadmin,ftpsync,games,gdm,git,gnomebackup,gopher,gregdek,grokmirror,halt,hostmaster,hotness,ident,info,ingres,jaboutboul,jan,jwf,keys,kojiadmin,ldap,legal,logo,lp,m8y,mail,mailnull,manager,marketing,masher,masta,mirrormanager,mysql,nagios,named,netdump,news,newsadm,newsadmin,nfsnobody,nobody,noc,notifications,nrpe,nscd,ntp,nut,openvideo,operator,packager,patrick,pcap,pkgdb,pkgsigner,postfix,postgres,postmaster,press,privoxy,pvm,quagga,radiusd,radvd,relnotes,relrod,rel-eng,root,rpc,rpcuser,rpm,rsc,s3-mirror,sales,scholarship,secalert,secondary-signer,security,server-wg,shutdown,smmsp,spevack,squid,sshd,support,sync,system,tickets,toor,updates,usenet,uucp,vcsa,vendors,vendor-support,voting,webalizer,webmaster,wikiadmin,wnn,www,xfs,zabbix" +username_blacklist = "abuse,accounts,adm,admin,amanda,apache,askfedora,asterisk,axk4545,bin,board,bodhi,bodhi2,canna,census,chair,chairman,containerbuild,cvsdirsec,cvsdocs,cvseclipse,cvsextras,cvsfont,daemon,dbus,decode,desktop,dgilmore,directors,dovecot,dumper,fama,famsco,fas,fax,fedora,fedorarewards,fesco,freemedia,freshmaker,ftbfs,ftp,ftpadm,ftpadmin,ftpsync,games,gdm,git,gnomebackup,gopher,gregdek,grokmirror,halt,hostmaster,hotness,ident,info,ingres,jaboutboul,jan,jwf,keys,kojiadmin,ldap,legal,logo,lp,m8y,mail,mailnull,manager,marketing,masher,masta,mirrormanager,mysql,nagios,named,netdump,news,newsadm,newsadmin,nfsnobody,nobody,noc,notifications,nrpe,nscd,ntp,nut,openvideo,operator,packager,pagure,patrick,pcap,pkgdb,pkgsigner,postfix,postgres,postmaster,press,privoxy,pvm,quagga,radiusd,radvd,relnotes,relrod,rel-eng,root,rpc,rpcuser,rpm,rsc,s3-mirror,sales,scholarship,secalert,secondary-signer,security,server-wg,shutdown,smmsp,spevack,squid,sshd,support,sync,system,tickets,toor,updates,usenet,uucp,vcsa,vendors,vendor-support,voting,webalizer,webmaster,wikiadmin,wnn,www,xfs,zabbix" {% endif %} email_domain_blacklist = "{{ fas_blocked_emails }}" From cc6ca2352a75c38f7721a2bb9c3d60277333fa37 Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Thu, 8 Feb 2018 12:24:56 +0100 Subject: [PATCH 138/242] Switch SELinux on Koschei back to enforcing Hopefully SELinux bug workaround shouldn't be needed any longer. --- roles/koschei/common/tasks/main.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/roles/koschei/common/tasks/main.yml b/roles/koschei/common/tasks/main.yml index a9d7cdc82d..af8988e54d 100644 --- a/roles/koschei/common/tasks/main.yml +++ b/roles/koschei/common/tasks/main.yml @@ -1,14 +1,4 @@ --- -# FIXME workaround for https://bugzilla.redhat.com/show_bug.cgi?id=1513704 -# See also: https://pagure.io/fedora-infrastructure/issue/6540 -- name: Put SELinux in permissive mode - selinux: state=permissive policy=targeted - when: ansible_distribution_major_version|int > 26 - tags: - - koschei - - config - - selinux - - name: Add koschei copr dev repo on stg when: env == "staging" template: src=copr.repo.j2 dest=/etc/yum.repos.d/copr.repo From d5db6392b4951871bdca3f1535e7a81bb77b0d9d Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Thu, 8 Feb 2018 12:25:32 +0100 Subject: [PATCH 139/242] Reinstall Koschei on Fedora 27 --- .../host_vars/koschei-web01.phx2.fedoraproject.org | 4 ++-- roles/koschei/common/tasks/main.yml | 12 +----------- roles/koschei/common/templates/copr.repo.j2 | 4 ---- 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/inventory/host_vars/koschei-web01.phx2.fedoraproject.org b/inventory/host_vars/koschei-web01.phx2.fedoraproject.org index 59f61fbf25..01b93972ef 100644 --- a/inventory/host_vars/koschei-web01.phx2.fedoraproject.org +++ b/inventory/host_vars/koschei-web01.phx2.fedoraproject.org @@ -3,8 +3,8 @@ nm: 255.255.255.0 gw: 10.5.125.254 dns: 10.5.126.21 -ks_url: http://10.5.126.23/repo/rhel/ks/kvm-rhel-7 -ks_repo: http://10.5.126.23/repo/rhel/RHEL7-x86_64/ +ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-27 +ks_repo: http://10.5.126.23/pub/fedora/linux/releases/27/Server/x86_64/os/ volgroup: /dev/vg_guests eth0_ip: 10.5.126.140 diff --git a/roles/koschei/common/tasks/main.yml b/roles/koschei/common/tasks/main.yml index af8988e54d..94cfb6d267 100644 --- a/roles/koschei/common/tasks/main.yml +++ b/roles/koschei/common/tasks/main.yml @@ -8,20 +8,10 @@ - packages - yumrepos -- name: Install common packages (Fedora >= 27) +- name: Install common packages package: name={{ item }} state=present with_items: - python3-memcached - when: ansible_distribution_major_version|int > 26 - tags: - - koschei - - packages - -- name: Install common packages (not Fedora or Fedora <= 26) - package: name={{ item }} state=present - with_items: - - python-memcached - when: ansible_distribution_major_version|int < 27 tags: - koschei - packages diff --git a/roles/koschei/common/templates/copr.repo.j2 b/roles/koschei/common/templates/copr.repo.j2 index a614a7ddb2..fccacb922f 100644 --- a/roles/koschei/common/templates/copr.repo.j2 +++ b/roles/koschei/common/templates/copr.repo.j2 @@ -1,10 +1,6 @@ [msimacek-koschei] name=Copr repo for koschei owned by msimacek -{% if is_fedora is defined %} baseurl=https://copr-be.cloud.fedoraproject.org/results/msimacek/koschei/fedora-$releasever-$basearch/ -{% else %} -baseurl=https://copr-be.cloud.fedoraproject.org/results/msimacek/koschei/epel-7-$basearch/ -{% endif %} skip_if_unavailable=True gpgcheck=1 gpgkey=https://copr-be.cloud.fedoraproject.org/results/msimacek/koschei/pubkey.gpg From 0eec23dcd856c80641ce8a2ecd1277a6d1290214 Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Thu, 8 Feb 2018 12:28:19 +0100 Subject: [PATCH 140/242] Switch Koschei web auth to OpenIDC --- inventory/group_vars/koschei-web | 4 +++- inventory/group_vars/koschei-web-stg | 5 ++++- roles/koschei/frontend/tasks/main.yml | 2 +- .../frontend/templates/config-frontend.cfg.j2 | 7 ------- roles/koschei/frontend/templates/httpd.conf.j2 | 18 +----------------- 5 files changed, 9 insertions(+), 27 deletions(-) diff --git a/inventory/group_vars/koschei-web b/inventory/group_vars/koschei-web index 9ddb030906..3f93bccff4 100644 --- a/inventory/group_vars/koschei-web +++ b/inventory/group_vars/koschei-web @@ -12,9 +12,11 @@ koschei_pgsql_hostname: db01.phx2.fedoraproject.org koschei_koji_hub: koji02.phx2.fedoraproject.org koschei_kojipkgs: kojipkgs.fedoraproject.org koschei_koji_web: koji.fedoraproject.org -koschei_openid_provider: id.fedoraproject.org +koschei_oidc_provider: id.fedoraproject.org koschei_bugzilla: bugzilla.redhat.com +koschei_oidc_client_secret: "{{ koschei_oidc_client_secret_prod }}" +koschei_oidc_crypto_secret: "{{ koschei_oidc_crypto_secret_prod }}" tcp_ports: [ 80, 443 ] diff --git a/inventory/group_vars/koschei-web-stg b/inventory/group_vars/koschei-web-stg index aabde7a764..c3692a7c30 100644 --- a/inventory/group_vars/koschei-web-stg +++ b/inventory/group_vars/koschei-web-stg @@ -11,9 +11,12 @@ koschei_topurl: https://apps.stg.fedoraproject.org/koschei koschei_pgsql_hostname: pgbdr.stg.phx2.fedoraproject.org koschei_kojipkgs: koji.stg.fedoraproject.org koschei_koji_web: koji.stg.fedoraproject.org -koschei_openid_provider: id.stg.fedoraproject.org +koschei_oidc_provider: id.stg.fedoraproject.org koschei_bugzilla: partner-bugzilla.redhat.com +koschei_oidc_client_secret: "{{ koschei_oidc_client_secret_stg }}" +koschei_oidc_crypto_secret: "{{ koschei_oidc_crypto_secret_stg }}" + tcp_ports: [ 80, 443 ] custom_rules: [ diff --git a/roles/koschei/frontend/tasks/main.yml b/roles/koschei/frontend/tasks/main.yml index dc8f201c00..85e5bc3010 100644 --- a/roles/koschei/frontend/tasks/main.yml +++ b/roles/koschei/frontend/tasks/main.yml @@ -4,7 +4,7 @@ - koschei-frontend - koschei-frontend-fedora - koschei-frontend-copr - - "{{ 'mod_auth_openidc' if env == 'staging' else 'mod_auth_openid' }}" + - mod_auth_openidc tags: - koschei - packages diff --git a/roles/koschei/frontend/templates/config-frontend.cfg.j2 b/roles/koschei/frontend/templates/config-frontend.cfg.j2 index dd18b0005c..9ae9d25ba9 100644 --- a/roles/koschei/frontend/templates/config-frontend.cfg.j2 +++ b/roles/koschei/frontend/templates/config-frontend.cfg.j2 @@ -61,20 +61,13 @@ config = { "frontend": { "builds_per_page": 8, "auth": { - {% if env == 'staging' %} "user_re": "(.+)", "user_env": "OIDC_CLAIM_nickname", - {% else %} - "user_re": "http://(.+)\\.id{{ env_prefix }}\\.fedoraproject\\.org/", - {% endif %} }, "fedora_assets_url": "/global", "fedmenu_url": "/fedmenu", "fedmenu_data_url": "/js/data.js", }, - "openid": { - "openid_provider": "{{ koschei_openid_provider }}", - }, "links": [ {"name": "Packages", "url": "https://apps{{ env_prefix }}.fedoraproject.org/packages/{package.name}"}, diff --git a/roles/koschei/frontend/templates/httpd.conf.j2 b/roles/koschei/frontend/templates/httpd.conf.j2 index 67032217b4..1dd652c575 100644 --- a/roles/koschei/frontend/templates/httpd.conf.j2 +++ b/roles/koschei/frontend/templates/httpd.conf.j2 @@ -16,17 +16,11 @@ Require all granted -{% if env == 'staging' %} OIDCRedirectURI "{{ koschei_topurl }}/login/redirect_uri" - OIDCProviderMetadataURL "https://{{ koschei_openid_provider }}/openidc/wellknown_openid_configuration" + OIDCProviderMetadataURL "https://{{ koschei_oidc_provider }}/openidc/wellknown_openid_configuration" OIDCClientID "koschei" - {% if env == 'staging' %} - OIDCClientSecret "{{ koschei_oidc_client_secret_stg }}" - OIDCCryptoPassphrase "{{ koschei_oidc_crypto_secret_stg }}" - {% else %} OIDCClientSecret "{{ koschei_oidc_client_secret }}" OIDCCryptoPassphrase "{{ koschei_oidc_crypto_secret }}" - {% endif %} OIDCSSLValidateServer On OIDCResponseType "code" @@ -36,14 +30,4 @@ AuthType openid-connect Require valid-user -{% else %} - - Require valid-user - AuthType OpenID - AuthOpenIDSingleIdP https://{{ koschei_openid_provider }}/ - AuthOpenIDServerName https://apps.fedoraproject.org - AuthOpenIDTrustRoot https://apps.fedoraproject.org/koschei/ - AuthOpenIDUseCookie off - -{% endif %} From da2e9766d7454e85eb1dc528f7b151366fdbc0de Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Thu, 8 Feb 2018 12:46:50 +0100 Subject: [PATCH 141/242] Make Koschei backend auth to Koji using GSSAPI --- roles/koschei/backend/tasks/main.yml | 1 - roles/koschei/backend/templates/config-backend.cfg.j2 | 11 ----------- roles/koschei/backend/templates/koji.conf.j2 | 10 +++------- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/roles/koschei/backend/tasks/main.yml b/roles/koschei/backend/tasks/main.yml index e49693f91d..3ebf99e2f4 100644 --- a/roles/koschei/backend/tasks/main.yml +++ b/roles/koschei/backend/tasks/main.yml @@ -56,7 +56,6 @@ src=systemd-environment.conf.j2 dest=/etc/systemd/system/{{ item }}.service.d/environment.conf with_items: "{{ koschei_backend_services }}" - when: env == 'staging' notify: - reload systemd - restart koschei backend services diff --git a/roles/koschei/backend/templates/config-backend.cfg.j2 b/roles/koschei/backend/templates/config-backend.cfg.j2 index e34c230d66..f24f61b83d 100644 --- a/roles/koschei/backend/templates/config-backend.cfg.j2 +++ b/roles/koschei/backend/templates/config-backend.cfg.j2 @@ -14,18 +14,7 @@ config = { "server": "https://{{ koschei_koji_hub }}/kojihub", "weburl": "https://{{ koschei_koji_web }}/koji", "topurl": "https://{{ koschei_kojipkgs }}", - {% if env == 'staging' %} "login_method": "gssapi_login", - {% else %} - "login_method": "krb_login", - "login_args": { - "keytab": "/etc/krb5.koschei_{{ inventory_hostname }}.keytab", - "principal": "koschei/{{ inventory_hostname }}@{{ ipa_realm }}", - }, - "session_opts": { - "krb_rdns": False, - }, - {% endif %} {% if env == 'staging' %} "max_builds": 16, "build_arches": ['x86_64'], diff --git a/roles/koschei/backend/templates/koji.conf.j2 b/roles/koschei/backend/templates/koji.conf.j2 index efaa5f37ad..b12fe932bd 100644 --- a/roles/koschei/backend/templates/koji.conf.j2 +++ b/roles/koschei/backend/templates/koji.conf.j2 @@ -1,11 +1,7 @@ [koji] -server = http://{{ koschei_koji_hub }}/kojihub -topurl = http://{{ koschei_kojipkgs }} -weburl = http://{{ koschei_koji_web }}/koji +server = https://{{ koschei_koji_hub }}/kojihub +topurl = https://{{ koschei_kojipkgs }} +weburl = https://{{ koschei_koji_web }}/koji topdir = /mnt/koji authtype = kerberos krbservice = host -principal = koschei/{{ inventory_hostname }}@{{ ipa_realm }} -keytab = /etc/krb5.koschei_{{ inventory_hostname }}.keytab -ccache = /tmp/koschei-koji-krb-ccache -krb_rdns = False From 9397a3579fc5f073276fd30a60ca8c809164e5cb Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Thu, 8 Feb 2018 13:22:52 +0100 Subject: [PATCH 142/242] Disable koschei logging mail handler --- roles/koschei/backend/templates/config-backend.cfg.j2 | 3 ++- roles/koschei/frontend/templates/config-frontend.cfg.j2 | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/roles/koschei/backend/templates/config-backend.cfg.j2 b/roles/koschei/backend/templates/config-backend.cfg.j2 index f24f61b83d..d3a17fefd1 100644 --- a/roles/koschei/backend/templates/config-backend.cfg.j2 +++ b/roles/koschei/backend/templates/config-backend.cfg.j2 @@ -63,7 +63,8 @@ config = { "logging": { "loggers": { "": { - "handlers": ["stderr", "email"], + # "handlers": ["stderr", "email"], + "handlers": ["stderr"], }, "fedmsg": { "level": "ERROR", diff --git a/roles/koschei/frontend/templates/config-frontend.cfg.j2 b/roles/koschei/frontend/templates/config-frontend.cfg.j2 index 9ae9d25ba9..c08b0738c6 100644 --- a/roles/koschei/frontend/templates/config-frontend.cfg.j2 +++ b/roles/koschei/frontend/templates/config-frontend.cfg.j2 @@ -21,7 +21,8 @@ config = { "loggers": { "": { "level": "INFO", - "handlers": ["stderr", "email"], + # "handlers": ["stderr", "email"], + "handlers": ["stderr"], }, "requests": { "level": "WARN", From 833d9e9babe7033a324360905752291529a37818 Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Thu, 8 Feb 2018 12:41:19 +0000 Subject: [PATCH 143/242] Increase memory and disk size on Koschei web prod --- inventory/group_vars/koschei-web | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inventory/group_vars/koschei-web b/inventory/group_vars/koschei-web index 3f93bccff4..1f41f22520 100644 --- a/inventory/group_vars/koschei-web +++ b/inventory/group_vars/koschei-web @@ -1,7 +1,7 @@ --- # Define resources for this group of hosts here. -lvm_size: 6000 -mem_size: 1024 +lvm_size: 8000 +mem_size: 2048 num_cpus: 1 # for systems that do not match the above - specify the same parameter in From c1ab76cef45f209cd416167ba3c05193adf42e60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Zatloukal?= Date: Thu, 8 Feb 2018 13:57:19 +0100 Subject: [PATCH 144/242] Taskotron: patch imagefactoryd service file so it can be enabled on boot --- roles/taskotron/imagefactory/tasks/main.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/roles/taskotron/imagefactory/tasks/main.yml b/roles/taskotron/imagefactory/tasks/main.yml index a25061a11c..d28843714f 100644 --- a/roles/taskotron/imagefactory/tasks/main.yml +++ b/roles/taskotron/imagefactory/tasks/main.yml @@ -16,6 +16,13 @@ - name: hotfix imagefactory's REST api to allow file download copy: src=hotfix_imgfac_RESTv2.py dest=/usr/lib/python2.7/site-packages/imgfac/rest/RESTv2.py owner=root group=root mode=0644 +- name: allow imagefactoryd.service to be enabled + blockinfile: + path: /usr/lib/systemd/system/imagefactoryd.service + block: | + [Install] + WantedBy=multi-user.target + - name: enable imagefactory service: name=imagefactoryd state=started enabled=yes From 557f79da69e92404926d6455982c3bf9cbe06087 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 8 Feb 2018 14:51:43 +0000 Subject: [PATCH 145/242] Add basis for pagure-proxy01 Signed-off-by: Patrick Uiterwijk --- inventory/group_vars/pagure-proxy | 23 ++++++++++++++++ .../pagure-proxy01.fedoraproject.org | 26 +++++++++++++++++++ inventory/inventory | 3 +++ 3 files changed, 52 insertions(+) create mode 100644 inventory/group_vars/pagure-proxy create mode 100644 inventory/host_vars/pagure-proxy01.fedoraproject.org diff --git a/inventory/group_vars/pagure-proxy b/inventory/group_vars/pagure-proxy new file mode 100644 index 0000000000..0f28d4c963 --- /dev/null +++ b/inventory/group_vars/pagure-proxy @@ -0,0 +1,23 @@ +--- +# for systems that do not match the above - specify the same parameter in +# the host_vars/$hostname file + +tcp_ports: [ 22, 25, 80, 443, 9418, + # Used for the eventsource + 8088, + # This is for the pagure public fedmsg relay + 9940] + +fas_client_groups: sysadmin-noc + +freezes: true +postfix_group: vpn.pagure + +# For the MOTD +csi_security_category: Low +csi_primary_contact: Fedora admins - admin@fedoraproject.org +csi_purpose: Proxy specific ports to OSUOSL for preventing slow peering +csi_relationship: | + This box proxies traffic over to pagure01.fedoraproject.org + + (This is done because OSUOSL has terribly slow peering to EU) diff --git a/inventory/host_vars/pagure-proxy01.fedoraproject.org b/inventory/host_vars/pagure-proxy01.fedoraproject.org new file mode 100644 index 0000000000..67791b0829 --- /dev/null +++ b/inventory/host_vars/pagure-proxy01.fedoraproject.org @@ -0,0 +1,26 @@ +--- +nm: 255.255.255.128 +gw: 140.211.169.193 +dns: 8.8.8.8 + +custom_rules: [ + '-t nat -A PREROUTING --dst 152.19.134.147 -p tcp --dport 22 -j DNAT --to-destination 140.211.169.204:22', + '-t nat -A POSTROUTING -p tcp --dst 140.211.169.204 --dport 22 -j SNAT --to-source 152.19.134.147', + '-t nat -A OUTPUT --dst 152.19.134.147 -p tcp --dport 22 -j DNAT --to-destination 140.211.169.204:22'] + + +ks_url: http://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-7-ext +ks_repo: http://infrastructure.fedoraproject.org/repo/rhel/RHEL7-x86_64/ + +volgroup: /dev/vg_guests + +eth0_ip: 152.19.134.146 +eth0_nm: 255.255.255.128 +has_ipv6: yes +eth0_ipv6: "2610:28:3090:3001:dead:beef:cafe:fe46" +eth0_ipv6_gw: "2610:28:3090:3001::1" + +sponsor: ibiblio +datacenter: ibiblio +postfix_group: vpn +vmhost: ibiblio01.fedoraproject.org diff --git a/inventory/inventory b/inventory/inventory index 1d4eeda9b2..696e28728d 100644 --- a/inventory/inventory +++ b/inventory/inventory @@ -1318,6 +1318,9 @@ pagure01.fedoraproject.org [pagure-stg] pagure-stg01.fedoraproject.org +[pagure-proxy] +pagure-proxy01.fedoraproject.org + [twisted-buildbots] twisted-fedora24-1.fedorainfracloud.org twisted-fedora24-2.fedorainfracloud.org From a5a51cadc520bd0720686313172ee5331195a8f7 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 8 Feb 2018 14:53:09 +0000 Subject: [PATCH 146/242] Add playbook for pagure-proxy Signed-off-by: Patrick Uiterwijk --- playbooks/groups/pagure-proxy.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 playbooks/groups/pagure-proxy.yml diff --git a/playbooks/groups/pagure-proxy.yml b/playbooks/groups/pagure-proxy.yml new file mode 100644 index 0000000000..91af5b917f --- /dev/null +++ b/playbooks/groups/pagure-proxy.yml @@ -0,0 +1,28 @@ +- import_playbook: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=pagure:pagure-stg" + +- name: make the boxen be real for real + hosts: pagure-proxy + user: root + gather_facts: True + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - base + - rkhunter + - nagios_client + - hosts + - fas_client + - sudo + - collectd/base + + tasks: + - import_tasks: "{{ tasks_path }}/yumrepos.yml" + - import_tasks: "{{ tasks_path }}/2fa_client.yml" + - import_tasks: "{{ tasks_path }}/motd.yml" + + handlers: + - import_tasks: "{{ handlers_path }}/restart_services.yml" From 1af277590e85c01fcad60483679157a8d8a387b4 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 8 Feb 2018 14:54:37 +0000 Subject: [PATCH 147/242] Virt-install the correct box Signed-off-by: Patrick Uiterwijk --- playbooks/groups/pagure-proxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/groups/pagure-proxy.yml b/playbooks/groups/pagure-proxy.yml index 91af5b917f..2f63511206 100644 --- a/playbooks/groups/pagure-proxy.yml +++ b/playbooks/groups/pagure-proxy.yml @@ -1,4 +1,4 @@ -- import_playbook: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=pagure:pagure-stg" +- import_playbook: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=pagure-proxy" - name: make the boxen be real for real hosts: pagure-proxy From 7a058ec0ba97b0554c59cb77706bca012c7250c0 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 8 Feb 2018 15:00:55 +0000 Subject: [PATCH 148/242] Fix up pagure-proxy01 gateway Signed-off-by: Patrick Uiterwijk --- inventory/host_vars/pagure-proxy01.fedoraproject.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/host_vars/pagure-proxy01.fedoraproject.org b/inventory/host_vars/pagure-proxy01.fedoraproject.org index 67791b0829..de9dda3f3c 100644 --- a/inventory/host_vars/pagure-proxy01.fedoraproject.org +++ b/inventory/host_vars/pagure-proxy01.fedoraproject.org @@ -1,6 +1,6 @@ --- nm: 255.255.255.128 -gw: 140.211.169.193 +gw: 152.19.134.129 dns: 8.8.8.8 custom_rules: [ From da39a966675cac746c6522d918df8c4fa3ed73be Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 8 Feb 2018 15:22:04 +0000 Subject: [PATCH 149/242] Temporarily override cpu model for virtinstall Signed-off-by: Patrick Uiterwijk --- inventory/group_vars/all | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/all b/inventory/group_vars/all index 1adcf0d772..fedcf44d7c 100644 --- a/inventory/group_vars/all +++ b/inventory/group_vars/all @@ -78,7 +78,7 @@ virt_install_command_one_nic: virt-install -n {{ inventory_hostname }} hostname={{ inventory_hostname }} nameserver={{ dns }} ip={{ eth0_ip }}::{{ gw }}:{{ nm }}:{{ inventory_hostname }}:eth0:none' --network bridge={{ main_bridge }},model=virtio - --autostart --noautoconsole --watchdog default + --autostart --noautoconsole --watchdog default --cpu host virt_install_command_two_nic: virt-install -n {{ inventory_hostname }} --memory={{ mem_size }},maxmemory={{ max_mem_size }} --memballoon virtio From ace5dbaa1764803819c01bbaf8a29d8f1e03c70c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Zatloukal?= Date: Thu, 8 Feb 2018 16:20:50 +0100 Subject: [PATCH 150/242] Taskotron: Properly fix imagefactory service Workaround for https://github.com/redhat-imaging/imagefactory/issues/417 --- files/imagefactoryd.service | 12 ++++++++++++ roles/taskotron/imagefactory/tasks/main.yml | 16 ++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 files/imagefactoryd.service diff --git a/files/imagefactoryd.service b/files/imagefactoryd.service new file mode 100644 index 0000000000..83d2341334 --- /dev/null +++ b/files/imagefactoryd.service @@ -0,0 +1,12 @@ +[Unit] +Requires=libvirtd.service +After=libvirtd.service + +[Service] +Type=forking +ExecStart=/usr/bin/imagefactoryd +ExecStop=/usr/bin/killall imagefactoryd +PIDFile=/var/run/imagefactoryd.pid + +[Install] +WantedBy=multi-user.target diff --git a/roles/taskotron/imagefactory/tasks/main.yml b/roles/taskotron/imagefactory/tasks/main.yml index d28843714f..85a5ec8b51 100644 --- a/roles/taskotron/imagefactory/tasks/main.yml +++ b/roles/taskotron/imagefactory/tasks/main.yml @@ -16,12 +16,16 @@ - name: hotfix imagefactory's REST api to allow file download copy: src=hotfix_imgfac_RESTv2.py dest=/usr/lib/python2.7/site-packages/imgfac/rest/RESTv2.py owner=root group=root mode=0644 -- name: allow imagefactoryd.service to be enabled - blockinfile: - path: /usr/lib/systemd/system/imagefactoryd.service - block: | - [Install] - WantedBy=multi-user.target +# Workaround for https://github.com/redhat-imaging/imagefactory/issues/417 +- name: fix issues in imagefactoryd.service + copy: + src: imagefactoryd.service + dest: /etc/systemd/system/imagefactoryd.service + register: imagefactory_service + +- name: reload systemd + command: systemctl daemon-reload + when: imagefactory_service.changed - name: enable imagefactory service: name=imagefactoryd state=started enabled=yes From d5ce7a014e9a49e357f4049480a092df028f0a43 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 8 Feb 2018 15:29:38 +0000 Subject: [PATCH 151/242] Add nat-rules Signed-off-by: Patrick Uiterwijk --- inventory/host_vars/pagure-proxy01.fedoraproject.org | 2 +- roles/base/templates/iptables/iptables | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/inventory/host_vars/pagure-proxy01.fedoraproject.org b/inventory/host_vars/pagure-proxy01.fedoraproject.org index de9dda3f3c..9eba1ec977 100644 --- a/inventory/host_vars/pagure-proxy01.fedoraproject.org +++ b/inventory/host_vars/pagure-proxy01.fedoraproject.org @@ -3,7 +3,7 @@ nm: 255.255.255.128 gw: 152.19.134.129 dns: 8.8.8.8 -custom_rules: [ +nat_rules: [ '-t nat -A PREROUTING --dst 152.19.134.147 -p tcp --dport 22 -j DNAT --to-destination 140.211.169.204:22', '-t nat -A POSTROUTING -p tcp --dst 140.211.169.204 --dport 22 -j SNAT --to-source 152.19.134.147', '-t nat -A OUTPUT --dst 152.19.134.147 -p tcp --dport 22 -j DNAT --to-destination 140.211.169.204:22'] diff --git a/roles/base/templates/iptables/iptables b/roles/base/templates/iptables/iptables index 0e2f4178ce..7efd6202cc 100644 --- a/roles/base/templates/iptables/iptables +++ b/roles/base/templates/iptables/iptables @@ -110,3 +110,14 @@ -A INPUT -j REJECT --reject-with icmp-host-prohibited -A FORWARD -j REJECT --reject-with icmp-host-prohibited COMMIT + +{%- if nat_rules %} +*filter +:INPUT ACCEPT [0:0] +:FORWARD ACCEPT [0:0] +:OUTPUT ACCEPT [0:0] + +{% for rule in nat_rules %} +{{ rule }} +{% endfor %} +{% endif %} From 40fbf2d5753887c16a534a77399b4d6e96751dc0 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 8 Feb 2018 15:31:24 +0000 Subject: [PATCH 152/242] Do not remove all whitespace Signed-off-by: Patrick Uiterwijk --- roles/base/templates/iptables/iptables | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/base/templates/iptables/iptables b/roles/base/templates/iptables/iptables index 7efd6202cc..7a1bb4a925 100644 --- a/roles/base/templates/iptables/iptables +++ b/roles/base/templates/iptables/iptables @@ -111,7 +111,7 @@ -A FORWARD -j REJECT --reject-with icmp-host-prohibited COMMIT -{%- if nat_rules %} +{% if nat_rules %} *filter :INPUT ACCEPT [0:0] :FORWARD ACCEPT [0:0] From a4f04d53f9cc8e79a28db0bad43d078da26c14b5 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 8 Feb 2018 15:32:55 +0000 Subject: [PATCH 153/242] -t nat not needed Signed-off-by: Patrick Uiterwijk --- inventory/host_vars/pagure-proxy01.fedoraproject.org | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/inventory/host_vars/pagure-proxy01.fedoraproject.org b/inventory/host_vars/pagure-proxy01.fedoraproject.org index 9eba1ec977..007155bbf6 100644 --- a/inventory/host_vars/pagure-proxy01.fedoraproject.org +++ b/inventory/host_vars/pagure-proxy01.fedoraproject.org @@ -4,9 +4,9 @@ gw: 152.19.134.129 dns: 8.8.8.8 nat_rules: [ - '-t nat -A PREROUTING --dst 152.19.134.147 -p tcp --dport 22 -j DNAT --to-destination 140.211.169.204:22', - '-t nat -A POSTROUTING -p tcp --dst 140.211.169.204 --dport 22 -j SNAT --to-source 152.19.134.147', - '-t nat -A OUTPUT --dst 152.19.134.147 -p tcp --dport 22 -j DNAT --to-destination 140.211.169.204:22'] + '-A PREROUTING --dst 152.19.134.147 -p tcp --dport 22 -j DNAT --to-destination 140.211.169.204:22', + '-A POSTROUTING -p tcp --dst 140.211.169.204 --dport 22 -j SNAT --to-source 152.19.134.147', + '-A OUTPUT --dst 152.19.134.147 -p tcp --dport 22 -j DNAT --to-destination 140.211.169.204:22'] ks_url: http://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-7-ext From 645654e584aa7acffe2f23f896585b7fc8d103d0 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 8 Feb 2018 15:35:49 +0000 Subject: [PATCH 154/242] Add empty nat rules Signed-off-by: Patrick Uiterwijk --- inventory/group_vars/all | 1 + 1 file changed, 1 insertion(+) diff --git a/inventory/group_vars/all b/inventory/group_vars/all index fedcf44d7c..1673c86f44 100644 --- a/inventory/group_vars/all +++ b/inventory/group_vars/all @@ -42,6 +42,7 @@ use_default_epel: true udp_ports: [] tcp_ports: [] custom_rules: [] +nat_rules: [] custom6_rules: [] # defaults for virt installs From 0bfb2a2d1f42c5efdf694aa30ac8ecc6da2d9d9e Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 8 Feb 2018 15:37:20 +0000 Subject: [PATCH 155/242] nat_rules go into the nat table Signed-off-by: Patrick Uiterwijk --- roles/base/templates/iptables/iptables | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/base/templates/iptables/iptables b/roles/base/templates/iptables/iptables index 7a1bb4a925..de08397146 100644 --- a/roles/base/templates/iptables/iptables +++ b/roles/base/templates/iptables/iptables @@ -112,7 +112,7 @@ COMMIT {% if nat_rules %} -*filter +*nat :INPUT ACCEPT [0:0] :FORWARD ACCEPT [0:0] :OUTPUT ACCEPT [0:0] From ce78bf8497ec52223812ee6dffd28b1a5eb84224 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 8 Feb 2018 15:38:58 +0000 Subject: [PATCH 156/242] Nat table has different entries Signed-off-by: Patrick Uiterwijk --- roles/base/templates/iptables/iptables | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/base/templates/iptables/iptables b/roles/base/templates/iptables/iptables index de08397146..b368221310 100644 --- a/roles/base/templates/iptables/iptables +++ b/roles/base/templates/iptables/iptables @@ -113,9 +113,10 @@ COMMIT {% if nat_rules %} *nat -:INPUT ACCEPT [0:0] -:FORWARD ACCEPT [0:0] +:PREROUTING ACCEPT [0:] +:INPUT ACCEPT [0:] :OUTPUT ACCEPT [0:0] +:POSTROUTING ACCEPT [0:0] {% for rule in nat_rules %} {{ rule }} From da7f7f89ebc1fcdcd82d7ed1bb0772c0eb1841d0 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 8 Feb 2018 15:39:49 +0000 Subject: [PATCH 157/242] Commit to our changes Signed-off-by: Patrick Uiterwijk --- roles/base/templates/iptables/iptables | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/base/templates/iptables/iptables b/roles/base/templates/iptables/iptables index b368221310..9e1876ef80 100644 --- a/roles/base/templates/iptables/iptables +++ b/roles/base/templates/iptables/iptables @@ -121,4 +121,5 @@ COMMIT {% for rule in nat_rules %} {{ rule }} {% endfor %} +COMMIT {% endif %} From 4936b644e903bd43c09e12d570b5ec24a0f54d3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Zatloukal?= Date: Thu, 8 Feb 2018 16:59:55 +0100 Subject: [PATCH 158/242] Taskotron: correct location of imagefactoryd.service --- .../taskotron/imagefactory/files}/imagefactoryd.service | 1 + 1 file changed, 1 insertion(+) rename {files => roles/taskotron/imagefactory/files}/imagefactoryd.service (75%) diff --git a/files/imagefactoryd.service b/roles/taskotron/imagefactory/files/imagefactoryd.service similarity index 75% rename from files/imagefactoryd.service rename to roles/taskotron/imagefactory/files/imagefactoryd.service index 83d2341334..762959314b 100644 --- a/files/imagefactoryd.service +++ b/roles/taskotron/imagefactory/files/imagefactoryd.service @@ -1,3 +1,4 @@ +# Workaround for https://github.com/redhat-imaging/imagefactory/issues/417 [Unit] Requires=libvirtd.service After=libvirtd.service From 78a7f59ae0a4710a0bfb5c5320f457e8f2dd4a58 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Thu, 8 Feb 2018 10:41:29 -0800 Subject: [PATCH 159/242] Bump openQA asset size limits for staging This is due to an upstream change in how the asset reduction logic works: it now includes 'fixed' assets in the total size calculation, so we should include the size of those assets (which is currently ~110GB) in our limits. This is only on stg for now as only stg has been updated to the new upstream code. Signed-off-by: Adam Williamson --- inventory/group_vars/openqa-stg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inventory/group_vars/openqa-stg b/inventory/group_vars/openqa-stg index 176ae8ffa7..63ad3a4839 100644 --- a/inventory/group_vars/openqa-stg +++ b/inventory/group_vars/openqa-stg @@ -26,8 +26,8 @@ openqa_dbname: openqa-stg openqa_dbhost: db-qa01.qa.fedoraproject.org openqa_dbuser: openqastg openqa_dbpassword: "{{ stg_openqa_dbpassword }}" -openqa_assetsize: 300 -openqa_assetsize_updates: 50 +openqa_assetsize: 410 +openqa_assetsize_updates: 160 openqa_key: "{{ stg_openqa_apikey }}" openqa_secret: "{{ stg_openqa_apisecret }}" From f75c4dfc557a73bedc35db207d98d1726dae115a Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 8 Feb 2018 20:25:32 +0000 Subject: [PATCH 160/242] Merge bodhi production.ini and staging.ini with the shipped version from the upstream rpm. --- roles/bodhi2/backend/tasks/main.yml | 18 +- roles/bodhi2/base/tasks/main.yml | 20 +- roles/bodhi2/base/templates/production.ini.j2 | 546 +++++++++++------- roles/bodhi2/base/templates/staging.ini.j2 | 540 ----------------- 4 files changed, 328 insertions(+), 796 deletions(-) delete mode 100644 roles/bodhi2/base/templates/staging.ini.j2 diff --git a/roles/bodhi2/backend/tasks/main.yml b/roles/bodhi2/backend/tasks/main.yml index 2a16d9b498..814f9d4b8b 100644 --- a/roles/bodhi2/backend/tasks/main.yml +++ b/roles/bodhi2/backend/tasks/main.yml @@ -338,28 +338,14 @@ - bodhi - cron -- name: setup basic /etc/bodhi/ contents (staging) - template: > - src="{{ roles_path }}/bodhi2/base/templates/staging.ini.j2" - dest="/etc/bodhi/production.ini" - owner=apache - group=apache - mode=0600 - when: inventory_hostname.startswith('bodhi-backend') and env == 'staging' - notify: - - reload bodhi httpd - tags: - - config - - bodhi - -- name: setup basic /etc/bodhi/ contents (production) +- name: setup basic /etc/bodhi/ contents template: > src="{{ roles_path }}/bodhi2/base/templates/production.ini.j2" dest="/etc/bodhi/production.ini" owner=apache group=apache mode=0600 - when: inventory_hostname.startswith('bodhi-backend') and env == 'production' + when: inventory_hostname.startswith('bodhi-backend') notify: - reload bodhi httpd tags: diff --git a/roles/bodhi2/base/tasks/main.yml b/roles/bodhi2/base/tasks/main.yml index 121525845f..2ebbb7e75e 100644 --- a/roles/bodhi2/base/tasks/main.yml +++ b/roles/bodhi2/base/tasks/main.yml @@ -19,14 +19,14 @@ - config - bodhi -- name: setup basic /etc/bodhi/ contents (staging) +- name: setup basic /etc/bodhi/ contents template: > - src="staging.ini.j2" + src="production.ini.j2" dest="/etc/bodhi/production.ini" owner=bodhi group=bodhi mode=0600 - when: inventory_hostname.startswith('bodhi0') and env == 'staging' + when: inventory_hostname.startswith('bodhi0') notify: - reload bodhi httpd tags: @@ -43,20 +43,6 @@ - config - bodhi -- name: setup basic /etc/bodhi/ contents (production) - template: > - src="production.ini.j2" - dest="/etc/bodhi/production.ini" - owner=bodhi - group=bodhi - mode=0600 - when: inventory_hostname.startswith('bodhi0') and env == 'production' - notify: - - reload bodhi httpd - tags: - - config - - bodhi - - name: Copy some fedmsg configuration of our own for fedmsg-hub template: > src={{item}} diff --git a/roles/bodhi2/base/templates/production.ini.j2 b/roles/bodhi2/base/templates/production.ini.j2 index bf067f1279..94789b4bb1 100644 --- a/roles/bodhi2/base/templates/production.ini.j2 +++ b/roles/bodhi2/base/templates/production.ini.j2 @@ -1,3 +1,4 @@ +# The commented values in this config file represent the defaults. [filter:proxy-prefix] use = egg:PasteDeploy#prefix prefix = / @@ -7,138 +8,176 @@ scheme = https use = egg:bodhi-server filter-with = proxy-prefix -# Release status -# pre-beta enforces the 'Pre Beta' policy defined here: -# https://fedoraproject.org/wiki/Updates_Policy -f27.status = post_beta - -f27.post_beta.mandatory_days_in_testing = 7 -f27.post_beta.critpath.num_admin_approvals = 0 -f27.post_beta.critpath.min_karma = 2 -f27.post_beta.critpath.stable_after_days_without_negative_karma = 14 - -f27.pre_beta.mandatory_days_in_testing = 3 -f27.pre_beta.critpath.num_admin_approvals = 0 -f27.pre_beta.critpath.min_karma = 1 - -## -## Atomic OSTree support -## This will compose Atomic OSTrees during the push process using the fedmsg-atomic-composer -## https://github.com/fedora-infra/fedmsg-atomic-composer -## -compose_atomic_trees = true - ## ## Messages ## -# A notice to flash on the front page -frontpage_notice = +# The bodhi-approve-testing cron job will post this message as a comment from the bodhi user on +# updates that reach the required time in testing if they are not stable yet. Positional +# substitution is used, and the %d will be replaced with the time in testing required for the +# update. +# testing_approval_msg = This update has reached %d days in testing and can be pushed to stable now if the maintainer wishes -# A notice to flash on the New Update page -newupdate_notice = +# not_yet_tested_msg = This update has not yet met the minimum testing requirements defined in the Package Update Acceptance Criteria -testing_approval_msg = This update has reached %d days in testing and can be pushed to stable now if the maintainer wishes -not_yet_tested_msg = This update has not yet met the minimum testing requirements defined in the Package Update Acceptance Criteria -not_yet_tested_epel_msg = This update has not yet met the minimum testing requirements defined in the EPEL Updates Policy -stablekarma_comment = This update has reached the stable karma threshold and will be pushed to the stable updates repository +# not_yet_tested_epel_msg = This update has not yet met the minimum testing requirements defined in the EPEL Update Policy -testing_approval_msg_based_on_karma = This update has reached the stable karma threshold and can be pushed to stable now if the maintainer wishes. -not_yet_tested_msg_based_on_karma = This update has not reached the stable karma threshold. +# Bodhi will post this comment on Updates that don't use autokarma when they reach the stable +# threshold. +# testing_approval_msg_based_on_karma = This update has reached the stable karma threshold and can be pushed to stable now if the maintainer wishes. + +# The comment that Bodhi will post on updates when a user posts negative karma. +# disable_automatic_push_to_stable = Bodhi is disabling automatic push to stable due to negative karma. The maintainer may push manually if they determine that the issue is not severe. # Libravatar - If this is true libravatar will work as normal. Otherwise, all # libravatar links will be replaced with the string "libravatar.org" so that # the tests can still pass. -libravatar_enabled = True +# libravatar_enabled = True + # Set this to true if you want to do federated dns libravatar lookup -libravatar_dns = False +# libravatar_dns = False + +# If libravatar_dns is True, prefer_ssl will define what gets handed to +# libravatar.libravatar_url()'s https setting. It may be set to True or False, but defaults to None, +# which is effectively False. +# prefer_ssl = # Set this to True in order to send fedmsg messages. +# fedmsg_enabled = False fedmsg_enabled = True - -# Captcha - if 'captcha.secret' is not None, then it will be used for comments -# captcha.secret must be 32 url-safe base64-encoded bytes -# you can generate afresh with >>> cryptography.fernet.Fernet.generate_key() +# Captcha - if 'captcha.secret' is set, then it will be used for comments. Comment it to turn it +# off. captcha.secret must be 32 url-safe base64-encoded bytes. +# You can generate one with >>> cryptography.fernet.Fernet.generate_key() +# captcha.secret = CHANGEME captcha.secret = {{ bodhi2CaptchaSecret }} -# Dimensions -captcha.image_width = 300 -captcha.image_height = 80 -# Any truetype font will do. -captcha.font_path = /usr/share/fonts/liberation/LiberationMono-Regular.ttf -captcha.font_size = 36 -# Colors -captcha.font_color = #000000 -captcha.background_color = #ffffff -# In pixels -captcha.padding = 5 -# If a captcha sits around for this many seconds, it will stop working. -captcha.ttl = 300 -#datagrepper_url = http://localhost:5000 -datagrepper_url = https://apps.fedoraproject.org/datagrepper -badge_ids = binary-star|both-bull-and-self-transcended-tester-viii|catching-the-bull-tester-iv|corporate-drone|corporate-overlord|corporate-shill|discovery-of-the-footprints-tester-ii|in-search-of-the-bull-tester-i|is-this-thing-on-updates-testing-i|is-this-thing-on-updates-testing-ii|is-this-thing-on-updates-testing-iii|is-this-thing-on-updates-testing-iv|it-still-works!|like-a-rock-updates-stable-i|like-a-rock-updates-stable-ii|like-a-rock-updates-stable-iii|like-a-rock-updates-stable-iv|mic-check!-updates-testing-v|missed-the-train|override,-you-say|perceiving-the-bull-tester-iii|reaching-the-source-tester-ix|return-to-society-tester-x|riding-the-bull-home-tester-vi|stop-that-update!|take-this-and-call-me-in-the-morning|taming-the-bull-tester-v|tectonic!-updates-stable-v|the-bull-transcended-tester-vii|what-goes-around-comes-around-karma-i|what-goes-around-comes-around-karma-ii|what-goes-around-comes-around-karma-iii|what-goes-around-comes-around-karma-iv|white-hat|you-can-pry-it-from-my-cold,-dead-hands +# Dimensions +# captcha.image_width = 300 +# captcha.image_height = 80 + +# Any truetype font will do. +# /usr/share/fonts/liberation/LiberationMono-Regular.ttf lives in liberation-mono-fonts. +# /usr/share/fonts/pcaro-hermit/Hermit-medium.otf lives in pcaro-hermit-fonts package. +# captcha.font_path = /usr/share/fonts/liberation/LiberationMono-Regular.ttf +# captcha.font_size = 36 + +# Colors +# captcha.font_color = #000000 +# captcha.background_color = #ffffff + +# In pixels +# captcha.padding = 5 + +# If a captcha sits around for this many seconds, it will stop working. +# captcha.ttl = 300 + + +# The URL for a datagrepper to use in various templates. +# datagrepper_url = https://apps.fedoraproject.org/datagrepper +datagrepper_url = https://apps{{env_suffix}}.fedoraproject.org/datagrepper +# badge_ids = binary-star|both-bull-and-self-transcended-tester-viii|catching-the-bull-tester-iv|corporate-drone|corporate-overlord|corporate-shill|discovery-of-the-footprints-tester-ii|in-search-of-the-bull-tester-i|is-this-thing-on-updates-testing-i|is-this-thing-on-updates-testing-ii|is-this-thing-on-updates-testing-iii|is-this-thing-on-updates-testing-iv|it-still-works!|like-a-rock-updates-stable-i|like-a-rock-updates-stable-ii|like-a-rock-updates-stable-iii|like-a-rock-updates-stable-iv|mic-check!-updates-testing-v|missed-the-train|override,-you-say|perceiving-the-bull-tester-iii|reaching-the-source-tester-ix|return-to-society-tester-x|riding-the-bull-home-tester-vi|stop-that-update!|take-this-and-call-me-in-the-morning|taming-the-bull-tester-v|tectonic!-updates-stable-v|the-bull-transcended-tester-vii|what-goes-around-comes-around-karma-i|what-goes-around-comes-around-karma-ii|what-goes-around-comes-around-karma-iii|what-goes-around-comes-around-karma-iv|white-hat|you-can-pry-it-from-my-cold,-dead-hands ## -## Wiki Test Cases +## Testing ## ## Query the wiki for test cases +# query_wiki_test_cases = False query_wiki_test_cases = True -wiki_url = https://fedoraproject.org/w/api.php -test_case_base_url = https://fedoraproject.org/wiki/ +# wiki_url = https://fedoraproject.org/w/api.php +# test_case_base_url = https://fedoraproject.org/wiki/ +wiki_url = https://{{env_suffix}}fedoraproject.org/w/api.php +test_case_base_url = https://{{env_suffix}}fedoraproject.org/wiki/ + +# URL of the resultsdb for integrating checks and stuff +# resultsdb_api_url = https://taskotron.fedoraproject.org/resultsdb_api/ +resultsdb_url = https://taskotron{{env_suffix}}.fedoraproject.org/resultsdb/ + +# Set this to True to enable gating based on policies enforced by Greenwave. If you set this to +# True, be sure to add a cron job to run the bodhi-check-policies CLI periodically. +# test_gating.required = False +test_gating.required = True + +# If this is set to a URL, a "More information about test gating" link will appear on update pages for users +# to click and learn more. +# test_gating.url = + +# The API url of Greenwave. +# greenwave_api_url = https://greenwave.fedoraproject.org/api/v1.0 +greenwave_api_url = https://greenwave-web-greenwave.app.os{{env_suffix}}.fedoraproject.org/api/v1.0 # Email domain to prepend usernames to -default_email_domain = fedoraproject.org +# default_email_domain = fedoraproject.org +default_email_domain = {{env_suffix}}fedoraproject.org # domain for generated message IDs -message_id_email_domain = admin.fedoraproject.org +# message_id_email_domain = admin.fedoraproject.org +message_id_email_domain = admin{{env_suffix}}.fedoraproject.org ## -## Mash settings +## Masher settings ## +releng_fedmsg_certname = shell-bodhi-backend01{{env_suffix}}.phx2.fedoraproject.org -# If defined, the bodhi masher will ensure that messages are signed with the given cert -{% if ansible_hostname == 'bodhi-backend01' %} -releng_fedmsg_certname = shell-bodhi-backend01.phx2.fedoraproject.org -{% else %} -releng_fedmsg_certname = shell-bodhi-backend03.phx2.fedoraproject.org -{% endif %} - -# The masher is a bodhi instance that is responsible for composing the update -# repositories, regenerating metrics, sending update notices, closing bugs, -# and other costly operations. To set an external masher, set the masher to -# the baseurl of the bodhi instance. If set to None, this bodhi instance -# will act as a masher as well. -#masher = None - -{% if 'backend' in inventory_hostname %} +# Where to initially mash repositories. You can use %(here)s to reference the location of this file. +# mash_dir = mash_dir = /mnt/koji/compose/updates/ -mash_stage_dir = /mnt/koji/compose/updates/ -{% endif %} -pungi.basepath = /etc/bodhi -pungi.conf.rpm = pungi.rpm.conf.j2 -pungi.conf.module = pungi.module.conf.j2 -pungi.labeltype = Update -pungi.extracmdline = --notification-script=/usr/bin/pungi-fedmsg-notification --notification-script=pungi-wait-for-signed-ostree-handler + +# The max number of mash threads running at the same time +# max_concurrent_mashes = 2 max_concurrent_mashes = 4 -## Our periodic jobs -#jobs = clean_repo nagmail fix_bug_titles cache_release_data approve_testing_updates -jobs = cache_release_data refresh_metrics approve_testing_updates +# Where to symlink the latest repos by their tag name. You can use %(here)s to reference the +# location of this file. +# mash_stage_dir = +mash_stage_dir = /mnt/koji/compose/updates/ -## Comps configuration -comps_dir = /var/cache/bodhi/comps -comps_url = https://pagure.io/fedora-comps.git +# The following jinja2 template variables are available for use to customize the Pungi configs and +# variants files to the Release and Updates: +# +# * 'id': The id of the Release being mashed. +# * 'release': The Release being mashed. +# * 'request': The request being mashed. +# * 'updates': The Updates being mashed. +# +# NOTE: The jinja2 configuration for these templates replaces the {'s and }'s with ['s and ]'. +# e.g.: a block becomes [% if Package Update Acceptance Criteria -not_yet_tested_epel_msg = This update has not yet met the minimum testing requirements defined in the EPEL Updates Policy -stablekarma_comment = This update has reached the stable karma threshold and will be pushed to the stable updates repository - -# Libravatar - If this is true libravatar will work as normal. Otherwise, all -# libravatar links will be replaced with the string "libravatar.org" so that -# the tests can still pass. -libravatar_enabled = True -# Set this to true if you want to do federated dns libravatar lookup -libravatar_dns = False - -# Set this to True in order to send fedmsg messages. -fedmsg_enabled = True - - -# Captcha - if 'captcha.secret' is not None, then it will be used for comments -# captcha.secret must be 32 url-safe base64-encoded bytes -# you can generate afresh with >>> cryptography.fernet.Fernet.generate_key() -captcha.secret = {{ bodhi2CaptchaSecretSTG }} -# Dimensions -captcha.image_width = 300 -captcha.image_height = 80 -# Any truetype font will do. -captcha.font_path = /usr/share/fonts/liberation/LiberationMono-Regular.ttf -captcha.font_size = 36 -# Colors -captcha.font_color = #000000 -captcha.background_color = #ffffff -# In pixels -captcha.padding = 5 -# If a captcha sits around for this many seconds, it will stop working. -captcha.ttl = 300 - -#datagrepper_url = http://localhost:5000 -datagrepper_url = https://apps.stg.fedoraproject.org/datagrepper -badge_ids = binary-star|both-bull-and-self-transcended-tester-viii|catching-the-bull-tester-iv|corporate-drone|corporate-overlord|corporate-shill|discovery-of-the-footprints-tester-ii|in-search-of-the-bull-tester-i|is-this-thing-on-updates-testing-i|is-this-thing-on-updates-testing-ii|is-this-thing-on-updates-testing-iii|is-this-thing-on-updates-testing-iv|it-still-works!|like-a-rock-updates-stable-i|like-a-rock-updates-stable-ii|like-a-rock-updates-stable-iii|like-a-rock-updates-stable-iv|mic-check!-updates-testing-v|missed-the-train|override,-you-say|perceiving-the-bull-tester-iii|reaching-the-source-tester-ix|return-to-society-tester-x|riding-the-bull-home-tester-vi|stop-that-update!|take-this-and-call-me-in-the-morning|taming-the-bull-tester-v|tectonic!-updates-stable-v|the-bull-transcended-tester-vii|what-goes-around-comes-around-karma-i|what-goes-around-comes-around-karma-ii|what-goes-around-comes-around-karma-iii|what-goes-around-comes-around-karma-iv|white-hat|you-can-pry-it-from-my-cold,-dead-hands - - -## -## Wiki Test Cases -## - -## Query the wiki for test cases -query_wiki_test_cases = False -wiki_url = https://fedoraproject.org/w/api.php -test_case_base_url = https://fedoraproject.org/wiki/ - -# Email domain to prepend usernames to -default_email_domain = fedoraproject.org - -# domain for generated message IDs -message_id_email_domain = admin.stg.fedoraproject.org - -## -## Mash settings -## - -# If defined, the bodhi masher will ensure that messages are signed with the given cert -releng_fedmsg_certname = shell-bodhi-backend01.stg.phx2.fedoraproject.org - -# The masher is a bodhi instance that is responsible for composing the update -# repositories, regenerating metrics, sending update notices, closing bugs, -# and other costly operations. To set an external masher, set the masher to -# the baseurl of the bodhi instance. If set to None, this bodhi instance -# will act as a masher as well. -#masher = None - -{% if 'backend' in inventory_hostname %} -mash_dir = /mnt/koji/compose/updates/ -mash_stage_dir = /mnt/koji/compose/updates/ -{% endif %} -pungi.basepath = /etc/bodhi -pungi.conf.rpm = pungi.rpm.conf.j2 -pungi.conf.module = pungi.module.conf.j2 -pungi.labeltype = Update -pungi.extracmdline = --notification-script=/usr/bin/pungi-fedmsg-notification --notification-script=pungi-wait-for-signed-ostree-handler - -## Our periodic jobs -#jobs = clean_repo nagmail fix_bug_titles cache_release_data approve_testing_updates -jobs = cache_release_data refresh_metrics approve_testing_updates - -## Comps configuration -comps_dir = /var/cache/bodhi/comps -comps_url = https://pagure.io/fedora-comps.git - -## -## Mirror settings -## -file_url = http://download.fedoraproject.org/pub/fedora/linux/updates - -# {release}_{request}_master_repomd: This is used by the masher to determine when a -# primary architecture push has been synchronized to the master mirror for a given release and -# request. The masher will verify that the checksum of repomd.xml at the master URL matches the -# expected value, and will poll the URL until this test passes. Substitute release and request -# for each release id (replacing -'s with _'s) and request (stable, testing). Used for the -# arches listed in {release}_{version}_primary_arches when it is defined, else used for all -# arches. You must put two %s's in this setting - the first will be replaced with the release -# version and the second will be replaced with the architecture. -fedora_stable_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-updates/compose/Everything/%s/os/repodata/repomd.xml -fedora_testing_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-updates-testing/compose/Everything/%s/os/repodata/repomd.xml -fedora_epel_stable_master_repomd = http://download01.phx2.fedoraproject.org/pub/epel/%s/%s/repodata/repomd.xml -fedora_epel_testing_master_repomd = http://download01.phx2.fedoraproject.org/pub/epel/testing/%s/%s/repodata/repomd.xml -fedora_modular_stable_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-modular-updates/compose/Everything/%s/os/repodata/repomd.xml -fedora_modular_testing_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-modular-updates-testing/compose/Everything/%s/os/repodata/repomd.xml - -# {release}_{request}_alt_master_repomd: This is used by the masher to determine when a -# secondary architecture push has been synchronized to the master mirror for a given release and -# request. The masher will verify that the checksum of repomd.xml at the master URL matches the -# expected value, and will poll the URL until this test passes. Substitute release and request -# for each release id (replacing -'s with _'s) and request (stable, testing). Used for the -# arches not listed in {release}_{version}_primary_arches if it is defined. You must put two %s's -# in this setting - the first will be replaced with the release version and the second will be -# replaced with the architecture. -fedora_stable_alt_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-updates/compose/Everything/%s/os/repodata/repomd.xml -fedora_testing_alt_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-updates-testing/compose/Everything/%s/os/repodata/repomd.xml - - -## The base url of this application -base_address = https://bodhi.stg.fedoraproject.org/ - - -## Primary architechures by release -## -## {release}_{version}_primary_arches: Releases that have alternative arches must define their -## primary arches here. Any arches found during mashing that are not present here are asssumed -## to be alternative arches. This is used during the wait_for_repo() step of the mash where -## Bodhi polls the master repo to find out whether the mash has made it to the repo or not. -## Bodhi looks for primary arches with the {release}_{request}_master_repomd setting above, and -## for alternative arches at the {release}_{request}_alt_master_repomd setting above. If this -## is not set, Bodhi will assume the release only has primary arches. -fedora_26_primary_arches = armhfp x86_64 - - -## Supported update types -update_types = bugfix enhancement security newpackage - -## Supported architechures -## -## To handle arch name changes between releases, you -## can also configure bodhi to support one arch *or* -## another. For example, EPEL5 mashes produce 'ppc' -## repos, where EPEL6 produces 'ppc64'. To handle this -## scenario, you can specify something like: -## -## arches = ppc/ppc64 -## -arches = i386 x86_64 armhfp - -## -## Email setting -## - -# Keep email disabled in staging so rube doesn't spam helpless packagers. -#smtp_server = bastion - -# The updates system itself. This email address is used in fetching Bugzilla -# information, as well as email notifications -bodhi_email = updates@fedoraproject.org -#bodhi_password = - -# The address that gets the requests -release_team_address = bodhiadmin-members@fedoraproject.org - -# The address to notify when security updates are initially added to bodhi -security_team = security_respons-members@fedoraproject.org - -# Public announcement lists -fedora_announce_list = package-announce@lists.fedoraproject.org -fedora_test_announce_list = test@lists.fedoraproject.org -fedora_epel_announce_list = epel-package-announce@lists.fedoraproject.org -fedora_epel_test_announce_list = epel-devel@lists.fedoraproject.org -fedora_modular_announce_list = package-announce@lists.fedoraproject.org -fedora_modular_test_announce_list = test@lists.fedoraproject.org - -# Superuser groups -admin_groups = proventesters security_respons bodhiadmin sysadmin-main - -# Users that we don't want to show up in the "leaderboard(s)" -stats_blacklist = bodhi anonymous autoqa taskotron - -# A list of non-person users -system_users = bodhi autoqa taskotron - -# The max length for an update title before we truncate it in the web ui -max_update_length_for_ui = 70 - -# The number of days used for calculating the 'top testers' metric -top_testers_timeframe = 900 - -# The email address of the proventesters -proventesters_email = proventesters-members@fedoraproject.org - -# Disabled for the initial release. -stacks_enabled = False - -# These are the default requirements that we apply to stacks, packages, and -# updates. Users have free-reign to override them for each kind of entity. At -# the end of the day, we only consider the requirements defined by single -# updates themselves when gating in the backend masher process. -site_requirements = dist.rpmdeplint dist.upgradepath -## Some day we'll have rpmgrill, and that will be cool. Ask tflink. -#site_requirements = depcheck upgradepath rpmgrill - -# Where do we send update announcements to ? -# These variables should be named per: Release.prefix_id.lower()_announce_list -#fedora_announce_list = -#fedora_test_announce_list = -#fedora_epel_announce_list = -#fedora_epel_test_announce_list = - -# Cache settings -dogpile.cache.backend = dogpile.cache.dbm -dogpile.cache.expiration_time = 100 -dogpile.cache.arguments.filename = /var/cache/bodhi/dogpile-cache.dbm - -# Exclude sending emails to these users -exclude_mail = autoqa taskotron - -## -## Buildsystem settings -## - -# What buildsystem do we want to use? For development, we'll use a fake -# buildsystem that always does what we tell it to do. For production, we'll -# want to use 'koji'. -buildsystem = koji - -# Koji's XML-RPC hub -koji_hub = https://koji.stg.fedoraproject.org/kojihub - -# Root url of the Koji instance to point to. No trailing slash -koji_url = http://koji.stg.fedoraproject.org - -# URL of where users should go to set up their notifications -fmn_url = https://apps.stg.fedoraproject.org/notifications/ - -# URL of the resultsdb for integrating checks and stuff -resultsdb_url = https://taskotron.stg.fedoraproject.org/resultsdb/ -resultsdb_api_url = https://taskotron.stg.fedoraproject.org/resultsdb_api/ - -# Set this to True to enable gating based on policies enforced by Greenwave. If you set this to True, -# be sure to add a cron job to run the bodhi-check-policies CLI periodically. -test_gating.required = True - -# If this is set to a URL, a "More information about test gating" link will appear on update pages for users -# to click and learn more. -# test_gating.url = - -# The API url of Greenwave. -greenwave_api_url = https://greenwave-web-greenwave.app.os.stg.fedoraproject.org/api/v1.0 - -fedmenu.url = https://apps.stg.fedoraproject.org/fedmenu -fedmenu.data_url = https://apps.stg.fedoraproject.org/js/data.js - -# Koji Krb stuff -krb_ccache = /tmp/krb5cc_%{uid} -krb_principal = bodhi/bodhi{{ env_suffix }}.fedoraproject.org@{{ ipa_realm }} -krb_keytab = /etc/krb5.bodhi_bodhi{{ env_suffix }}.fedoraproject.org.keytab - -## -## ACL system -## Choices are 'pkgdb', which will send a JSON query to the pkgdb_url below, -## 'pagure', which will query the pagure_url below, or 'dummy', which will -## always return guest credentials (used for local development). -## -acl_system = pagure - -## -## Package DB -## -pkgdb_url = https://admin.stg.fedoraproject.org/pkgdb - -## -## Pagure -## -pagure_url = https://src.stg.fedoraproject.org/ - -## -## Product Definition Center (PDC) -## -pdc_url = https://pdc.stg.fedoraproject.org/ - - -# We used to get our package tags from pkgdb, but they come from tagger now. -# https://github.com/fedora-infra/fedora-tagger/pull/74 -#pkgtags_url = https://apps.fedoraproject.org/tagger/api/v1/tag/sqlitebuildtags/ - -## -## Bug tracker settings -## -#bugtracker = bugzilla - -initial_bug_msg = %s has been submitted as an update to %s. %s -stable_bug_msg = %s has been pushed to the %s repository. If problems still persist, please make note of it in this bug report. -testing_bug_msg = - If you want to test the update, you can install it with - $ su -c 'dnf --enablerepo=updates-testing update %s' - You can provide feedback for this update here: %s -testing_bug_epel_msg = - If you want to test the update, you can install it with - $ su -c 'yum --enablerepo=epel-testing update %s' - You can provide feedback for this update here: %s - -## -## Bugzilla settings. -## - -# The username/password for our bugzilla account comes -# from the bodhi_{email,password} fields. - -bz_server = https://partner-bugzilla.redhat.com/xmlrpc.cgi -#bz_cookie = - -# Bodhi will avoid touching bugs that are not against the following products -bz_products = Fedora,Fedora EPEL - -buglink = https://partner-bugzilla.redhat.com/show_bug.cgi?id=%s - -## -## Packages that should suggest a reboot -## -reboot_pkgs = kernel kernel-smp kernel-xen-hypervisor kernel-PAE kernel-xen0 kernel-xenU kernel-xen kernel-xen-guest glibc hal dbus - -## -## Critical Path Packages -## https://fedoraproject.org/wiki/Critical_path_package -## - -# You can allow Bodhi to query for critpath packages from the Fedora Package -# Database by setting this value to `pkgdb` or the Product Definition -# Center by setting this value to `pdc`. If it isn't set, it'll just use the -# hardcoded list below. -critpath.type = pdc - -# You can hardcode a list of critical path packages instead of using the PackageDB -critpath_pkgs = kernel - -# The number of admin approvals it takes to be able to push a critical path -# update to stable for a pending release. -critpath.num_admin_approvals = 0 - -# The net karma required to submit a critial path update to a pending release) -critpath.min_karma = 2 - -# Allow critpath to submit for stable after 2 weeks with no negative karma -critpath.stable_after_days_without_negative_karma = 14 - -# The minimum amount of time an update must spend in testing before -# it can reach the stable repository -fedora.mandatory_days_in_testing = 7 -fedora_epel.mandatory_days_in_testing = 14 -fedora_modular.mandatory_days_in_testing = 7 - -## -## Release status -## - -# Pre-beta enforces the Pre Beta policy defined here: -# https://fedoraproject.org/wiki/Updates_Policy -f27.status = pre_beta - -f27.post_beta.mandatory_days_in_testing = 7 -f27.post_beta.critpath.num_admin_approvals = 0 -f27.post_beta.critpath.min_karma = 2 -f27.post_beta.critpath.stable_after_days_without_negative_karma = 14 - -f27.pre_beta.mandatory_days_in_testing = 3 -f27.pre_beta.critpath.num_admin_approvals = 0 -f27.pre_beta.critpath.min_karma = 1 - -# The number of days worth of updates/comments to display -feeds.num_days_to_show = 7 -feeds.max_entries = 20 - -## -## Buildroot Override -## - -# Number of days before expiring overrides -buildroot_overrides.expire_after = 1 - -## -## Groups -## - -# FAS Groups that we want to pay attention to -# When a user logs in, bodhi will look for any of these groups and associate # -# them with the user. They will then appear as the users effective principals in -# the format "group:groupname" and can be used in Pyramid ACE's. -important_groups = proventesters provenpackager releng-team security_respons packager bodhiadmin virtmaint-sig kde-sig eclipse-sig infra-sig gnome-sig python-sig robotics-sig - -# Groups that can push updates for any package -admin_packager_groups = provenpackager releng-team security_respons - -# User must be a member of this group to submit updates -mandatory_packager_groups = packager - -## -## updateinfo.xml configuraiton -## -updateinfo_rights = Copyright (C) 2015 Red Hat, Inc. and others. - -## -## Authentication & Authorization -## - -# pyramid.openid -openid.success_callback = bodhi.server.security:remember_me -openid.provider = https://id.stg.fedoraproject.org/openid/ -openid.url = https://id.stg.fedoraproject.org/ -openid_template = {username}.id.fedoraproject.org -openid.sreg_required = email - -# CORS allowed origins for cornice services -# This can be wide-open. read-only, we don't care as much about. -cors_origins_ro = * -# This should be more locked down to avoid cross-site request forgery. -cors_origins_rw = https://bodhi.stg.fedoraproject.org -cors_connect_src = https://*.fedoraproject.org/ wss://hub.fedoraproject.org:9939/ - - -## -## Pyramid settings -## -pyramid.reload_templates = false -pyramid.debug_authorization = false -pyramid.debug_notfound = false -pyramid.debug_routematch = false -pyramid.default_locale_name = en - -pyramid.includes = - pyramid_tm - -debugtoolbar.hosts = 127.0.0.1 ::1 - -## -## Database -## -sqlalchemy.url = postgresql://bodhi2:{{ bodhi2PasswordSTG }}@pgbdr.stg.phx2.fedoraproject.org/bodhi2 - -## -## Templates -## -mako.directories = bodhi:server/templates - -## -## Authentication & Sessions -## - -authtkt.secret = {{ bodhi2AuthTktSTG }} -session.secret = {{ bodhi2SessionSecretSTG }} -authtkt.secure = true -# How long should an authorization ticket be valid for, in seconds? Defaults to one day. -authtkt.timeout = 1209600 - -# pyramid_beaker -session.type = file -session.data_dir = /var/cache/bodhi/sessions/data -session.lock_dir = /var/cache/bodhi/sessions/lock -session.key = {{ bodhi2SessionKeySTG }} -session.cookie_on_exception = true -# Tell the browser to only send the cookie over TLS -session.secure = true -# Create a cookie that is only valid for one day -session.timeout = 86400 -cache.regions = default_term, second, short_term, long_term -cache.type = memory -cache.second.expire = 1 -cache.short_term.expire = 60 -cache.default_term.expire = 300 -cache.long_term.expire = 3600 - -[server:main] -use = egg:waitress#main -host = 0.0.0.0 -port = 6543 - - -[pshell] -m = bodhi.server.models -t = transaction - -# Begin logging configuration - -[loggers] -keys = root, bodhi, sqlalchemy - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = INFO -handlers = console - -[logger_bodhi] -level = DEBUG -handlers = -qualname = bodhi - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine -# "level = INFO" logs SQL queries. -# "level = DEBUG" logs SQL queries and results. -# "level = WARN" logs neither. (Recommended for production systems.) - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s - -# End logging configuration From 56e9dd9dee959b14d42cace42efafd30769697c2 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 8 Feb 2018 20:36:36 +0000 Subject: [PATCH 161/242] fix conditionals --- roles/bodhi2/base/templates/production.ini.j2 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/bodhi2/base/templates/production.ini.j2 b/roles/bodhi2/base/templates/production.ini.j2 index 94789b4bb1..75cc755cff 100644 --- a/roles/bodhi2/base/templates/production.ini.j2 +++ b/roles/bodhi2/base/templates/production.ini.j2 @@ -403,7 +403,7 @@ bugtracker = bugzilla # bz_server = https://bugzilla.redhat.com/xmlrpc.cgi {% if env == 'production' %} bz_server = https://bugzilla.redhat.com/xmlrpc.cgi -{% if env == 'staging' %} +{% elif env == 'staging' %} bz_server = https://partner-bugzilla.redhat.com/xmlrpc.cgi {% endif %} @@ -416,7 +416,7 @@ bz_products = Fedora,Fedora EPEL # buglink = https://bugzilla.redhat.com/show_bug.cgi?id=%s {% if env == 'production' %} buglink = https://bugzilla.redhat.com/show_bug.cgi?id=%s -{% if env == 'staging' %} +{% elif env == 'staging' %} buglink = https://partner-bugzilla.redhat.com/show_bug.cgi?id=%s {% endif %} @@ -570,7 +570,7 @@ debugtoolbar.hosts = 127.0.0.1 ::1 # sqlalchemy.url = sqlite:////var/cache/bodhi.db {% if env == 'production' %} sqlalchemy.url = postgresql://bodhi2:{{ bodhi2Password }}@db-bodhi/bodhi2 -{% if env == 'staging' %} +{% elif env == 'staging' %} sqlalchemy.url = postgresql://bodhi2:{{ bodhi2PasswordSTG }}@pgbdr.stg.phx2.fedoraproject.org/bodhi2 {% endif %} @@ -593,7 +593,7 @@ sqlalchemy.url = postgresql://bodhi2:{{ bodhi2PasswordSTG }}@pgbdr.stg.phx2.fedo {% if env == 'production' %} authtkt.secret = {{ bodhi2AuthTkt }} session.secret = {{ bodhi2SessionSecret }} -{% if env == 'staging' %} +{% elif env == 'staging' %} authtkt.secret = {{ bodhi2AuthTktSTG }} session.secret = {{ bodhi2SessionSecretSTG }} {% endif %} @@ -606,7 +606,7 @@ session.data_dir = %(here)s/data/sessions/data session.lock_dir = %(here)s/data/sessions/lock {% if env == 'production' %} session.key = {{ bodhi2SessionKey }} -{% if env == 'staging' %} +{% elif env == 'staging' %} session.key = {{ bodhi2SessionKeySTG }} {% endif %} session.cookie_on_exception = true From 595a2c40ad22e97f0e900f4eb7c15ba51c05d981 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 8 Feb 2018 21:33:56 +0000 Subject: [PATCH 162/242] changes to get a working bodhi config. Cannot list mounts that the frontend doesnt have and also need to specify the dogpile cache file as it is different than default --- roles/bodhi2/base/templates/production.ini.j2 | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/roles/bodhi2/base/templates/production.ini.j2 b/roles/bodhi2/base/templates/production.ini.j2 index 75cc755cff..65bdc48641 100644 --- a/roles/bodhi2/base/templates/production.ini.j2 +++ b/roles/bodhi2/base/templates/production.ini.j2 @@ -123,7 +123,12 @@ releng_fedmsg_certname = shell-bodhi-backend01{{env_suffix}}.phx2.fedoraproject. # Where to initially mash repositories. You can use %(here)s to reference the location of this file. # mash_dir = +{% if env == 'production' %} mash_dir = /mnt/koji/compose/updates/ +{% else %} +# do not use on frontends as bodhi will check the mount and refuse to run without it. +#mash_dir = /mnt/koji/compose/updates/ +{% endif %} # The max number of mash threads running at the same time # max_concurrent_mashes = 2 @@ -132,7 +137,12 @@ max_concurrent_mashes = 4 # Where to symlink the latest repos by their tag name. You can use %(here)s to reference the # location of this file. # mash_stage_dir = +{% if env == 'production' %} mash_stage_dir = /mnt/koji/compose/updates/ +{% else %} +# do not use on frontends as bodhi will check the mount and refuse to run without it. +#mash_stage_dir = /mnt/koji/compose/updates/ +{% endif %} # The following jinja2 template variables are available for use to customize the Pungi configs and # variants files to the Release and Updates: @@ -293,6 +303,7 @@ top_testers_timeframe = 900 # dogpile.cache.backend = dogpile.cache.dbm # dogpile.cache.expiration_time = 100 # dogpile.cache.arguments.filename = /var/cache/bodhi-dogpile-cache.dbm +dogpile.cache.arguments.filename = /var/cache/bodhi/dogpile-cache.dbm # Exclude sending emails to these users # exclude_mail = autoqa taskotron From cd65be27740122ce2d69d985193ef3de4d5f1164 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 8 Feb 2018 21:47:19 +0000 Subject: [PATCH 163/242] fix conditional --- roles/bodhi2/base/templates/production.ini.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/bodhi2/base/templates/production.ini.j2 b/roles/bodhi2/base/templates/production.ini.j2 index 65bdc48641..e500a51b75 100644 --- a/roles/bodhi2/base/templates/production.ini.j2 +++ b/roles/bodhi2/base/templates/production.ini.j2 @@ -123,7 +123,7 @@ releng_fedmsg_certname = shell-bodhi-backend01{{env_suffix}}.phx2.fedoraproject. # Where to initially mash repositories. You can use %(here)s to reference the location of this file. # mash_dir = -{% if env == 'production' %} +{% if ansible_hostname.startswith('bodhi-backend') %} mash_dir = /mnt/koji/compose/updates/ {% else %} # do not use on frontends as bodhi will check the mount and refuse to run without it. @@ -137,7 +137,7 @@ max_concurrent_mashes = 4 # Where to symlink the latest repos by their tag name. You can use %(here)s to reference the # location of this file. # mash_stage_dir = -{% if env == 'production' %} +{% if ansible_hostname.startswith('bodhi-backend') %} mash_stage_dir = /mnt/koji/compose/updates/ {% else %} # do not use on frontends as bodhi will check the mount and refuse to run without it. From 4aa1c5bcc398da82a57945da66c3f867f5d0778c Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 8 Feb 2018 22:44:06 +0000 Subject: [PATCH 164/242] Block all but internal from pkgs.fp.o and set up robots.txt files Signed-off-by: Patrick Uiterwijk --- roles/distgit/files/robots-pkgs.txt | 2 ++ roles/distgit/files/robots-src.txt | 8 ++++++ roles/distgit/tasks/main.yml | 8 ++++++ roles/distgit/templates/lookaside-upload.conf | 25 ++++++++++++++++--- 4 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 roles/distgit/files/robots-pkgs.txt create mode 100644 roles/distgit/files/robots-src.txt diff --git a/roles/distgit/files/robots-pkgs.txt b/roles/distgit/files/robots-pkgs.txt new file mode 100644 index 0000000000..1f53798bb4 --- /dev/null +++ b/roles/distgit/files/robots-pkgs.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/roles/distgit/files/robots-src.txt b/roles/distgit/files/robots-src.txt new file mode 100644 index 0000000000..437658ade1 --- /dev/null +++ b/roles/distgit/files/robots-src.txt @@ -0,0 +1,8 @@ +User-agent: * +Disallow: /cgit/ + +User-agent: * +Disallow: /git/ + +User-agent: * +Disallow: /repo/ diff --git a/roles/distgit/tasks/main.yml b/roles/distgit/tasks/main.yml index ac3972171b..9d8c949bf0 100644 --- a/roles/distgit/tasks/main.yml +++ b/roles/distgit/tasks/main.yml @@ -114,6 +114,14 @@ tags: - distgit +- name: Install robots.txt files + copy: src={{item}} dest=/var/www/{{item}} + with_items: + - robots-pkgs.txt + - robots-src.txt + tags: + - distgit + - name: install the DistGit related httpd config copy: src=git-smart-http.conf dest=/etc/httpd/conf.d/dist-git/git-smart-http.conf notify: diff --git a/roles/distgit/templates/lookaside-upload.conf b/roles/distgit/templates/lookaside-upload.conf index ec9afabc35..dc2b8826c1 100644 --- a/roles/distgit/templates/lookaside-upload.conf +++ b/roles/distgit/templates/lookaside-upload.conf @@ -12,13 +12,32 @@ SSLRandomSeed startup file:/dev/urandom 256 SSLRandomSeed connect builtin SSLCryptoDevice builtin +Alias /robots.txt /var/www/robots-src.txt + + Require all granted + + ServerName pkgs{{ env_suffix }}.fedoraproject.org - RewriteCond expr "! -R '192.168.0.0/16'" - RewriteCond expr "! -R '10.0.0.0/8'" - RewriteRule ^(.*)$ https://src.fedoraproject.org/$1 [L,R] + #RewriteCond expr "! -R '192.168.0.0/16'" + #RewriteCond expr "! -R '10.0.0.0/8'" + #RewriteRule ^(.*)$ https://src.fedoraproject.org/$1 [L,R] Alias /repo/ /srv/cache/lookaside/ + + Require ip 127.0.0.1 + Require ip ::1 + Require ip 10.0.0.0/8 + Require ip 192.168.0.0/16 + + + CustomLog "logs/pkgs-access.log" combined + ErrorLog "logs/pkgs-error.log" + Alias /robots.txt /var/www/robots-pkgs.txt + + Require all granted + + RewriteEngine on RewriteRule "^/$" "https://src{{ env_suffix }}.fedoraproject.org/" RewriteRule "^/login/$" "https://src{{ env_suffix }}.fedoraproject.org/login/" From 31fe8d6bcbf89473bcc86648f833e10da09fae8f Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 8 Feb 2018 23:01:22 +0000 Subject: [PATCH 165/242] Open ports 80 and 443 to everyone Signed-off-by: Patrick Uiterwijk --- inventory/group_vars/pkgs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/pkgs b/inventory/group_vars/pkgs index 023f8952a2..1d51c237d5 100644 --- a/inventory/group_vars/pkgs +++ b/inventory/group_vars/pkgs @@ -3,7 +3,7 @@ lvm_size: 100000 mem_size: 4096 num_cpus: 4 -tcp_ports: [ 9418 ] +tcp_ports: [ 9418, 80, 443 ] # We have both celery (pagure_worker) and web thread wanting to send out fedmsg's. # To make things easy on the listening side (so avoid contention of binding ports), let's set the pkgs boxes to active fedmsg. From a2a2d897188805d3717d2e9608a4e5ee31cfba9b Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Fri, 9 Feb 2018 14:54:12 +0800 Subject: [PATCH 166/242] freshmaker: add vars for mbs auth token --- roles/freshmaker/base/defaults/main.yml | 2 ++ roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/roles/freshmaker/base/defaults/main.yml b/roles/freshmaker/base/defaults/main.yml index 44d59bbea4..450f70b940 100644 --- a/roles/freshmaker/base/defaults/main.yml +++ b/roles/freshmaker/base/defaults/main.yml @@ -17,6 +17,8 @@ freshmaker_stg_odcs_server_url: https://odcs.fedoraproject.org freshmaker_prod_odcs_server_url: https://odcs.stg.fedoraproject.org freshmaker_stg_odcs_sigkeys: [] freshmaker_prod_odcs_sigkeys: [] +freshmaker_stg_mbs_auth_token: null +freshmaker_prod_mbs_auth_token: null freshmaker_dry_run: False freshmaker_admins: {"users": [], "groups": []} freshmaker_log_level: info diff --git a/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 b/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 index 2649a6f1bd..092bed3980 100644 --- a/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 +++ b/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 @@ -172,6 +172,8 @@ class ProdConfiguration(BaseConfiguration): KOJI_PROFILE = 'freshmaker_stg' + MBS_AUTH_TOKEN = "{{ freshmaker_stg_mbs_auth_token }}" + PDC_URL = 'https://pdc.stg.fedoraproject.org/rest_api/v1' GIT_BASE_URL = "{{ freshmaker_stg_git_base_url }}" @@ -192,6 +194,8 @@ class ProdConfiguration(BaseConfiguration): KOJI_PROFILE = "freshmaker_production" + MBS_AUTH_TOKEN = "{{ freshmaker_prod_mbs_auth_token }}" + PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1' GIT_BASE_URL = "{{ freshmaker_prod_git_base_url }}" From f97e5bc3711348dd76e675a91268f4a0fa5db3c5 Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Fri, 9 Feb 2018 15:52:49 +0800 Subject: [PATCH 167/242] freshmaker: enable git parser and module build handlers on stage --- inventory/group_vars/freshmaker-stg | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 inventory/group_vars/freshmaker-stg diff --git a/inventory/group_vars/freshmaker-stg b/inventory/group_vars/freshmaker-stg new file mode 100644 index 0000000000..3d1c1fa61e --- /dev/null +++ b/inventory/group_vars/freshmaker-stg @@ -0,0 +1,21 @@ +--- +# For app config +freshmaker_messaging_topic_prefix: +- org.fedoraproject.stg + +freshmaker_parsers: +- freshmaker.parsers.git:GitReceiveParser + +freshmaker_handlers: +- freshmaker.handlers.git:GitModuleMetadataChangeHandler +- freshmaker.handlers.git:GitRPMSpecChangeHandler + +freshmaker_admins: + users: + - jkaluza + - cqi + - qwan + - sochotni + groups: [] + +freshmaker_dry_run: True From 22930dead9e9607f8bd73aae3b29651ca33bfdeb Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Fri, 9 Feb 2018 17:49:07 +0800 Subject: [PATCH 168/242] freshmaker: enable debug log level on stage --- inventory/group_vars/freshmaker-stg | 1 + 1 file changed, 1 insertion(+) diff --git a/inventory/group_vars/freshmaker-stg b/inventory/group_vars/freshmaker-stg index 3d1c1fa61e..76c0217980 100644 --- a/inventory/group_vars/freshmaker-stg +++ b/inventory/group_vars/freshmaker-stg @@ -19,3 +19,4 @@ freshmaker_admins: groups: [] freshmaker_dry_run: True +freshmaker_log_level: debug From ef6f884a0af7d6afcd69e4bb1e22c24cc0f861d7 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Fri, 9 Feb 2018 10:58:29 +0100 Subject: [PATCH 169/242] Disable koschei pagure plugin until I figure out why it segfaults --- roles/koschei/frontend/templates/config-frontend.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/koschei/frontend/templates/config-frontend.cfg.j2 b/roles/koschei/frontend/templates/config-frontend.cfg.j2 index c08b0738c6..8cd3837a3b 100644 --- a/roles/koschei/frontend/templates/config-frontend.cfg.j2 +++ b/roles/koschei/frontend/templates/config-frontend.cfg.j2 @@ -42,7 +42,7 @@ config = { {% if env == 'staging' %} "plugins": ['pagure', 'copr'], {% else %} - "plugins": ['pagure'], + "plugins": [], {% endif %} "caching": { "pagure": { From 05ad37b3f89a1a920875e316cba0ab25c8792109 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Fri, 9 Feb 2018 12:42:38 +0100 Subject: [PATCH 170/242] Set httpd_execmem sebool on koschei-web --- roles/koschei/frontend/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/koschei/frontend/tasks/main.yml b/roles/koschei/frontend/tasks/main.yml index 85e5bc3010..7427e7e5b1 100644 --- a/roles/koschei/frontend/tasks/main.yml +++ b/roles/koschei/frontend/tasks/main.yml @@ -40,6 +40,7 @@ - httpd_can_network_connect - httpd_can_network_connect_db - httpd_can_network_memcache + - httpd_execmem notify: - reload httpd tags: From e66501a312186d7459aed15cffff2ca8e0a33076 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Fri, 9 Feb 2018 12:44:39 +0100 Subject: [PATCH 171/242] Revert "Disable koschei pagure plugin until I figure out why it segfaults" It was selinux vs dogpile.cache. This reverts commit ef6f884a0af7d6afcd69e4bb1e22c24cc0f861d7. --- roles/koschei/frontend/templates/config-frontend.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/koschei/frontend/templates/config-frontend.cfg.j2 b/roles/koschei/frontend/templates/config-frontend.cfg.j2 index 8cd3837a3b..c08b0738c6 100644 --- a/roles/koschei/frontend/templates/config-frontend.cfg.j2 +++ b/roles/koschei/frontend/templates/config-frontend.cfg.j2 @@ -42,7 +42,7 @@ config = { {% if env == 'staging' %} "plugins": ['pagure', 'copr'], {% else %} - "plugins": [], + "plugins": ['pagure'], {% endif %} "caching": { "pagure": { From 45fde4032d5faaaf91a4080d625ff2808bf5137d Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Fri, 9 Feb 2018 12:50:24 +0100 Subject: [PATCH 172/242] Disable distributed memcached lock on koschei. It seems to deadlock for some reason and is not really needed. --- roles/koschei/backend/templates/config-backend.cfg.j2 | 1 - roles/koschei/frontend/templates/config-frontend.cfg.j2 | 1 - 2 files changed, 2 deletions(-) diff --git a/roles/koschei/backend/templates/config-backend.cfg.j2 b/roles/koschei/backend/templates/config-backend.cfg.j2 index d3a17fefd1..aeedd14005 100644 --- a/roles/koschei/backend/templates/config-backend.cfg.j2 +++ b/roles/koschei/backend/templates/config-backend.cfg.j2 @@ -103,7 +103,6 @@ config = { "expiration_time": None, "arguments": { "url": "memcached01", - "distributed_lock": True, }, }, }, diff --git a/roles/koschei/frontend/templates/config-frontend.cfg.j2 b/roles/koschei/frontend/templates/config-frontend.cfg.j2 index c08b0738c6..39041536cd 100644 --- a/roles/koschei/frontend/templates/config-frontend.cfg.j2 +++ b/roles/koschei/frontend/templates/config-frontend.cfg.j2 @@ -51,7 +51,6 @@ config = { "expiration_time": 300, "arguments": { "url": "memcached01:11211", - "distributed_lock": True, }, }, }, From 6995f012324375dce6234a8eea9ce0b7e2587620 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Fri, 9 Feb 2018 12:56:14 +0100 Subject: [PATCH 173/242] Increase koschei pagure cache expiration time --- roles/koschei/frontend/templates/config-frontend.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/koschei/frontend/templates/config-frontend.cfg.j2 b/roles/koschei/frontend/templates/config-frontend.cfg.j2 index 39041536cd..d5ab5612d6 100644 --- a/roles/koschei/frontend/templates/config-frontend.cfg.j2 +++ b/roles/koschei/frontend/templates/config-frontend.cfg.j2 @@ -48,7 +48,7 @@ config = { "pagure": { "users": { "backend": "dogpile.cache.memcached", - "expiration_time": 300, + "expiration_time": 21600, # 6 hours "arguments": { "url": "memcached01:11211", }, From d5bab6cc8a05893db8d54ce624a6e8bb4c7f67d8 Mon Sep 17 00:00:00 2001 From: Randy Barlow Date: Fri, 9 Feb 2018 13:23:10 +0000 Subject: [PATCH 174/242] Correct the CORS setting for staging Bodhi. Signed-off-by: Randy Barlow --- roles/bodhi2/base/templates/production.ini.j2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/bodhi2/base/templates/production.ini.j2 b/roles/bodhi2/base/templates/production.ini.j2 index e500a51b75..bc42f0a9e0 100644 --- a/roles/bodhi2/base/templates/production.ini.j2 +++ b/roles/bodhi2/base/templates/production.ini.j2 @@ -555,7 +555,11 @@ cors_origins_ro = * # This should be more locked down to avoid cross-site request forgery. cors_origins_rw = https://bodhi{{env_suffix}}.fedoraproject.org -cors_connect_src = https://*.{{env_suffix}}fedoraproject.org/ wss://hub.{{env_suffix}}fedoraproject.org:9939/ +{% if env == 'production' %} +cors_connect_src = https://*.fedoraproject.org/ wss://hub.fedoraproject.org:9939/ +{% elif env == 'staging' %} +cors_connect_src = https://*.stg.fedoraproject.org/ wss://hub.stg.fedoraproject.org:9939/ +{% endif %} ## From 24dbc5cf4d00bbcc1f9fa3e94541a839d5ba0414 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Fri, 9 Feb 2018 15:13:32 +0000 Subject: [PATCH 175/242] Open port 80/443 on pkgs01.stg Signed-off-by: Patrick Uiterwijk --- inventory/group_vars/pkgs-stg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/pkgs-stg b/inventory/group_vars/pkgs-stg index 708d8b4406..a26704a8a6 100644 --- a/inventory/group_vars/pkgs-stg +++ b/inventory/group_vars/pkgs-stg @@ -3,7 +3,7 @@ lvm_size: 100000 mem_size: 4096 num_cpus: 4 -tcp_ports: [ 9418 ] +tcp_ports: [ 9418, 80, 443 ] # Definining these vars has a number of effects # 1) mod_wsgi is configured to use the vars for its own setup # 2) iptables opens enough ports for all threads for fedmsg From ab1f81b0ce42091be958d845eff3db80fd4c39ef Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 2 Feb 2018 17:25:46 +0000 Subject: [PATCH 176/242] Just set inventory_hostname to freshmaker_servername for now. --- playbooks/groups/freshmaker.yml | 2 +- .../templates/etc/httpd/conf.d/freshmaker.conf.j2 | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/playbooks/groups/freshmaker.yml b/playbooks/groups/freshmaker.yml index d9b021be00..a9e4586214 100644 --- a/playbooks/groups/freshmaker.yml +++ b/playbooks/groups/freshmaker.yml @@ -61,7 +61,7 @@ - role: freshmaker/frontend # TLS is terminated for us at the proxy layer (like for every other app). freshmaker_force_ssl: False - freshmaker_servername: freshmaker{{env_suffix}}.fedoraproject.org + freshmaker_servername: {{inventory_hostname}} handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml" diff --git a/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 b/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 index 3885635a9c..5ac8340005 100644 --- a/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 +++ b/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 @@ -8,13 +8,13 @@ RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI} WSGIDaemonProcess freshmaker user=fedmsg group=fedmsg processes={{wsgi_procs}} threads={{wsgi_threads}} home=/usr/share/freshmaker WSGIScriptAlias /{{ freshmaker_endpoint }} /usr/share/freshmaker/freshmaker.wsgi -##{% if freshmaker_servername != inventory_hostname %} -### Redirect from the hostname of this machine to user-visible hostname. -##RewriteEngine On -## -##RewriteRule (.*) "%{REQUEST_SCHEME}://{{ freshmaker_servername }}%{REQUEST_URI}" [R,L] -## -##{% endif %} +{% if freshmaker_servername != inventory_hostname %} +# Redirect from the hostname of this machine to user-visible hostname. +RewriteEngine On + +RewriteRule (.*) "%{REQUEST_SCHEME}://{{ freshmaker_servername }}%{REQUEST_URI}" [R,L] + +{% endif %} {% if env == 'staging' %} OIDCOAuthClientID {{ freshmaker_stg_oidc_client_id }} From 86c78df3f992c07570cf22d0f1f751f1872c0ef3 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 9 Feb 2018 16:29:50 +0000 Subject: [PATCH 177/242] Try to fix freshmaker server_name handling. --- playbooks/groups/freshmaker.yml | 5 +++-- roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 | 2 ++ .../frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/playbooks/groups/freshmaker.yml b/playbooks/groups/freshmaker.yml index a9e4586214..8eeb09cdff 100644 --- a/playbooks/groups/freshmaker.yml +++ b/playbooks/groups/freshmaker.yml @@ -61,7 +61,7 @@ - role: freshmaker/frontend # TLS is terminated for us at the proxy layer (like for every other app). freshmaker_force_ssl: False - freshmaker_servername: {{inventory_hostname}} + freshmaker_servername: null handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml" @@ -78,7 +78,8 @@ roles: - fedmsg/base - - freshmaker/backend + - role: freshmaker/backend + freshmaker_servername: freshmaker{{env_suffix}}.fedoraproject.org - role: keytab/service service: freshmaker diff --git a/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 b/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 index 092bed3980..1db04ae470 100644 --- a/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 +++ b/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 @@ -256,4 +256,6 @@ class ProdConfiguration(BaseConfiguration): ADMINS = {{ freshmaker_admins }} LOG_LEVEL = "{{ freshmaker_log_level }}" +{% if freshmaker_servername %} SERVER_NAME = "{{ freshmaker_servername }}" +{% else %} diff --git a/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 b/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 index 5ac8340005..9045aac568 100644 --- a/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 +++ b/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 @@ -8,7 +8,7 @@ RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI} WSGIDaemonProcess freshmaker user=fedmsg group=fedmsg processes={{wsgi_procs}} threads={{wsgi_threads}} home=/usr/share/freshmaker WSGIScriptAlias /{{ freshmaker_endpoint }} /usr/share/freshmaker/freshmaker.wsgi -{% if freshmaker_servername != inventory_hostname %} +{% if freshmaker_servername != inventory_hostname and freshmaker_servername != None %} # Redirect from the hostname of this machine to user-visible hostname. RewriteEngine On From db817ee0f3bbd99b762a2d6b78d84212a0770935 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 9 Feb 2018 16:34:09 +0000 Subject: [PATCH 178/242] Typofix. --- roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 b/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 index 1db04ae470..e7128c694d 100644 --- a/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 +++ b/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 @@ -258,4 +258,4 @@ class ProdConfiguration(BaseConfiguration): LOG_LEVEL = "{{ freshmaker_log_level }}" {% if freshmaker_servername %} SERVER_NAME = "{{ freshmaker_servername }}" -{% else %} +{% endif %} From d65b46a20984939465a6209205662f0030d80f54 Mon Sep 17 00:00:00 2001 From: Randy Barlow Date: Fri, 9 Feb 2018 17:39:36 +0000 Subject: [PATCH 179/242] Redefine master_repomd settings that don't have defaults. Signed-off-by: Randy Barlow --- roles/bodhi2/base/templates/production.ini.j2 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/bodhi2/base/templates/production.ini.j2 b/roles/bodhi2/base/templates/production.ini.j2 index bc42f0a9e0..42cda5087d 100644 --- a/roles/bodhi2/base/templates/production.ini.j2 +++ b/roles/bodhi2/base/templates/production.ini.j2 @@ -219,11 +219,17 @@ fedora_epel_testing_master_repomd = http://download01.phx2.fedoraproject.org/pub # fedora_stable_alt_master_repomd = http://download01.phx2.fedoraproject.org/pub/fedora-secondary/updates/%s/%s/repodata/repomd.xml # fedora_testing_alt_master_repomd = http://download01.phx2.fedoraproject.org/pub/fedora-secondary/updates/testing/%s/%s/repodata/repomd.xml {% if env == 'production' %} +fedora_epel_stable_master_repomd = http://download01.phx2.fedoraproject.org/pub/epel/%s/%s/repodata/repomd.xml +fedora_epel_testing_master_repomd = http://download01.phx2.fedoraproject.org/pub/epel/testing/%s/%s/repodata/repomd.xml fedora_modular_stable_master_repomd = http://download01.phx2.fedoraproject.org/pub/fedora/linux/modular/updates/%s/Server/%s/repodata/repomd.xml fedora_modular_testing_master_repomd = http://download01.phx2.fedoraproject.org/pub/fedora/linux/modular/updates/testing/%s/Server/%s/repodata/repomd.xml +fedora_stable_alt_master_repomd = http://download01.phx2.fedoraproject.org/pub/fedora-secondary/updates/%s/%s/repodata/repomd.xml +fedora_testing_alt_master_repomd = http://download01.phx2.fedoraproject.org/pub/fedora-secondary/updates/testing/%s/%s/repodata/repomd.xml {% elif env == 'staging' %} fedora_modular_stable_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-modular-updates/compose/Everything/%s/os/repodata/repomd.xml fedora_modular_testing_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-modular-updates-testing/compose/Everything/%s/os/repodata/repomd.xml +fedora_stable_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-updates/compose/Everything/%s/os/repodata/repomd.xml +fedora_testing_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-updates-testing/compose/Everything/%s/os/repodata/repomd.xml {% endif %} ## The base url of this application From 39fb321ecd65676b12ba00132ac4a88b472accf1 Mon Sep 17 00:00:00 2001 From: Randy Barlow Date: Fri, 9 Feb 2018 18:39:19 +0000 Subject: [PATCH 180/242] Add the staging alt_master_repomd settings. Also, rearrange the settings to correspond to their blocks for primary and alt arches. Signed-off-by: Randy Barlow --- roles/bodhi2/base/templates/production.ini.j2 | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/roles/bodhi2/base/templates/production.ini.j2 b/roles/bodhi2/base/templates/production.ini.j2 index 42cda5087d..0b63d701a2 100644 --- a/roles/bodhi2/base/templates/production.ini.j2 +++ b/roles/bodhi2/base/templates/production.ini.j2 @@ -201,11 +201,18 @@ fedora_epel_master_repomd = http://download01.phx2.fedoraproject.org/pub/epel/%s # fedora_testing_master_repomd = http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/testing/%s/%s/repodata/repomd.xml # fedora_epel_stable_master_repomd = http://download01.phx2.fedoraproject.org/pub/epel/%s/%s/repodata/repomd.xml # fedora_epel_testing_master_repomd = http://download01.phx2.fedoraproject.org/pub/epel/testing/%s/%s/repodata/repomd.xml -{% if env == 'staging' %} +{% if env == 'production' %} +fedora_epel_stable_master_repomd = http://download01.phx2.fedoraproject.org/pub/epel/%s/%s/repodata/repomd.xml +fedora_epel_testing_master_repomd = http://download01.phx2.fedoraproject.org/pub/epel/testing/%s/%s/repodata/repomd.xml +fedora_modular_stable_master_repomd = http://download01.phx2.fedoraproject.org/pub/fedora/linux/modular/updates/%s/Server/%s/repodata/repomd.xml +fedora_modular_testing_master_repomd = http://download01.phx2.fedoraproject.org/pub/fedora/linux/modular/updates/testing/%s/Server/%s/repodata/repomd.xml +{% elif env == 'staging' %} fedora_stable_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-updates/compose/Everything/%s/os/repodata/repomd.xml fedora_testing_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-updates-testing/compose/Everything/%s/os/repodata/repomd.xml fedora_epel_stable_master_repomd = http://download01.phx2.fedoraproject.org/pub/epel/%s/%s/repodata/repomd.xml fedora_epel_testing_master_repomd = http://download01.phx2.fedoraproject.org/pub/epel/testing/%s/%s/repodata/repomd.xml +fedora_modular_stable_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-modular-updates/compose/Everything/%s/os/repodata/repomd.xml +fedora_modular_testing_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-modular-updates-testing/compose/Everything/%s/os/repodata/repomd.xml {% endif %} # {release}_{request}_alt_master_repomd: This is used by the masher to determine when a @@ -219,17 +226,11 @@ fedora_epel_testing_master_repomd = http://download01.phx2.fedoraproject.org/pub # fedora_stable_alt_master_repomd = http://download01.phx2.fedoraproject.org/pub/fedora-secondary/updates/%s/%s/repodata/repomd.xml # fedora_testing_alt_master_repomd = http://download01.phx2.fedoraproject.org/pub/fedora-secondary/updates/testing/%s/%s/repodata/repomd.xml {% if env == 'production' %} -fedora_epel_stable_master_repomd = http://download01.phx2.fedoraproject.org/pub/epel/%s/%s/repodata/repomd.xml -fedora_epel_testing_master_repomd = http://download01.phx2.fedoraproject.org/pub/epel/testing/%s/%s/repodata/repomd.xml -fedora_modular_stable_master_repomd = http://download01.phx2.fedoraproject.org/pub/fedora/linux/modular/updates/%s/Server/%s/repodata/repomd.xml -fedora_modular_testing_master_repomd = http://download01.phx2.fedoraproject.org/pub/fedora/linux/modular/updates/testing/%s/Server/%s/repodata/repomd.xml fedora_stable_alt_master_repomd = http://download01.phx2.fedoraproject.org/pub/fedora-secondary/updates/%s/%s/repodata/repomd.xml fedora_testing_alt_master_repomd = http://download01.phx2.fedoraproject.org/pub/fedora-secondary/updates/testing/%s/%s/repodata/repomd.xml {% elif env == 'staging' %} -fedora_modular_stable_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-modular-updates/compose/Everything/%s/os/repodata/repomd.xml -fedora_modular_testing_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-modular-updates-testing/compose/Everything/%s/os/repodata/repomd.xml -fedora_stable_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-updates/compose/Everything/%s/os/repodata/repomd.xml -fedora_testing_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-updates-testing/compose/Everything/%s/os/repodata/repomd.xml +fedora_stable_alt_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-updates/compose/Everything/%s/os/repodata/repomd.xml +fedora_testing_alt_master_repomd = https://kojipkgs.stg.fedoraproject.org/compose/updates/f%s-updates-testing/compose/Everything/%s/os/repodata/repomd.xml {% endif %} ## The base url of this application From 00f727dbaf53d07c9976d0693b9711141b5220fe Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Fri, 9 Feb 2018 12:10:33 -0500 Subject: [PATCH 181/242] atomic host life support for f26 Sync out the stable ref too, which is just aliased to the updates ref at this point. --- roles/bodhi2/backend/files/new-updates-sync | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/bodhi2/backend/files/new-updates-sync b/roles/bodhi2/backend/files/new-updates-sync index 142a735166..e220d71210 100755 --- a/roles/bodhi2/backend/files/new-updates-sync +++ b/roles/bodhi2/backend/files/new-updates-sync @@ -57,6 +57,9 @@ RELEASES = {'f27': {'topic': 'fedora', 'from': 'f26-updates', 'ostrees': [{'ref': 'fedora/26/x86_64/updates/atomic-host', 'dest': os.path.join(ATOMICDEST, '26')}], + # Hack around for the fact that ostree on f25 doesn't know links + {'ref': 'fedora/26/x86_64/atomic-host', + 'dest': os.path.join(ATOMICDEST, '26')}], 'to': [{'arches': ['x86_64', 'armhfp', 'source'], 'dest': os.path.join(FEDORADEST, '26')}, {'arches': ['aarch64', 'i386', 'ppc64', 'ppc64le'], From 7eaec298856d6271c5a064f2431b651d4036a9ad Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Sat, 10 Feb 2018 03:01:32 +0000 Subject: [PATCH 182/242] Update koji-reset-staging.sql to recent Koji schema --- .../manual/staging-sync/templates/koji-reset-staging.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/manual/staging-sync/templates/koji-reset-staging.sql b/playbooks/manual/staging-sync/templates/koji-reset-staging.sql index f9b8d2f3ac..2e068666ab 100644 --- a/playbooks/manual/staging-sync/templates/koji-reset-staging.sql +++ b/playbooks/manual/staging-sync/templates/koji-reset-staging.sql @@ -57,7 +57,7 @@ delete from rpminfo where build_id in (select id from build where state<>1); -- expire any active buildroots select now() as time, 'expiring active buildroots' as msg; -update buildroot set state=3, retire_event=get_event() where state=0; +update standard_buildroot set state=3, retire_event=get_event() where state=0; -- enable/disable hosts update host set enabled=False; @@ -152,7 +152,7 @@ insert into user_perms (user_id, perm_id, active, creator_id) values ( ('hotness', 'hotness/hotness01.stg.phx2.fedoraproject.org'), ('containerbuild', 'osbs/osbs.stg.fedoraproject.org'), ('kojira', 'kojira/koji.stg.fedoraproject.org@STG.FEDORAPROJECT.ORG')] %} -update users set krb_principal='{{principal}}@STG.FEDORAPROJECT.ORG' where username='{{username}}'; +update users set krb_principal='{{principal}}@STG.FEDORAPROJECT.ORG' where name='{{username}}'; {% endfor %} update users set krb_principal=replace(krb_principal, '@FEDORAPROJECT.ORG', '@STG.FEDORAPROJECT.ORG'); From cda638a3fe2dddc0025444657cf4c59a82847560 Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Sat, 10 Feb 2018 03:04:06 +0000 Subject: [PATCH 183/242] Koji staging sync: remove hosts from DB prior to adding them Some staging hosts already exist in production Koji (they were added by mistake?), either as hosts or pure users, but can be removed. --- .../staging-sync/templates/koji-reset-staging.sql | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/playbooks/manual/staging-sync/templates/koji-reset-staging.sql b/playbooks/manual/staging-sync/templates/koji-reset-staging.sql index 2e068666ab..e639f94b3c 100644 --- a/playbooks/manual/staging-sync/templates/koji-reset-staging.sql +++ b/playbooks/manual/staging-sync/templates/koji-reset-staging.sql @@ -75,6 +75,8 @@ update repo set state = 3 where state in (0, 1, 2); -- The koji hub is x86_64 and i386 and has createrepo ability {% for host in groups['koji-stg'] %} select now() as time, 'adding staging host {{ host }}' as msg; +delete from host where name='{{ host }}'; +delete from users where name='{{ host }}'; insert into users (name, usertype, krb_principal, status) values ('{{ host }}', 1, 'compile/{{ host }}@STG.FEDORAPROJECT.ORG', 0); insert into host (user_id, name, arches) values ( (select id from users where name='{{host}}'), '{{host}}', 'i386 x86_64'); @@ -87,6 +89,8 @@ insert into host_channels (host_id, channel_id) values ( -- The buildvms are x86_64 and i386 and also have createrepo ability {% for host in groups['buildvm-stg'] %} select now() as time, 'adding staging host {{ host }}' as msg; +delete from host where name='{{ host }}'; +delete from users where name='{{ host }}'; insert into users (name, usertype, krb_principal, status) values ('{{ host }}', 1, 'compile/{{ host }}@STG.FEDORAPROJECT.ORG', 0); insert into host (user_id, name, arches) values ( (select id from users where name='{{host}}'), '{{host}}', 'i386 x86_64'); @@ -100,6 +104,8 @@ insert into host_channels (host_id, channel_id) values ( {% for host in groups['buildvm-aarch64-stg'] %} select now() as time, 'adding staging host {{ host }}' as msg; +delete from host where name='{{ host }}'; +delete from users where name='{{ host }}'; insert into users (name, usertype, krb_principal, status) values ('{{ host }}', 1, 'compile/{{ host }}@STG.FEDORAPROJECT.ORG', 0); insert into host (user_id, name, arches) values ( (select id from users where name='{{host}}'), '{{host}}', 'aarch64'); @@ -113,6 +119,8 @@ insert into host_channels (host_id, channel_id) values ( {% for host in groups['buildvm-ppc64-stg'] %} select now() as time, 'adding staging host {{ host }}' as msg; +delete from host where name='{{ host }}'; +delete from users where name='{{ host }}'; insert into users (name, usertype, krb_principal, status) values ('{{ host }}', 1, 'compile/{{ host }}@STG.FEDORAPROJECT.ORG', 0); insert into host (user_id, name, arches) values ( (select id from users where name='{{host}}'), '{{host}}', 'ppc64'); @@ -126,6 +134,8 @@ insert into host_channels (host_id, channel_id) values ( {% for host in groups['buildvm-ppc64le-stg'] %} select now() as time, 'adding staging host {{ host }}' as msg; +delete from host where name='{{ host }}'; +delete from users where name='{{ host }}'; insert into users (name, usertype, krb_principal, status) values ('{{ host }}', 1, 'compile/{{ host }}@STG.FEDORAPROJECT.ORG', 0); insert into host (user_id, name, arches) values ( (select id from users where name='{{host}}'), '{{host}}', 'ppc64le'); From 861c36103d3f2a3a09d3d3204832b516e8abf631 Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Sat, 10 Feb 2018 02:58:54 +0000 Subject: [PATCH 184/242] Remove ralph and puiterwijk from list of stg-only Koji admins Lets not try to grant admin perm to users that already have it, or we'd get duplicate constraint violation errors. --- playbooks/manual/staging-sync/templates/koji-reset-staging.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/manual/staging-sync/templates/koji-reset-staging.sql b/playbooks/manual/staging-sync/templates/koji-reset-staging.sql index e639f94b3c..8bd43f919c 100644 --- a/playbooks/manual/staging-sync/templates/koji-reset-staging.sql +++ b/playbooks/manual/staging-sync/templates/koji-reset-staging.sql @@ -147,7 +147,7 @@ insert into host_channels (host_id, channel_id) values ( -- Add some people to be admins, only in staging. Feel free to grow this list.. -{% for username in ['modularity', 'mizdebsk', 'ralph', 'psabata', 'puiterwijk', 'jkaluza', 'fivaldi', 'mprahl'] %} +{% for username in ['modularity', 'mizdebsk', 'psabata', 'jkaluza', 'fivaldi', 'mprahl'] %} select now() as time, 'adding staging admin {{username}}' as msg; insert into user_perms (user_id, perm_id, active, creator_id) values ( (select id from users where name='{{username}}'), From 4891cfb03ef2393c8d746264c2065465fd9ac577 Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Sat, 10 Feb 2018 03:14:01 +0000 Subject: [PATCH 185/242] Clean imageinfo listings during Koji staging sync --- .../manual/staging-sync/templates/koji-reset-staging.sql | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/playbooks/manual/staging-sync/templates/koji-reset-staging.sql b/playbooks/manual/staging-sync/templates/koji-reset-staging.sql index 8bd43f919c..80367c53f9 100644 --- a/playbooks/manual/staging-sync/templates/koji-reset-staging.sql +++ b/playbooks/manual/staging-sync/templates/koji-reset-staging.sql @@ -24,6 +24,11 @@ -- [unset kojihub ServerOffline setting] +-- wipe obsolete table that only causes problems with the sync, could +-- even be dropped entirely (together with imageinfo table). +select now() as time, 'wiping imageinfo listings' as msg; +delete from imageinfo_listing; + -- bump sequences (not strictly needed anymore) select now() as time, 'bumping sequences' as msg; alter sequence task_id_seq restart with 90000000; From 1f694e6b5b597cd2ff1d83f12f0f78d2df24968a Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Sat, 10 Feb 2018 03:23:50 +0000 Subject: [PATCH 186/242] Don't sync Koji buildroot listings in staging buildroot_listing is by far the biggest table in koji db (>100 GB for table data, plus indexes) and it's rarely used - skipping it saves us some considerable amount of time. If the table turns out to be needed, the playbook can always be updated not to exclude it. --- playbooks/manual/staging-sync/koji.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/playbooks/manual/staging-sync/koji.yml b/playbooks/manual/staging-sync/koji.yml index 4ac9e96a18..d4172b61db 100644 --- a/playbooks/manual/staging-sync/koji.yml +++ b/playbooks/manual/staging-sync/koji.yml @@ -46,15 +46,14 @@ dest=/var/tmp/koji.dump.xz owner=postgres group=postgres - - command: unxz /var/tmp/koji.dump.xz - creates=/var/tmp/koji.dump # TODO -- stop replication and wipe db's - command: dropdb koji - command: createdb -O koji koji +# buildroot_listing is excluded from the sync to save some time - name: Import the prod db. This will take quite a while. Go get a snack! - shell: cat /var/tmp/koji.dump | psql koji + shell: xzcat /var/tmp/koji.dump.xz | sed '/COPY buildroot_listing /,/\./d' | psql koji - name: repoint all the prod rpm entries at the secondary volume (and other stuff) shell: psql koji < /var/lib/pgsql/koji-reset-staging.sql From fd46e74adcfe5b1f8eec67f54efd9ffb5fb72b56 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Sat, 10 Feb 2018 20:33:44 +0000 Subject: [PATCH 187/242] Finish pagure-proxy nat rules Signed-off-by: Patrick Uiterwijk --- .../pagure-proxy01.fedoraproject.org | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/inventory/host_vars/pagure-proxy01.fedoraproject.org b/inventory/host_vars/pagure-proxy01.fedoraproject.org index 007155bbf6..eff949a00b 100644 --- a/inventory/host_vars/pagure-proxy01.fedoraproject.org +++ b/inventory/host_vars/pagure-proxy01.fedoraproject.org @@ -4,9 +4,35 @@ gw: 152.19.134.129 dns: 8.8.8.8 nat_rules: [ + # SSH '-A PREROUTING --dst 152.19.134.147 -p tcp --dport 22 -j DNAT --to-destination 140.211.169.204:22', '-A POSTROUTING -p tcp --dst 140.211.169.204 --dport 22 -j SNAT --to-source 152.19.134.147', - '-A OUTPUT --dst 152.19.134.147 -p tcp --dport 22 -j DNAT --to-destination 140.211.169.204:22'] + '-A OUTPUT --dst 152.19.134.147 -p tcp --dport 22 -j DNAT --to-destination 140.211.169.204:22', + # SMTP + '-A PREROUTING --dst 152.19.134.147 -p tcp --dport 25 -j DNAT --to-destination 140.211.169.204:25', + '-A POSTROUTING -p tcp --dst 140.211.169.204 --dport 25 -j SNAT --to-source 152.19.134.147', + '-A OUTPUT --dst 152.19.134.147 -p tcp --dport 25 -j DNAT --to-destination 140.211.169.204:25', + # web-80 + '-A PREROUTING --dst 152.19.134.147 -p tcp --dport 80 -j DNAT --to-destination 140.211.169.204:80', + '-A POSTROUTING -p tcp --dst 140.211.169.204 --dport 80 -j SNAT --to-source 152.19.134.147', + '-A OUTPUT --dst 152.19.134.147 -p tcp --dport 80 -j DNAT --to-destination 140.211.169.204:80', + # web-443 + '-A PREROUTING --dst 152.19.134.147 -p tcp --dport 443 -j DNAT --to-destination 140.211.169.204:443', + '-A POSTROUTING -p tcp --dst 140.211.169.204 --dport 443 -j SNAT --to-source 152.19.134.147', + '-A OUTPUT --dst 152.19.134.147 -p tcp --dport 443 -j DNAT --to-destination 140.211.169.204:443', + # 9418 + '-A PREROUTING --dst 152.19.134.147 -p tcp --dport 9418 -j DNAT --to-destination 140.211.169.204:9418', + '-A POSTROUTING -p tcp --dst 140.211.169.204 --dport 9418 -j SNAT --to-source 152.19.134.147', + '-A OUTPUT --dst 152.19.134.147 -p tcp --dport 9418 -j DNAT --to-destination 140.211.169.204:9418', + # Eventsource + '-A PREROUTING --dst 152.19.134.147 -p tcp --dport 8088 -j DNAT --to-destination 140.211.169.204:8088', + '-A POSTROUTING -p tcp --dst 140.211.169.204 --dport 8088 -j SNAT --to-source 152.19.134.147', + '-A OUTPUT --dst 152.19.134.147 -p tcp --dport 8088 -j DNAT --to-destination 140.211.169.204:8088', + # Fedmsg + '-A PREROUTING --dst 152.19.134.147 -p tcp --dport 9940 -j DNAT --to-destination 140.211.169.204:9940', + '-A POSTROUTING -p tcp --dst 140.211.169.204 --dport 9940 -j SNAT --to-source 152.19.134.147', + '-A OUTPUT --dst 152.19.134.147 -p tcp --dport 9940 -j DNAT --to-destination 140.211.169.204:9940', +] ks_url: http://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-7-ext From 5eecd68868a3dab514e44df4d050ed18fe8baa92 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Sat, 10 Feb 2018 22:05:34 +0100 Subject: [PATCH 188/242] Enable ip_forward for proxy Signed-off-by: Patrick Uiterwijk --- playbooks/groups/pagure-proxy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/playbooks/groups/pagure-proxy.yml b/playbooks/groups/pagure-proxy.yml index 2f63511206..68a650b397 100644 --- a/playbooks/groups/pagure-proxy.yml +++ b/playbooks/groups/pagure-proxy.yml @@ -24,5 +24,8 @@ - import_tasks: "{{ tasks_path }}/2fa_client.yml" - import_tasks: "{{ tasks_path }}/motd.yml" + - name: Enable ipv4_forward in sysctl + sysctl: name=net.ipv4.ip_forward value=1 state=present sysctl_set=yes reload=yes + handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml" From 18f1320eb3953b4b26c084b72370b8b61d644662 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Sat, 10 Feb 2018 21:12:24 +0000 Subject: [PATCH 189/242] Support secondary IP Signed-off-by: Patrick Uiterwijk --- inventory/host_vars/pagure-proxy01.fedoraproject.org | 1 + roles/base/templates/ifcfg.j2 | 3 +++ 2 files changed, 4 insertions(+) diff --git a/inventory/host_vars/pagure-proxy01.fedoraproject.org b/inventory/host_vars/pagure-proxy01.fedoraproject.org index eff949a00b..bff11f29ba 100644 --- a/inventory/host_vars/pagure-proxy01.fedoraproject.org +++ b/inventory/host_vars/pagure-proxy01.fedoraproject.org @@ -45,6 +45,7 @@ eth0_nm: 255.255.255.128 has_ipv6: yes eth0_ipv6: "2610:28:3090:3001:dead:beef:cafe:fe46" eth0_ipv6_gw: "2610:28:3090:3001::1" +eth0_secondary_ip: 152.19.134.147 sponsor: ibiblio datacenter: ibiblio diff --git a/roles/base/templates/ifcfg.j2 b/roles/base/templates/ifcfg.j2 index 966803bc56..9656c35e4d 100644 --- a/roles/base/templates/ifcfg.j2 +++ b/roles/base/templates/ifcfg.j2 @@ -36,3 +36,6 @@ IPV6_DEFAULTDEV={{item}} IPV6_DEFAULTGW={{ hostvars[inventory_hostname][item + '_ipv6_gw'] }} IPV6_MTU=1280 {% endif %} +{% if hostvars[inventory_hostname][item + '_secondary_ip'] is defined %} +IPADDR1="{{ hostvars[inventory_hostname][item + '_secondary_ip'] }}" +{% endif %} From 6e7e0bacc59a21ad5081b00e3fb4ba3c1bdfab21 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Sat, 10 Feb 2018 21:21:02 +0000 Subject: [PATCH 190/242] Pagure-proxy doees forwarding... Signed-off-by: Patrick Uiterwijk --- inventory/host_vars/pagure-proxy01.fedoraproject.org | 2 ++ 1 file changed, 2 insertions(+) diff --git a/inventory/host_vars/pagure-proxy01.fedoraproject.org b/inventory/host_vars/pagure-proxy01.fedoraproject.org index bff11f29ba..ea6e9dffce 100644 --- a/inventory/host_vars/pagure-proxy01.fedoraproject.org +++ b/inventory/host_vars/pagure-proxy01.fedoraproject.org @@ -3,6 +3,8 @@ nm: 255.255.255.128 gw: 152.19.134.129 dns: 8.8.8.8 +custom_rules: ['-A FORWARD -j ACCEPT'] + nat_rules: [ # SSH '-A PREROUTING --dst 152.19.134.147 -p tcp --dport 22 -j DNAT --to-destination 140.211.169.204:22', From 173c68df67559c7e8c7a449573602085c4466921 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Sat, 10 Feb 2018 21:27:14 +0000 Subject: [PATCH 191/242] Enable h2 on pagure.io Signed-off-by: Patrick Uiterwijk --- roles/pagure/frontend/templates/0_pagure.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/pagure/frontend/templates/0_pagure.conf b/roles/pagure/frontend/templates/0_pagure.conf index b51d77cba2..fcd7c4d431 100644 --- a/roles/pagure/frontend/templates/0_pagure.conf +++ b/roles/pagure/frontend/templates/0_pagure.conf @@ -68,6 +68,8 @@ WSGIDaemonProcess paguredocs user=git group=git maximum-requests=1000 display-na SSLEngine on SSLProtocol {{ ssl_protocols }} SSLCipherSuite {{ ssl_ciphers }} + Protocols h2 http/1.1 + # Use secure TLSv1.1 and TLSv1.2 ciphers Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" @@ -124,6 +126,7 @@ WSGIDaemonProcess paguredocs user=git group=git maximum-requests=1000 display-na SSLCertificateFile /etc/pki/tls/certs/docs.pagure.org.crt SSLCertificateChainFile /etc/pki/tls/certs/docs.pagure.org.intermediate.crt SSLCertificateKeyFile /etc/pki/tls/certs/docs.pagure.org.key + Protocols h2 http/1.1 {% if env == 'pagure-staging' %} Redirect permanent / https://stg.pagure.io/ {% else %} @@ -144,6 +147,7 @@ WSGIDaemonProcess paguredocs user=git group=git maximum-requests=1000 display-na SSLEngine on SSLProtocol {{ ssl_protocols }} SSLCipherSuite {{ ssl_ciphers }} + Protocols h2 http/1.1 # Use secure TLSv1.1 and TLSv1.2 ciphers Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" @@ -176,6 +180,7 @@ WSGIDaemonProcess paguredocs user=git group=git maximum-requests=1000 display-na {% endif %} AddType application/octet-stream msi + Protocols h2 http/1.1 Options +Indexes From 311a60d262448ba4cd8c195acc42dd3da91eb85a Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Sat, 10 Feb 2018 21:42:16 +0000 Subject: [PATCH 192/242] Revert "Enable h2 on pagure.io" This reverts commit 173c68df67559c7e8c7a449573602085c4466921. --- roles/pagure/frontend/templates/0_pagure.conf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/roles/pagure/frontend/templates/0_pagure.conf b/roles/pagure/frontend/templates/0_pagure.conf index fcd7c4d431..b51d77cba2 100644 --- a/roles/pagure/frontend/templates/0_pagure.conf +++ b/roles/pagure/frontend/templates/0_pagure.conf @@ -68,8 +68,6 @@ WSGIDaemonProcess paguredocs user=git group=git maximum-requests=1000 display-na SSLEngine on SSLProtocol {{ ssl_protocols }} SSLCipherSuite {{ ssl_ciphers }} - Protocols h2 http/1.1 - # Use secure TLSv1.1 and TLSv1.2 ciphers Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" @@ -126,7 +124,6 @@ WSGIDaemonProcess paguredocs user=git group=git maximum-requests=1000 display-na SSLCertificateFile /etc/pki/tls/certs/docs.pagure.org.crt SSLCertificateChainFile /etc/pki/tls/certs/docs.pagure.org.intermediate.crt SSLCertificateKeyFile /etc/pki/tls/certs/docs.pagure.org.key - Protocols h2 http/1.1 {% if env == 'pagure-staging' %} Redirect permanent / https://stg.pagure.io/ {% else %} @@ -147,7 +144,6 @@ WSGIDaemonProcess paguredocs user=git group=git maximum-requests=1000 display-na SSLEngine on SSLProtocol {{ ssl_protocols }} SSLCipherSuite {{ ssl_ciphers }} - Protocols h2 http/1.1 # Use secure TLSv1.1 and TLSv1.2 ciphers Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" @@ -180,7 +176,6 @@ WSGIDaemonProcess paguredocs user=git group=git maximum-requests=1000 display-na {% endif %} AddType application/octet-stream msi - Protocols h2 http/1.1 Options +Indexes From f8017a26cf99965f67fe0cfa08983022752e184c Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Sun, 11 Feb 2018 10:00:45 +0800 Subject: [PATCH 193/242] freshmaker: restart fedmsg-hub daemon 'restart fedmsg-hub' handler from restart_services doesn't work, use our own handler before that fix that one (it's used by many roles, I don't want to break anything before confirm the issue). And update app config (add prod fedmsg topic for stage to listen on) to notify the handler. --- inventory/group_vars/freshmaker-stg | 1 + roles/freshmaker/base/handlers/main.yml | 3 +++ roles/freshmaker/base/tasks/main.yml | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 roles/freshmaker/base/handlers/main.yml diff --git a/inventory/group_vars/freshmaker-stg b/inventory/group_vars/freshmaker-stg index 76c0217980..21cf20127c 100644 --- a/inventory/group_vars/freshmaker-stg +++ b/inventory/group_vars/freshmaker-stg @@ -2,6 +2,7 @@ # For app config freshmaker_messaging_topic_prefix: - org.fedoraproject.stg +- org.fedoraproject.prod freshmaker_parsers: - freshmaker.parsers.git:GitReceiveParser diff --git a/roles/freshmaker/base/handlers/main.yml b/roles/freshmaker/base/handlers/main.yml new file mode 100644 index 0000000000..a536a3b7a7 --- /dev/null +++ b/roles/freshmaker/base/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: restart fedmsg-hub daemon + command: /usr/local/bin/conditional-restart.sh fedmsg-hub python2-fedmsg diff --git a/roles/freshmaker/base/tasks/main.yml b/roles/freshmaker/base/tasks/main.yml index 3b669fbced..0493b360be 100644 --- a/roles/freshmaker/base/tasks/main.yml +++ b/roles/freshmaker/base/tasks/main.yml @@ -34,6 +34,6 @@ mode: 0440 notify: - restart apache - - restart fedmsg-hub + - restart fedmsg-hub daemon tags: - freshmaker From a1175e60a71452808aa38a78e7c045fca7117b5a Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Sun, 11 Feb 2018 14:11:01 +0800 Subject: [PATCH 194/242] freshmaker: show app config on stage for temp troubleshooting Seems the parsers on stage are not enabled successfully, show app config file content for troubleshooting, remove this later. And a minor var change to restart fedmsg-hub. --- inventory/group_vars/freshmaker-stg | 2 +- playbooks/groups/freshmaker.yml | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/inventory/group_vars/freshmaker-stg b/inventory/group_vars/freshmaker-stg index 21cf20127c..ca3e5b5dad 100644 --- a/inventory/group_vars/freshmaker-stg +++ b/inventory/group_vars/freshmaker-stg @@ -1,8 +1,8 @@ --- # For app config freshmaker_messaging_topic_prefix: -- org.fedoraproject.stg - org.fedoraproject.prod +- org.fedoraproject.stg freshmaker_parsers: - freshmaker.parsers.git:GitReceiveParser diff --git a/playbooks/groups/freshmaker.yml b/playbooks/groups/freshmaker.yml index 8eeb09cdff..704ab7f4f0 100644 --- a/playbooks/groups/freshmaker.yml +++ b/playbooks/groups/freshmaker.yml @@ -89,3 +89,14 @@ handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml" + +- name: show freshmaker app config on stage (for temp troubleshoot) + hosts: freshmaker-stg + + tasks: + - name: Slurp hosts file + slurp: + src: /etc/freshmaker/config.py + register: slurpfile + + - debug: msg="{{ slurpfile['content'] | b64decode }}" From f20559c617ac169a268f93ce48390af2fa34dbf9 Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Sun, 11 Feb 2018 15:06:07 +0800 Subject: [PATCH 195/242] freshmaker: remove the temporary task Seems nothing wrong with app config, revert the slurp task, and remove 'org.fedoraproject.prod' from message topic prefix on stage that was added for troubleshooting too. --- inventory/group_vars/freshmaker-stg | 2 +- playbooks/groups/freshmaker.yml | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/inventory/group_vars/freshmaker-stg b/inventory/group_vars/freshmaker-stg index ca3e5b5dad..21cf20127c 100644 --- a/inventory/group_vars/freshmaker-stg +++ b/inventory/group_vars/freshmaker-stg @@ -1,8 +1,8 @@ --- # For app config freshmaker_messaging_topic_prefix: -- org.fedoraproject.prod - org.fedoraproject.stg +- org.fedoraproject.prod freshmaker_parsers: - freshmaker.parsers.git:GitReceiveParser diff --git a/playbooks/groups/freshmaker.yml b/playbooks/groups/freshmaker.yml index 704ab7f4f0..8eeb09cdff 100644 --- a/playbooks/groups/freshmaker.yml +++ b/playbooks/groups/freshmaker.yml @@ -89,14 +89,3 @@ handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml" - -- name: show freshmaker app config on stage (for temp troubleshoot) - hosts: freshmaker-stg - - tasks: - - name: Slurp hosts file - slurp: - src: /etc/freshmaker/config.py - register: slurpfile - - - debug: msg="{{ slurpfile['content'] | b64decode }}" From e5a9e25aa25217718ce89cb73b95c5118e100255 Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Sun, 11 Feb 2018 16:40:05 +0800 Subject: [PATCH 196/242] freshmaker: whitelist testmodule on stage --- inventory/group_vars/freshmaker-stg | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/inventory/group_vars/freshmaker-stg b/inventory/group_vars/freshmaker-stg index 21cf20127c..8dc4dfb9ff 100644 --- a/inventory/group_vars/freshmaker-stg +++ b/inventory/group_vars/freshmaker-stg @@ -2,7 +2,6 @@ # For app config freshmaker_messaging_topic_prefix: - org.fedoraproject.stg -- org.fedoraproject.prod freshmaker_parsers: - freshmaker.parsers.git:GitReceiveParser @@ -21,3 +20,9 @@ freshmaker_admins: freshmaker_dry_run: True freshmaker_log_level: debug + +freshmaker_handler_build_whitelist: + global: + module: + - name: + - testmodule From a4096ccd0d30ca07addafdc230153d64d8916b07 Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Sun, 11 Feb 2018 16:55:41 +0800 Subject: [PATCH 197/242] freshmaker: add MBS urls in app config --- roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 b/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 index e7128c694d..4b38a8e5cc 100644 --- a/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 +++ b/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 @@ -172,6 +172,7 @@ class ProdConfiguration(BaseConfiguration): KOJI_PROFILE = 'freshmaker_stg' + MBS_BASE_URL = "https://mbs.stg.fedoraproject.org" MBS_AUTH_TOKEN = "{{ freshmaker_stg_mbs_auth_token }}" PDC_URL = 'https://pdc.stg.fedoraproject.org/rest_api/v1' @@ -194,6 +195,7 @@ class ProdConfiguration(BaseConfiguration): KOJI_PROFILE = "freshmaker_production" + MBS_BASE_URL = "https://mbs.fedoraproject.org" MBS_AUTH_TOKEN = "{{ freshmaker_prod_mbs_auth_token }}" PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1' From 40df3ce04a0f5020212d0b6613af98ed4bdbeb90 Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Mon, 12 Feb 2018 11:04:37 +0000 Subject: [PATCH 198/242] Open web ports on commops.fedorainfracloud.org, #6687 --- inventory/host_vars/commops.fedorainfracloud.org | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inventory/host_vars/commops.fedorainfracloud.org b/inventory/host_vars/commops.fedorainfracloud.org index 7308615384..679851c81f 100644 --- a/inventory/host_vars/commops.fedorainfracloud.org +++ b/inventory/host_vars/commops.fedorainfracloud.org @@ -2,9 +2,9 @@ image: "{{ fedora27_x86_64 }}" instance_type: m1.medium keypair: fedora-admin-20130801 -security_group: ssh-anywhere-persistent,all-icmp-persistent,default +security_group: ssh-anywhere-persistent,web-80-anywhere-persistent,web-443-anywhere-persistent,all-icmp-persistent,default zone: nova -tcp_ports: [22] +tcp_ports: [22, 80, 443] inventory_tenant: persistent inventory_instance_name: commops From e35814f1406243e7e984a08fe6c1d1067d3cf32d Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Mon, 12 Feb 2018 11:49:41 +0000 Subject: [PATCH 199/242] Add mizdebsk to all cloud tenants --- .../hosts/fed-cloud09.cloud.fedoraproject.org.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index c81f259413..437d3bbc8c 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -866,7 +866,18 @@ - { user: puiterwijk, tenant: transient } - { user: puiterwijk, tenant: maintainertest } - { user: puiterwijk, tenant: aos-ci-cd } + - { user: mizdebsk, tenant: aos-ci-cd } + - { user: mizdebsk, tenant: cloudintern } + - { user: mizdebsk, tenant: cloudsig } + - { user: mizdebsk, tenant: copr } + - { user: mizdebsk, tenant: coprdev } - { user: mizdebsk, tenant: infrastructure } + - { user: mizdebsk, tenant: maintainertest } + - { user: mizdebsk, tenant: openshift } + - { user: mizdebsk, tenant: persistent } + - { user: mizdebsk, tenant: pythonbots } + - { user: mizdebsk, tenant: qa } + - { user: mizdebsk, tenant: scratch } - { user: mizdebsk, tenant: transient } - { user: clime, tenant: coprdev } - { user: clime, tenant: persistent } From 5c4f49ade3eac747b24f037556df6d5f58f2b956 Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Mon, 12 Feb 2018 14:51:34 +0100 Subject: [PATCH 200/242] Staging hosts should use staging fedmsg CA (#6621) --- roles/fedmsg/base/templates/ssl.py.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/fedmsg/base/templates/ssl.py.j2 b/roles/fedmsg/base/templates/ssl.py.j2 index bbd338fdcd..d77e4456dc 100644 --- a/roles/fedmsg/base/templates/ssl.py.j2 +++ b/roles/fedmsg/base/templates/ssl.py.j2 @@ -4,8 +4,10 @@ config = dict( ssldir="/etc/pki/fedmsg", {% if env == 'staging' %} + ca_cert_location="https://stg.fedoraproject.org/fedmsg/ca.crt" crl_location="https://stg.fedoraproject.org/fedmsg/crl.pem", {% else %} + ca_cert_location="https://fedoraproject.org/fedmsg/ca.crt" crl_location="https://fedoraproject.org/fedmsg/crl.pem", {% endif %} crl_cache="/var/run/fedmsg/crl.pem", From f73b9f89343e151a1feec2d2b322ce27522a9c6b Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Mon, 12 Feb 2018 15:21:34 +0100 Subject: [PATCH 201/242] Open firewall port to pagure proxy Signed-off-by: Patrick Uiterwijk --- roles/base/templates/iptables/iptables.kojibuilder | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/base/templates/iptables/iptables.kojibuilder b/roles/base/templates/iptables/iptables.kojibuilder index 8ae5fdf096..59a9b62bdb 100644 --- a/roles/base/templates/iptables/iptables.kojibuilder +++ b/roles/base/templates/iptables/iptables.kojibuilder @@ -91,6 +91,7 @@ # git on pagure,io -A OUTPUT -p tcp -m tcp -d 140.211.169.204 --dport 443 -j ACCEPT +-A OUTPUT -p tcp -m tcp -d -152.19.134.147 --dport 443 -j ACCEPT # admin.fedoraproject.org for fas (proyx(1)01 and proxy(1)10) -A OUTPUT -p tcp -m tcp -d 10.5.126.8 --dport 80 -j ACCEPT From d56a613b5d92ba2c75b78a82254a8f4c63d0d618 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Mon, 12 Feb 2018 15:26:13 +0100 Subject: [PATCH 202/242] Remove stray characters Signed-off-by: Patrick Uiterwijk --- roles/base/templates/iptables/iptables.kojibuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/base/templates/iptables/iptables.kojibuilder b/roles/base/templates/iptables/iptables.kojibuilder index 59a9b62bdb..51daa98bd8 100644 --- a/roles/base/templates/iptables/iptables.kojibuilder +++ b/roles/base/templates/iptables/iptables.kojibuilder @@ -91,7 +91,7 @@ # git on pagure,io -A OUTPUT -p tcp -m tcp -d 140.211.169.204 --dport 443 -j ACCEPT --A OUTPUT -p tcp -m tcp -d -152.19.134.147 --dport 443 -j ACCEPT +-A OUTPUT -p tcp -m tcp -d 152.19.134.147 --dport 443 -j ACCEPT # admin.fedoraproject.org for fas (proyx(1)01 and proxy(1)10) -A OUTPUT -p tcp -m tcp -d 10.5.126.8 --dport 80 -j ACCEPT From ff890ad0858e1c4a3d812dcb8a3b3fbb292cd1ad Mon Sep 17 00:00:00 2001 From: Randy Barlow Date: Mon, 12 Feb 2018 16:31:31 +0000 Subject: [PATCH 203/242] Add new waiverdb settings to Bodhi's config. Signed-off-by: Randy Barlow --- roles/bodhi2/base/templates/production.ini.j2 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/roles/bodhi2/base/templates/production.ini.j2 b/roles/bodhi2/base/templates/production.ini.j2 index 0b63d701a2..0d3af1e256 100644 --- a/roles/bodhi2/base/templates/production.ini.j2 +++ b/roles/bodhi2/base/templates/production.ini.j2 @@ -108,6 +108,13 @@ test_gating.required = True # greenwave_api_url = https://greenwave.fedoraproject.org/api/v1.0 greenwave_api_url = https://greenwave-web-greenwave.app.os{{env_suffix}}.fedoraproject.org/api/v1.0 +# The URL for waiverdb's API +# waiverdb_api_url = https://waiverdb-web-waiverdb.app.os.fedoraproject.org/api/v1.0 +waiverdb_api_url = https://waiverdb-web-waiverdb.app.os{{env_suffix}}.fedoraproject.org/api/v1.0 + +# An access token used to authenticate to waiverdb +# waiverdb.access_token = + # Email domain to prepend usernames to # default_email_domain = fedoraproject.org default_email_domain = {{env_suffix}}fedoraproject.org From 30594ba5d158cc2f2d0ab385aa774f40141b7d71 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 12 Feb 2018 18:31:21 +0000 Subject: [PATCH 204/242] these lists need to be defined apparently --- roles/bodhi2/base/templates/production.ini.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/bodhi2/base/templates/production.ini.j2 b/roles/bodhi2/base/templates/production.ini.j2 index 0d3af1e256..c19ac06d62 100644 --- a/roles/bodhi2/base/templates/production.ini.j2 +++ b/roles/bodhi2/base/templates/production.ini.j2 @@ -283,6 +283,10 @@ bodhi_password = {{ bodhiBugzillaPassword }} # fedora_test_announce_list = test@lists.fedoraproject.org # fedora_epel_announce_list = epel-package-announce@lists.fedoraproject.org # fedora_epel_test_announce_list = epel-devel@lists.fedoraproject.org +fedora_announce_list = package-announce@lists.fedoraproject.org +fedora_test_announce_list = test@lists.fedoraproject.org +fedora_epel_announce_list = epel-package-announce@lists.fedoraproject.org +fedora_epel_test_announce_list = epel-devel@lists.fedoraproject.org # Superuser groups # admin_groups = proventesters security_respons bodhiadmin sysadmin-main From 86d52721159ce58f28ab3029c05e0c9fa077e25b Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Mon, 12 Feb 2018 13:46:00 -0500 Subject: [PATCH 205/242] fix typo In 00f727dbaf53d07c9976d0693b9711141b5220fe I had a syntax error. --- roles/bodhi2/backend/files/new-updates-sync | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/bodhi2/backend/files/new-updates-sync b/roles/bodhi2/backend/files/new-updates-sync index e220d71210..862541bc71 100755 --- a/roles/bodhi2/backend/files/new-updates-sync +++ b/roles/bodhi2/backend/files/new-updates-sync @@ -56,7 +56,7 @@ RELEASES = {'f27': {'topic': 'fedora', 'repos': {'updates': { 'from': 'f26-updates', 'ostrees': [{'ref': 'fedora/26/x86_64/updates/atomic-host', - 'dest': os.path.join(ATOMICDEST, '26')}], + 'dest': os.path.join(ATOMICDEST, '26')}, # Hack around for the fact that ostree on f25 doesn't know links {'ref': 'fedora/26/x86_64/atomic-host', 'dest': os.path.join(ATOMICDEST, '26')}], From 40879a4fa8437c69e8fc87a71fb6dd84a9a7f937 Mon Sep 17 00:00:00 2001 From: Lee Keitel Date: Tue, 13 Feb 2018 02:45:45 +0000 Subject: [PATCH 206/242] Fixed rabbitmq Nagios service checks for FMN --- roles/nagios_server/files/nagios/services/fmn.cfg | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/roles/nagios_server/files/nagios/services/fmn.cfg b/roles/nagios_server/files/nagios/services/fmn.cfg index 1faf0f0aa1..5df39f288b 100644 --- a/roles/nagios_server/files/nagios/services/fmn.cfg +++ b/roles/nagios_server/files/nagios/services/fmn.cfg @@ -1,7 +1,14 @@ define service { host_name notifs-backend01.phx2.fedoraproject.org - service_description Check backend queue size - check_command check_by_nrpe!check_fmn_backend_queue + service_description Check backend irc queue size + check_command check_by_nrpe!check_fmn_backend_irc_queue + use defaulttemplate +} + +define service { + host_name notifs-backend01.phx2.fedoraproject.org + service_description Check backend email queue size + check_command check_by_nrpe!check_fmn_backend_email_queue use defaulttemplate } From de71bacf72f5d6d573a789ae6a709be3ace3cb85 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 13 Feb 2018 03:42:37 +0000 Subject: [PATCH 207/242] add in missing ,s --- roles/fedmsg/base/templates/ssl.py.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/fedmsg/base/templates/ssl.py.j2 b/roles/fedmsg/base/templates/ssl.py.j2 index d77e4456dc..d94fda8e5f 100644 --- a/roles/fedmsg/base/templates/ssl.py.j2 +++ b/roles/fedmsg/base/templates/ssl.py.j2 @@ -4,10 +4,10 @@ config = dict( ssldir="/etc/pki/fedmsg", {% if env == 'staging' %} - ca_cert_location="https://stg.fedoraproject.org/fedmsg/ca.crt" + ca_cert_location="https://stg.fedoraproject.org/fedmsg/ca.crt", crl_location="https://stg.fedoraproject.org/fedmsg/crl.pem", {% else %} - ca_cert_location="https://fedoraproject.org/fedmsg/ca.crt" + ca_cert_location="https://fedoraproject.org/fedmsg/ca.crt", crl_location="https://fedoraproject.org/fedmsg/crl.pem", {% endif %} crl_cache="/var/run/fedmsg/crl.pem", From d80ead80878c50a1e9d8408bc0a2e845c58406ef Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Fri, 9 Feb 2018 17:16:08 +0100 Subject: [PATCH 208/242] Allow httpd to write the git repo used by the source tab The Source tab is using git repository that are cloned under /var/cache/fedoracommunity/git.fp.o. This commit allow httpd to access this folder. Signed-off-by: Clement Verna --- roles/packages3/web/files/packages-httpd.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/packages3/web/files/packages-httpd.conf b/roles/packages3/web/files/packages-httpd.conf index 4d255a9bc2..ae873c3acb 100644 --- a/roles/packages3/web/files/packages-httpd.conf +++ b/roles/packages3/web/files/packages-httpd.conf @@ -61,6 +61,10 @@ WSGIScriptAlias /packages /usr/share/fedoracommunity/production/apache/fedoracom WSGIProcessGroup fedoracommunity + + Require all granted + + # If someone tries to access an icon that doesn't exist, # then send them to the default icon. This is used by From 734e9939a4cda5b93d588f279e6131024781fb67 Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Tue, 13 Feb 2018 11:10:31 +0000 Subject: [PATCH 209/242] Install Copr-dev credentials for Koschei --- inventory/group_vars/koschei-backend | 3 +++ inventory/group_vars/koschei-backend-stg | 4 ++++ roles/koschei/backend/tasks/main.yml | 3 +-- roles/koschei/backend/templates/copr-config.j2 | 5 +++++ 4 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 roles/koschei/backend/templates/copr-config.j2 diff --git a/inventory/group_vars/koschei-backend b/inventory/group_vars/koschei-backend index 914da089da..be170dd1dd 100644 --- a/inventory/group_vars/koschei-backend +++ b/inventory/group_vars/koschei-backend @@ -12,6 +12,9 @@ koschei_pgsql_hostname: db01.phx2.fedoraproject.org koschei_koji_hub: koji.fedoraproject.org koschei_kojipkgs: kojipkgs.fedoraproject.org koschei_koji_web: koji.fedoraproject.org +koschei_copr_url: http://copr-fe.cloud.fedoraproject.org +koschei_copr_login: NOT-USED-YET +koschei_copr_token: NOT-USED-YET host_group: koschei-backend diff --git a/inventory/group_vars/koschei-backend-stg b/inventory/group_vars/koschei-backend-stg index 7afe9c30ef..b12523ea24 100644 --- a/inventory/group_vars/koschei-backend-stg +++ b/inventory/group_vars/koschei-backend-stg @@ -12,6 +12,9 @@ koschei_pgsql_hostname: pgbdr.stg.phx2.fedoraproject.org koschei_koji_hub: koji.stg.fedoraproject.org koschei_kojipkgs: koji.stg.fedoraproject.org koschei_koji_web: koji.stg.fedoraproject.org +koschei_copr_url: http://copr-fe-dev.cloud.fedoraproject.org +koschei_copr_login: "{{ koschei_copr_login_stg }}" +koschei_copr_token: "{{ koschei_copr_token_stg }}" tcp_ports: [ @@ -55,6 +58,7 @@ csi_relationship: | - fedmsg hub - bastion (for mail relay) - memcached01 + - Copr development instance koschei_backend_services: - koschei-polling diff --git a/roles/koschei/backend/tasks/main.yml b/roles/koschei/backend/tasks/main.yml index 3ebf99e2f4..39d9e3cc66 100644 --- a/roles/koschei/backend/tasks/main.yml +++ b/roles/koschei/backend/tasks/main.yml @@ -74,6 +74,7 @@ with_items: - config-admin.cfg - config-backend.cfg + - copr-config notify: - restart koschei backend services tags: @@ -89,8 +90,6 @@ - config - fedmsgdconfig -# TODO install copr config, /etc/koschei/copr-config - - name: install koji client config file template: > src="koji.conf.j2" diff --git a/roles/koschei/backend/templates/copr-config.j2 b/roles/koschei/backend/templates/copr-config.j2 new file mode 100644 index 0000000000..f5489af5fa --- /dev/null +++ b/roles/koschei/backend/templates/copr-config.j2 @@ -0,0 +1,5 @@ +[copr-cli] +login = {{ koschei_copr_login }} +username = koschei +token = {{ koschei_copr_token }} +copr_url = {{ koschei_copr_url }} From d15072a4e65fc2e5962d223134fe1f85c26e2722 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Tue, 13 Feb 2018 15:47:13 +0000 Subject: [PATCH 210/242] Make pagure01 skip the proxy Signed-off-by: Patrick Uiterwijk --- roles/hosts/files/pagure01.fedoraproject.org-hosts | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/hosts/files/pagure01.fedoraproject.org-hosts b/roles/hosts/files/pagure01.fedoraproject.org-hosts index f61e44f312..aafff66e5d 100644 --- a/roles/hosts/files/pagure01.fedoraproject.org-hosts +++ b/roles/hosts/files/pagure01.fedoraproject.org-hosts @@ -1,3 +1,4 @@ 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 127.0.0.1 db-pagure db-pagure +127.0.0.1 pagure.io From 23e624fe20a7023d7599a6bf59e7b57d8659e59c Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Tue, 13 Feb 2018 16:44:06 +0000 Subject: [PATCH 211/242] Koschei-web is now running on Fedora, not EPEL --- playbooks/manual/upgrade/koschei.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/manual/upgrade/koschei.yml b/playbooks/manual/upgrade/koschei.yml index cf572309d6..44d002e789 100644 --- a/playbooks/manual/upgrade/koschei.yml +++ b/playbooks/manual/upgrade/koschei.yml @@ -62,7 +62,7 @@ - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml vars: fedora_repos: - - epel + - updates pre_tasks: - name: schedule nagios downtime nagios: action=downtime minutes=20 service=host host={{ inventory_hostname_short }}{{ env_suffix }} From 7cf8d4f4511c834ce6715d303a384b28e931ba5a Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 13 Feb 2018 18:20:35 +0000 Subject: [PATCH 212/242] add sysadmin-osbs to the various osbs groups --- inventory/group_vars/osbs | 2 +- inventory/group_vars/osbs-control | 2 +- inventory/group_vars/osbs-control-stg | 2 +- inventory/group_vars/osbs-masters | 2 +- inventory/group_vars/osbs-nodes | 2 +- inventory/group_vars/osbs-stg | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/inventory/group_vars/osbs b/inventory/group_vars/osbs index 525fb26aa9..9463129609 100644 --- a/inventory/group_vars/osbs +++ b/inventory/group_vars/osbs @@ -6,7 +6,7 @@ num_cpus: 2 tcp_ports: [ 80, 443, 8443] -fas_client_groups: sysadmin-releng,fi-apprentice,sysadmin-noc,sysadmin-veteran +fas_client_groups: sysadmin-releng,fi-apprentice,sysadmin-noc,sysadmin-veteran,sysadmin-osbs sudoers: "{{ private }}/files/sudo/00releng-sudoers" docker_cert_dir: "/etc/docker/certs.d/candidate-registry.fedoraproject.org" diff --git a/inventory/group_vars/osbs-control b/inventory/group_vars/osbs-control index 5777ead3da..75f1046a57 100644 --- a/inventory/group_vars/osbs-control +++ b/inventory/group_vars/osbs-control @@ -1,6 +1,6 @@ --- # Define resources for this group of hosts here. -fas_client_groups: sysadmin-releng,sysadmin-noc,sysadmin-veteran +fas_client_groups: sysadmin-releng,sysadmin-noc,sysadmin-veteran,sysadmin-osbs sudoers: "{{ private }}/files/sudo/00releng-sudoers" osbs_url: "osbs.fedoraproject.org" diff --git a/inventory/group_vars/osbs-control-stg b/inventory/group_vars/osbs-control-stg index b6f29da32f..62e2d68748 100644 --- a/inventory/group_vars/osbs-control-stg +++ b/inventory/group_vars/osbs-control-stg @@ -1,6 +1,6 @@ --- # Define resources for this group of hosts here. -fas_client_groups: sysadmin-releng,sysadmin-noc,sysadmin-veteran +fas_client_groups: sysadmin-releng,sysadmin-noc,sysadmin-veteran,sysadmin-osbs sudoers: "{{ private }}/files/sudo/00releng-sudoers" osbs_url: "osbs.stg.fedoraproject.org" diff --git a/inventory/group_vars/osbs-masters b/inventory/group_vars/osbs-masters index 127d511613..893e997c4a 100644 --- a/inventory/group_vars/osbs-masters +++ b/inventory/group_vars/osbs-masters @@ -6,7 +6,7 @@ num_cpus: 2 tcp_ports: [ 80, 443, 8443] -fas_client_groups: sysadmin-releng,fi-apprentice,sysadmin-noc,sysadmin-veteran +fas_client_groups: sysadmin-releng,fi-apprentice,sysadmin-noc,sysadmin-veteran,sysadmin-osbs sudoers: "{{ private }}/files/sudo/00releng-sudoers" docker_cert_dir: "/etc/docker/certs.d/candidate-registry.fedoraproject.org" diff --git a/inventory/group_vars/osbs-nodes b/inventory/group_vars/osbs-nodes index b05656f688..aad303bec1 100644 --- a/inventory/group_vars/osbs-nodes +++ b/inventory/group_vars/osbs-nodes @@ -6,7 +6,7 @@ num_cpus: 2 tcp_ports: [ 80, 443, 8443, 10250] -fas_client_groups: sysadmin-releng,fi-apprentice,sysadmin-noc,sysadmin-veteran +fas_client_groups: sysadmin-releng,fi-apprentice,sysadmin-noc,sysadmin-veteran,sysadmin-osbs sudoers: "{{ private }}/files/sudo/00releng-sudoers" docker_cert_dir: "/etc/docker/certs.d/candidate-registry.fedoraproject.org" diff --git a/inventory/group_vars/osbs-stg b/inventory/group_vars/osbs-stg index 0da2a9434f..b896143aea 100644 --- a/inventory/group_vars/osbs-stg +++ b/inventory/group_vars/osbs-stg @@ -6,7 +6,7 @@ num_cpus: 2 tcp_ports: [ 80, 443, 8443] -fas_client_groups: sysadmin-releng,fi-apprentice,sysadmin-noc,sysadmin-veteran +fas_client_groups: sysadmin-releng,fi-apprentice,sysadmin-noc,sysadmin-veteran,sysadmin-osbs sudoers: "{{ private }}/files/sudo/00releng-sudoers" docker_cert_dir: "/etc/docker/certs.d/candidate-registry.stg.fedoraproject.org" From 5885e444be110f712e0aa60365431f061f6dee0e Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Tue, 13 Feb 2018 20:32:22 +0100 Subject: [PATCH 213/242] Set the correct selinux context to allow httpd to clone git repos Signed-off-by: Clement Verna --- roles/packages3/web/tasks/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/packages3/web/tasks/main.yml b/roles/packages3/web/tasks/main.yml index 3fd9d13adc..08e0f4bc1e 100644 --- a/roles/packages3/web/tasks/main.yml +++ b/roles/packages3/web/tasks/main.yml @@ -179,6 +179,12 @@ - packages/web - selinux +- name: /var/cache/fedoracommunity/git.fedoraproject.org file contexts + sefcontext: + target: '/var/cache/fedoracommunity/git.fedoraproject.org(/.*)?' + setype: httpd_sys_rw_content_t + state: present + - name: Build the database the first time. This takes a while command: /usr/bin/fcomm-index-packages --index-db-dest=/var/cache/fedoracommunity/packages/xapian --icons-dest /var/cache/fedoracommunity/packages/icons --mdapi-url=https://apps{{env_suffix}}.fedoraproject.org/mdapi --icons-url=https://dl.fedoraproject.org/pub/alt/screenshots --tagger-url=https://apps{{env_suffix}}.fedoraproject.org/tagger creates=/var/cache/fedoracommunity/packages/xapian/search/termlist.glass tags: From 8c6165ecbb3618af12ecdf5209ced23d0b6ddb45 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Wed, 14 Feb 2018 10:04:32 +0100 Subject: [PATCH 214/242] Replace yum by dnf since packages now runs on Fedora Signed-off-by: Clement Verna --- playbooks/manual/upgrade/packages.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/playbooks/manual/upgrade/packages.yml b/playbooks/manual/upgrade/packages.yml index 7e7c2479c3..7dfdbee10c 100644 --- a/playbooks/manual/upgrade/packages.yml +++ b/playbooks/manual/upgrade/packages.yml @@ -12,13 +12,13 @@ tasks: - name: clean all metadata {%if testing%}(with infrastructure-testing on){%endif%} - command: yum clean all {%if testing%} --enablerepo=infrastructure-tags-stg {%endif%} + command: dnf clean all {%if testing%} --enablerepo=infrastructure-tags-stg {%endif%} check_mode: no - - name: yum update fedora-packages packages from main repo - yum: name="fedora-packages" state=latest + - name: dnf update fedora-packages packages from main repo + dnf: name="fedora-packages" state=latest when: not testing - - name: yum update fedora-packages packages from testing repo - yum: name="fedora-packages" state=latest enablerepo=infrastructure-tags-stg + - name: dnf update fedora-packages packages from testing repo + dnf: name="fedora-packages" state=latest enablerepo=infrastructure-tags-stg when: testing - name: verify the config and restart it From a4b3eee7e6cd3b99e0e6095099785a3305df1b05 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Wed, 14 Feb 2018 10:05:22 +0100 Subject: [PATCH 215/242] Cleanup the packages03 role Signed-off-by: Clement Verna --- roles/packages3/web/tasks/main.yml | 32 ++---------------------------- 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/roles/packages3/web/tasks/main.yml b/roles/packages3/web/tasks/main.yml index 08e0f4bc1e..6310eb8bec 100644 --- a/roles/packages3/web/tasks/main.yml +++ b/roles/packages3/web/tasks/main.yml @@ -1,33 +1,14 @@ --- # Configuration for the fedora-packages webapp - -- name: Set require selinux booleans - seboolean: name={{item}} persistent=yes state=yes - with_items: - - httpd_use_nfs - - httpd_execmem - tags: - - packages - - packages/web - - selinux - - name: install needed packages package: name={{ item }} state=present with_items: - fedora-packages - - python-psycopg2 - python-memcached tags: - packages - packages/web -- name: install python-sqlalchemy0.8 only on rhel6 - package: name=python-sqlalchemy0.8 state=present - tags: - - packages - - packages/web - when: ansible_distribution_major_version|int < 7 - - name: Create some directories file: path={{ item }} @@ -39,7 +20,6 @@ - /etc/fedoracommunity - /var/cache/fedoracommunity # the gluster role usually creates this one - /var/tmp/fedoracommunity - - /var/log/fedoracommunity tags: - packages - packages/web @@ -156,16 +136,6 @@ - hotfix when: ansible_distribution_major_version|int < 7 -# Our fedmsg updater should handle everything, no more need for cron. -#- name: Copy the indexer cronjobs -# copy: src="{{item}}" dest="/etc/cron.d/{{item}}" -# with_items: -# - cron-sync-package-index -# when: install_packages_indexer -# tags: -# - packages -# - packages/web - # Lastly, here's some selinux stuff. - name: set some selinux booleans seboolean: name={{item}} persistent=yes state=yes @@ -174,6 +144,8 @@ - httpd_can_network_memcache - httpd_can_network_connect - httpd_use_fusefs + - httpd_use_nfs + - httpd_execmem tags: - packages - packages/web From 9a8a06d28bc34874a619968699a488698a84d933 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Wed, 14 Feb 2018 13:40:39 +0100 Subject: [PATCH 216/242] Update the command use to rebuild xapian db Signed-off-by: Clement Verna --- playbooks/manual/rebuild/fedora-packages.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/manual/rebuild/fedora-packages.yml b/playbooks/manual/rebuild/fedora-packages.yml index a0cefa4b7f..8468724c58 100644 --- a/playbooks/manual/rebuild/fedora-packages.yml +++ b/playbooks/manual/rebuild/fedora-packages.yml @@ -39,7 +39,7 @@ tasks: - name: Pull in the list of packages from pkgdb. Go get a snack. (2 hours) - command: /usr/bin/fcomm-index-packages --index-db-dest /var/cache/fedoracommunity/packages/xapian --icons-dest /var/cache/fedoracommunity/packages/icons --tagger-url https://apps.stg.fedoraproject.org/tagger --pkgdb-url https://admin.stg.fedoraproject.org/pkgdb --mdapi-url https://apps.stg.fedoraproject.org/mdapi --icons-url http://download01.phx2.fedoraproject.org/pub/alt/screenshots/ + command: /usr/bin/fcomm-index-packages --index-db-dest=/var/cache/fedoracommunity/packages/xapian --icons-dest /var/cache/fedoracommunity/packages/icons --mdapi-url=https://apps.fedoraproject.org/mdapi --icons-url=https://dl.fedoraproject.org/pub/alt/screenshots --tagger-url=https://apps.fedoraproject.org/tagger creates=/var/cache/fedoracommunity/packages/xapian/search/termlist.glass async: 12000 poll: 60 when: install_packages_indexer @@ -58,7 +58,7 @@ tasks: - name: Pull in the list of packages from pkgdb. Go get a snack. (2 hours) - command: /usr/bin/fcomm-index-packages --index-db-dest /var/cache/fedoracommunity/packages/xapian --icons-dest /var/cache/fedoracommunity/packages/icons --tagger-url https://apps.fedoraproject.org/tagger --pkgdb-url https://admin.fedoraproject.org/pkgdb --mdapi-url https://apps.fedoraproject.org/mdapi --icons-url http://download01.phx2.fedoraproject.org/pub/alt/screenshots/ + command: /usr/bin/fcomm-index-packages --index-db-dest=/var/cache/fedoracommunity/packages/xapian --icons-dest /var/cache/fedoracommunity/packages/icons --mdapi-url=https://apps.fedoraproject.org/mdapi --icons-url=https://dl.fedoraproject.org/pub/alt/screenshots --tagger-url=https://apps.fedoraproject.org/tagger creates=/var/cache/fedoracommunity/packages/xapian/search/termlist.glass async: 12000 poll: 60 when: install_packages_indexer From 70343cf97732b81cec4b13b24dc520e8df7ba976 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Wed, 14 Feb 2018 14:55:34 +0100 Subject: [PATCH 217/242] Fix syntax error in playbooks/manual/rebuild/fedora-packages.yml Signed-off-by: Clement Verna --- playbooks/manual/rebuild/fedora-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/manual/rebuild/fedora-packages.yml b/playbooks/manual/rebuild/fedora-packages.yml index 8468724c58..3395e4adab 100644 --- a/playbooks/manual/rebuild/fedora-packages.yml +++ b/playbooks/manual/rebuild/fedora-packages.yml @@ -58,7 +58,7 @@ tasks: - name: Pull in the list of packages from pkgdb. Go get a snack. (2 hours) - command: /usr/bin/fcomm-index-packages --index-db-dest=/var/cache/fedoracommunity/packages/xapian --icons-dest /var/cache/fedoracommunity/packages/icons --mdapi-url=https://apps.fedoraproject.org/mdapi --icons-url=https://dl.fedoraproject.org/pub/alt/screenshots --tagger-url=https://apps.fedoraproject.org/tagger creates=/var/cache/fedoracommunity/packages/xapian/search/termlist.glass + command: /usr/bin/fcomm-index-packages --index-db-dest=/var/cache/fedoracommunity/packages/xapian --icons-dest /var/cache/fedoracommunity/packages/icons --mdapi-url=https://apps.fedoraproject.org/mdapi --icons-url=https://dl.fedoraproject.org/pub/alt/screenshots --tagger-url=https://apps.fedoraproject.org/tagger creates=/var/cache/fedoracommunity/packages/xapian/search/termlist.glass async: 12000 poll: 60 when: install_packages_indexer From b8100fc5ff8d2cb1a186648a7c5e4570bee185f2 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Wed, 14 Feb 2018 19:43:45 +0100 Subject: [PATCH 218/242] Fix packages stg bugzilla. This commit add the correct bugzilla address for packages.stg. It also cleanup the template by making use of the env_suffix variable. Signed-off-by: Clement Verna --- .../web/templates/packages-app.ini.j2 | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/roles/packages3/web/templates/packages-app.ini.j2 b/roles/packages3/web/templates/packages-app.ini.j2 index a0ad8bcfb8..cfdb5be884 100644 --- a/roles/packages3/web/templates/packages-app.ini.j2 +++ b/roles/packages3/web/templates/packages-app.ini.j2 @@ -21,24 +21,17 @@ fedoracommunity.extensions_dir = {{ pythonsitelib }}/fedoracommunity/plugins/ext fedoracommunity.script_name = /packages fedoracommunity.connector.kojihub.baseurl = https://koji{{env_suffix}}.fedoraproject.org/kojihub -fedoracommunity.connector.bugzilla.baseurl = https://bugzilla.redhat.com/xmlrpc.cgi -fedoracommunity.connector.bugzilla.cookiefile = /var/cache/fedoracommunity/bugzillacookies +fedoracommunity.connector.bodhi.baseurl = https://bodhi{{env_suffix}}.fedoraproject.org/ +fedoracommunity.connector.mdapi.baseurl = https://apps{{env_suffix}}.fedoraproject.org/mdapi +fedoracommunity.connector.tagger.baseurl = https://apps{{env_suffix}}.fedoraproject.org/tagger +fedoracommunity.connector.fas.baseurl = https://admin{{env_suffix}}.fedoraproject.org/accounts/ +fedoracommunity.connector.icons.baseurl = http://download01.phx2.fedoraproject.org/pub/alt/screenshots {% if env == "staging" %} -fedoracommunity.connector.fas.baseurl = https://admin.stg.fedoraproject.org/accounts/ -fedoracommunity.connector.bodhi.baseurl = https://bodhi.stg.fedoraproject.org/ -fedoracommunity.connector.pkgdb.baseurl = https://admin.stg.fedoraproject.org/pkgdb -fedoracommunity.connector.tagger.baseurl = https://apps.stg.fedoraproject.org/tagger -fedoracommunity.connector.mdapi.baseurl = https://apps.stg.fedoraproject.org/mdapi -fedoracommunity.connector.icons.baseurl = http://download01.phx2.fedoraproject.org/pub/alt/screenshots +fedoracommunity.connector.bugzilla.baseurl = https://partner-bugzilla.redhat.com/xmlrpc.cgi {% else %} -fedoracommunity.connector.fas.baseurl = https://admin.fedoraproject.org/accounts/ -fedoracommunity.connector.bodhi.baseurl = https://bodhi.fedoraproject.org/ -fedoracommunity.connector.pkgdb.baseurl = https://admin.fedoraproject.org/pkgdb -fedoracommunity.connector.tagger.baseurl = https://apps.fedoraproject.org/tagger -fedoracommunity.connector.mdapi.baseurl = https://apps.fedoraproject.org/mdapi -fedoracommunity.connector.icons.baseurl = http://download01.phx2.fedoraproject.org/pub/alt/screenshots +fedoracommunity.connector.bugzilla.baseurl = https://bugzilla.redhat.com/xmlrpc.cgi {% endif %} - +fedoracommunity.connector.bugzilla.cookiefile = /var/cache/fedoracommunity/bugzillacookies fedoracommunity.connector.xapian.package-search.db = /var/cache/fedoracommunity/packages/xapian/search fedoracommunity.resource_path_prefix = /packages/_res/ From b52aeaa9526ee47c1bea72b71d0bafc7c640a9d3 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 14 Feb 2018 19:43:00 +0000 Subject: [PATCH 219/242] direct retrace.stg to retrace02 per request in staging --- roles/haproxy/templates/haproxy.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/haproxy/templates/haproxy.cfg b/roles/haproxy/templates/haproxy.cfg index 4420927f87..32f7aedd9d 100644 --- a/roles/haproxy/templates/haproxy.cfg +++ b/roles/haproxy/templates/haproxy.cfg @@ -428,7 +428,7 @@ frontend retrace-frontend backend retrace-backend balance hdr(appserver) - server retrace01 retrace01:80 check inter 10s rise 1 fall 2 + server retrace02.qa.fedoraproject.org retrace02.qa.fedoraproject.org:80 check inter 10s rise 1 fall 2 {% endif %} {% if env == "staging" %} From a3aa09430c18cbc24018f7c1d787d525b77fa1b1 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Thu, 15 Feb 2018 08:42:03 +0100 Subject: [PATCH 220/242] Use stg mdapi and tagger during stg indexing Signed-off-by: Clement Verna --- playbooks/manual/rebuild/fedora-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/manual/rebuild/fedora-packages.yml b/playbooks/manual/rebuild/fedora-packages.yml index 3395e4adab..adf79496ee 100644 --- a/playbooks/manual/rebuild/fedora-packages.yml +++ b/playbooks/manual/rebuild/fedora-packages.yml @@ -39,7 +39,7 @@ tasks: - name: Pull in the list of packages from pkgdb. Go get a snack. (2 hours) - command: /usr/bin/fcomm-index-packages --index-db-dest=/var/cache/fedoracommunity/packages/xapian --icons-dest /var/cache/fedoracommunity/packages/icons --mdapi-url=https://apps.fedoraproject.org/mdapi --icons-url=https://dl.fedoraproject.org/pub/alt/screenshots --tagger-url=https://apps.fedoraproject.org/tagger creates=/var/cache/fedoracommunity/packages/xapian/search/termlist.glass + command: /usr/bin/fcomm-index-packages --index-db-dest=/var/cache/fedoracommunity/packages/xapian --icons-dest /var/cache/fedoracommunity/packages/icons --mdapi-url=https://apps.stg.fedoraproject.org/mdapi --icons-url=https://dl.fedoraproject.org/pub/alt/screenshots --tagger-url=https://apps.stg.fedoraproject.org/tagger creates=/var/cache/fedoracommunity/packages/xapian/search/termlist.glass async: 12000 poll: 60 when: install_packages_indexer From 1fb1628608d7a9c225d858e696bdb92aaaaba933 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Thu, 15 Feb 2018 10:47:20 +0100 Subject: [PATCH 221/242] Drop async command from playbook Signed-off-by: Clement Verna --- playbooks/manual/rebuild/fedora-packages.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/playbooks/manual/rebuild/fedora-packages.yml b/playbooks/manual/rebuild/fedora-packages.yml index adf79496ee..393947fde1 100644 --- a/playbooks/manual/rebuild/fedora-packages.yml +++ b/playbooks/manual/rebuild/fedora-packages.yml @@ -40,8 +40,6 @@ tasks: - name: Pull in the list of packages from pkgdb. Go get a snack. (2 hours) command: /usr/bin/fcomm-index-packages --index-db-dest=/var/cache/fedoracommunity/packages/xapian --icons-dest /var/cache/fedoracommunity/packages/icons --mdapi-url=https://apps.stg.fedoraproject.org/mdapi --icons-url=https://dl.fedoraproject.org/pub/alt/screenshots --tagger-url=https://apps.stg.fedoraproject.org/tagger creates=/var/cache/fedoracommunity/packages/xapian/search/termlist.glass - async: 12000 - poll: 60 when: install_packages_indexer - name: Rebuild that search index on the side and install it. (just prod) @@ -59,8 +57,6 @@ tasks: - name: Pull in the list of packages from pkgdb. Go get a snack. (2 hours) command: /usr/bin/fcomm-index-packages --index-db-dest=/var/cache/fedoracommunity/packages/xapian --icons-dest /var/cache/fedoracommunity/packages/icons --mdapi-url=https://apps.fedoraproject.org/mdapi --icons-url=https://dl.fedoraproject.org/pub/alt/screenshots --tagger-url=https://apps.fedoraproject.org/tagger creates=/var/cache/fedoracommunity/packages/xapian/search/termlist.glass - async: 12000 - poll: 60 when: install_packages_indexer - name: leave maintenance mode From d434a4c8d236a257b9af2be15584c8681a46f33e Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Thu, 15 Feb 2018 10:54:59 +0100 Subject: [PATCH 222/242] Make we reindex the database and fix the file recurse param Signed-off-by: Clement Verna --- playbooks/manual/rebuild/fedora-packages.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/playbooks/manual/rebuild/fedora-packages.yml b/playbooks/manual/rebuild/fedora-packages.yml index 393947fde1..e56fe26e58 100644 --- a/playbooks/manual/rebuild/fedora-packages.yml +++ b/playbooks/manual/rebuild/fedora-packages.yml @@ -39,7 +39,7 @@ tasks: - name: Pull in the list of packages from pkgdb. Go get a snack. (2 hours) - command: /usr/bin/fcomm-index-packages --index-db-dest=/var/cache/fedoracommunity/packages/xapian --icons-dest /var/cache/fedoracommunity/packages/icons --mdapi-url=https://apps.stg.fedoraproject.org/mdapi --icons-url=https://dl.fedoraproject.org/pub/alt/screenshots --tagger-url=https://apps.stg.fedoraproject.org/tagger creates=/var/cache/fedoracommunity/packages/xapian/search/termlist.glass + command: /usr/bin/fcomm-index-packages --index-db-dest=/var/cache/fedoracommunity/packages/xapian --icons-dest /var/cache/fedoracommunity/packages/icons --mdapi-url=https://apps.stg.fedoraproject.org/mdapi --icons-url=https://dl.fedoraproject.org/pub/alt/screenshots --tagger-url=https://apps.stg.fedoraproject.org/tagger when: install_packages_indexer - name: Rebuild that search index on the side and install it. (just prod) @@ -56,7 +56,7 @@ tasks: - name: Pull in the list of packages from pkgdb. Go get a snack. (2 hours) - command: /usr/bin/fcomm-index-packages --index-db-dest=/var/cache/fedoracommunity/packages/xapian --icons-dest /var/cache/fedoracommunity/packages/icons --mdapi-url=https://apps.fedoraproject.org/mdapi --icons-url=https://dl.fedoraproject.org/pub/alt/screenshots --tagger-url=https://apps.fedoraproject.org/tagger creates=/var/cache/fedoracommunity/packages/xapian/search/termlist.glass + command: /usr/bin/fcomm-index-packages --index-db-dest=/var/cache/fedoracommunity/packages/xapian --icons-dest /var/cache/fedoracommunity/packages/icons --mdapi-url=https://apps.fedoraproject.org/mdapi --icons-url=https://dl.fedoraproject.org/pub/alt/screenshots --tagger-url=https://apps.fedoraproject.org/tagger when: install_packages_indexer - name: leave maintenance mode @@ -70,8 +70,8 @@ - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml tasks: - - name: Make sure the perms are straight - file: path=/var/cache/fedoracommunity/packages/ state=directory owner=apache group=fedmsg mode="g+rw" recursive=yes + - name: Make sure the perms are straight + file: path=/var/cache/fedoracommunity/packages/ state=directory owner=apache group=fedmsg mode="g+rw" recurse=yes - name: Restart the cache worker service: name={{item}} state=started From 04aebd9fc057dc7c16eae7b9373f676e90ad5848 Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Thu, 15 Feb 2018 13:10:25 +0000 Subject: [PATCH 223/242] Staging apps in OpenShift are not frozen --- inventory/group_vars/openshift-pseudohosts-stg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 inventory/group_vars/openshift-pseudohosts-stg diff --git a/inventory/group_vars/openshift-pseudohosts-stg b/inventory/group_vars/openshift-pseudohosts-stg new file mode 100644 index 0000000000..3d8f2c30da --- /dev/null +++ b/inventory/group_vars/openshift-pseudohosts-stg @@ -0,0 +1,2 @@ +--- +freezes: false From bfb9d492b0767d9893fb57dc95c6ffd4e2eb1af0 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 15 Feb 2018 18:27:31 +0000 Subject: [PATCH 224/242] do not need to compose this anymore --- roles/releng/files/rawhide | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/releng/files/rawhide b/roles/releng/files/rawhide index d91d0d86a0..2ba2cdcfc8 100644 --- a/roles/releng/files/rawhide +++ b/roles/releng/files/rawhide @@ -2,4 +2,4 @@ MAILTO=releng-cron@lists.fedoraproject.org 15 5 * * * root TMPDIR=`mktemp -d /tmp/rawhide.XXXXXX` && cd $TMPDIR && git clone https://pagure.io/pungi-fedora.git && cd pungi-fedora && LANG=en_US.UTF-8 ./nightly.sh && sudo -u ftpsync /usr/local/bin/update-fullfiletimelist -l /pub/fedora-secondary/update-fullfiletimelist.lock -t /pub fedora fedora-secondary #15 17 * * * root TMPDIR=$(mktemp -d /tmp/rawhide-dnf.XXXXXX) && cd $TMPDIR && git clone https://pagure.io/pungi-fedora.git && cd pungi-fedora && LANG=en_US.UTF-8 ./nightly-dnf.sh -15 18 * * * root TMPDIR=$(mktemp -d /tmp/rawhide-modular.XXXXXX) && cd $TMPDIR && git clone https://pagure.io/pungi-fedora.git && cd pungi-fedora && LANG=en_US.UTF-8 ./nightly-modular.sh +#15 18 * * * root TMPDIR=$(mktemp -d /tmp/rawhide-modular.XXXXXX) && cd $TMPDIR && git clone https://pagure.io/pungi-fedora.git && cd pungi-fedora && LANG=en_US.UTF-8 ./nightly-modular.sh From e43c8b50d15ffdc0405abd79444e4c60f0290234 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 15 Feb 2018 20:16:42 +0000 Subject: [PATCH 225/242] stop bodhi-check-policies cron job from sending out No handlers could be found for logger bodhi.server emails all the time --- roles/bodhi2/backend/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/bodhi2/backend/tasks/main.yml b/roles/bodhi2/backend/tasks/main.yml index 814f9d4b8b..294a9db791 100644 --- a/roles/bodhi2/backend/tasks/main.yml +++ b/roles/bodhi2/backend/tasks/main.yml @@ -320,7 +320,7 @@ - name: bodhi-check-policies cron job. cron: name="bodhi-check-policies" hour="*/6" minute=0 user="apache" - job="/usr/bin/bodhi-check-policies > /dev/null" + job="/usr/bin/bodhi-check-policies >& /dev/null" cron_file=bodhi-check-policies-job when: (inventory_hostname.startswith('bodhi-backend01') and env == "staging") or (inventory_hostname.startswith('bodhi-backend02') and env == "production") tags: From 1b286043bfbb99148a793988b966a0004c1ff2b1 Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Thu, 15 Feb 2018 19:19:10 +0530 Subject: [PATCH 226/242] fedimg: Compress the multiple tasks into one Signed-off-by: Sayan Chowdhury --- playbooks/manual/upgrade/fedimg.yml | 60 ++++++----------------------- 1 file changed, 11 insertions(+), 49 deletions(-) diff --git a/playbooks/manual/upgrade/fedimg.yml b/playbooks/manual/upgrade/fedimg.yml index 6a171603f1..593111da45 100644 --- a/playbooks/manual/upgrade/fedimg.yml +++ b/playbooks/manual/upgrade/fedimg.yml @@ -1,12 +1,10 @@ - name: push packages out hosts: fedimg:fedimg-stg user: root - vars_files: + vars_files: - /srv/web/infra/ansible/vars/global.yml - "/srv/private/ansible/vars.yml" - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml - vars: - testing: False handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml" @@ -15,53 +13,17 @@ command: yum clean all {%if testing%} --enablerepo=infrastructure-tags-stg {%endif%} check_mode: no - name: yum update fedimg packages from main repo - yum: name="python-fedimg" state=latest - when: not testing + yum: name"{{ item }}" state=latest + with_items: + - python-fedimg + - python2-libcloud + - python2-fedfind - name: yum update fedimg packages from testing repo - yum: name="python-fedimg" state=latest enablerepo=infrastructure-tags-stg - when: testing - - name: yum update libcloud from testing repo - yum: name="python2-libcloud" state=latest enablerepo=epel-testing - when: not testing - -- name: update fedfind - hosts: fedimg:fedimg-stg - user: root - vars_files: - - /srv/web/infra/ansible/vars/global.yml - - "/srv/private/ansible/vars.yml" - - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml - vars: - testing: False - handlers: - - import_tasks: "{{ handlers_path }}/restart_services.yml" - - tasks: - - name: yum update fedfind packages from main repo - yum: name="fedfind" state=latest - when: not testing - - name: yum update fedfind packages from testing repo - yum: name="fedfind" state=latest enablerepo=infrastructure-tags-stg - when: testing - -- name: update python2-fedfind - hosts: fedimg:fedimg-stg - user: root - vars_files: - - /srv/web/infra/ansible/vars/global.yml - - "/srv/private/ansible/vars.yml" - - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml - vars: - testing: False - handlers: - - import_tasks: "{{ handlers_path }}/restart_services.yml" - - tasks: - - name: yum update fedfind packages from main repo - yum: name="python2-fedfind" state=latest - when: not testing - - name: yum update fedfind packages from testing repo - yum: name="python2-fedfind" state=latest enablerepo=infrastructure-tags-stg + yum: name="{{ item }}" state=latest enablerepo=infrastructure-tags-stg + with_items: + - python-fedimg + - python2-libcloud + - python2-fedfind when: testing - name: verify the backend and restart it From 493e28c61e2fcd216d47882038bf97653526ca84 Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Fri, 16 Feb 2018 15:12:26 +0530 Subject: [PATCH 227/242] fedimg: Add testing vars to fedimg-stg Signed-off-by: Sayan Chowdhury --- inventory/group_vars/fedimg-stg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/inventory/group_vars/fedimg-stg b/inventory/group_vars/fedimg-stg index c6e7339a61..56bbd99801 100644 --- a/inventory/group_vars/fedimg-stg +++ b/inventory/group_vars/fedimg-stg @@ -3,6 +3,9 @@ lvm_size: 20000 mem_size: 6144 num_cpus: 2 +# Use infrastructure-tags-stg repo +testing: True + # for systems that do not match the above - specify the same parameter in # the host_vars/$hostname file From dc2af9f8b04d494f0fed9ce71fda6f4d4af6f8c0 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Fri, 16 Feb 2018 11:49:33 +0100 Subject: [PATCH 228/242] Add cverna to nagios access Signed-off-by: Clement Verna --- roles/nagios_server/templates/nagios/configs/cgi.cfg.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/nagios_server/templates/nagios/configs/cgi.cfg.j2 b/roles/nagios_server/templates/nagios/configs/cgi.cfg.j2 index dfd306c239..44c583afc5 100644 --- a/roles/nagios_server/templates/nagios/configs/cgi.cfg.j2 +++ b/roles/nagios_server/templates/nagios/configs/cgi.cfg.j2 @@ -173,9 +173,9 @@ authorized_for_all_hosts=* #authorized_for_all_service_commands=nagiosadmin #authorized_for_all_host_commands=nagiosadmin -authorized_for_all_service_commands=athmane,ausil,averi,badone,codeblock,dwa,hvivani,ianweller,jspaleta,jstanley,kevin,lbazan,lmacken,maxamillion,mmahut,mmcgrath,nb,pfrields,puiterwijk,rafaelgomes,ralph,sijis,smooge,susmit,tibbs,tmz,wsterling,mdomsch,notting,pbrobinson,ricky,toshio,spot,mahrud,dwa,karsten,pingou,tflink,mizdebsk,msimacek,stickster +authorized_for_all_service_commands=athmane,ausil,averi,badone,codeblock,dwa,hvivani,ianweller,jspaleta,jstanley,kevin,lbazan,lmacken,maxamillion,mmahut,mmcgrath,nb,pfrields,puiterwijk,rafaelgomes,ralph,sijis,smooge,susmit,tibbs,tmz,wsterling,mdomsch,notting,pbrobinson,ricky,toshio,spot,mahrud,dwa,karsten,pingou,tflink,mizdebsk,msimacek,stickster,cverna -authorized_for_all_host_commands=athmane,ausil,averi,badone,codeblock,dwa,hvivani,ianweller,jspaleta,jstanley,kevin,lbazan,lmacken,maxamillion,mmahut,mmcgrath,nb,pfrields,puiterwijk,rafaelgomes,ralph,sijis,smooge,susmit,tibbs,tmz,wsterling,mdomsch,notting,pbrobinson,ricky,toshio,spot,mahrud,dwa,karsten,pingou,tflink,mizdebsk,msimacek,stickster +authorized_for_all_host_commands=athmane,ausil,averi,badone,codeblock,dwa,hvivani,ianweller,jspaleta,jstanley,kevin,lbazan,lmacken,maxamillion,mmahut,mmcgrath,nb,pfrields,puiterwijk,rafaelgomes,ralph,sijis,smooge,susmit,tibbs,tmz,wsterling,mdomsch,notting,pbrobinson,ricky,toshio,spot,mahrud,dwa,karsten,pingou,tflink,mizdebsk,msimacek,stickster,cverna From f8e520e736946c1df9381892d9d686314b5fc162 Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Fri, 16 Feb 2018 17:09:12 +0530 Subject: [PATCH 229/242] fedimg: Fix the typo in the installation task Signed-off-by: Sayan Chowdhury --- playbooks/manual/upgrade/fedimg.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/manual/upgrade/fedimg.yml b/playbooks/manual/upgrade/fedimg.yml index 593111da45..56f06c9c1a 100644 --- a/playbooks/manual/upgrade/fedimg.yml +++ b/playbooks/manual/upgrade/fedimg.yml @@ -13,7 +13,7 @@ command: yum clean all {%if testing%} --enablerepo=infrastructure-tags-stg {%endif%} check_mode: no - name: yum update fedimg packages from main repo - yum: name"{{ item }}" state=latest + yum: name="{{ item }}" state=latest with_items: - python-fedimg - python2-libcloud From 5c9458022336540aca7220f3803ace4ce5913481 Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Fri, 16 Feb 2018 17:24:32 +0530 Subject: [PATCH 230/242] fedimg: Don't pull from main repo in stg Signed-off-by: Sayan Chowdhury --- playbooks/manual/upgrade/fedimg.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/manual/upgrade/fedimg.yml b/playbooks/manual/upgrade/fedimg.yml index 56f06c9c1a..9f0cdcbf64 100644 --- a/playbooks/manual/upgrade/fedimg.yml +++ b/playbooks/manual/upgrade/fedimg.yml @@ -18,6 +18,7 @@ - python-fedimg - python2-libcloud - python2-fedfind + when: not testing - name: yum update fedimg packages from testing repo yum: name="{{ item }}" state=latest enablerepo=infrastructure-tags-stg with_items: From 55bb54f1222720f66136541f73f4982c18d5373a Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Wed, 14 Feb 2018 10:43:31 +0000 Subject: [PATCH 231/242] Allow unauthenticated nagios read access Signed-off-by: Patrick Uiterwijk --- roles/nagios_server/files/httpd/robots.txt | 2 ++ roles/nagios_server/tasks/main.yml | 5 +++++ .../nagios_server/templates/httpd/nagios.conf.j2 | 16 +++++++++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 roles/nagios_server/files/httpd/robots.txt diff --git a/roles/nagios_server/files/httpd/robots.txt b/roles/nagios_server/files/httpd/robots.txt new file mode 100644 index 0000000000..c6742d8a8c --- /dev/null +++ b/roles/nagios_server/files/httpd/robots.txt @@ -0,0 +1,2 @@ +User-Agent: * +Disallow: / diff --git a/roles/nagios_server/tasks/main.yml b/roles/nagios_server/tasks/main.yml index e357d628dc..a110f2aa09 100644 --- a/roles/nagios_server/tasks/main.yml +++ b/roles/nagios_server/tasks/main.yml @@ -274,6 +274,11 @@ - nagios_server +- name: Copy robots.txt + copy: src=httpd/robots.txt dest=/var/www/robots.txt + tags: + - nagios_server + ## Build template files # This one may go to being just a regular config file if we can make remote monitoring work - name: Template out the nagios httpd conf diff --git a/roles/nagios_server/templates/httpd/nagios.conf.j2 b/roles/nagios_server/templates/httpd/nagios.conf.j2 index c3735e898d..1287c9267a 100644 --- a/roles/nagios_server/templates/httpd/nagios.conf.j2 +++ b/roles/nagios_server/templates/httpd/nagios.conf.j2 @@ -2,6 +2,8 @@ ScriptAlias /nagios/cgi-bin/ /usr/lib64/nagios/cgi-bin/ ScriptAlias /tac.cgi /usr/lib64/nagios/cgi-bin/tac.cgi +Alias /robots.txt /var/www/robots.txt + # Set up the authorization @@ -16,13 +18,25 @@ ScriptAlias /tac.cgi /usr/lib64/nagios/cgi-bin/tac.cgi {% endif %} GssapiLocalName on AuthType GSSAPI - Require valid-user + + Require all granted + + + Require valid-user + Options ExecCGI + + Require valid-user + + + Require valid-user + + SetHandler server-status {% if vars['nagios_location'] == 'external' %} From fb1b32c6ed491e22d61dcbd9a0a1b147f89ecaaa Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Fri, 16 Feb 2018 12:16:21 +0000 Subject: [PATCH 232/242] Add initial transtats playbook et al Signed-off-by: Patrick Uiterwijk --- playbooks/openshift-apps/transtats.yml | 24 ++++++ .../transtats/files/buildconfig.yml | 30 ++++++++ .../transtats/files/deploymentconfig.yml | 74 +++++++++++++++++++ .../transtats/files/imagestream.yml | 12 +++ .../openshift-apps/transtats/files/route.yml | 16 ++++ .../transtats/files/service.yml | 14 ++++ .../transtats/templates/secret.yml | 11 +++ 7 files changed, 181 insertions(+) create mode 100644 playbooks/openshift-apps/transtats.yml create mode 100644 roles/openshift-apps/transtats/files/buildconfig.yml create mode 100644 roles/openshift-apps/transtats/files/deploymentconfig.yml create mode 100644 roles/openshift-apps/transtats/files/imagestream.yml create mode 100644 roles/openshift-apps/transtats/files/route.yml create mode 100644 roles/openshift-apps/transtats/files/service.yml create mode 100644 roles/openshift-apps/transtats/templates/secret.yml diff --git a/playbooks/openshift-apps/transtats.yml b/playbooks/openshift-apps/transtats.yml new file mode 100644 index 0000000000..ddd64fcd02 --- /dev/null +++ b/playbooks/openshift-apps/transtats.yml @@ -0,0 +1,24 @@ +- name: make the app be real + hosts: os-masters-stg + user: root + gather_facts: True + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - role: openshift/project + app: transtats + description: transtats + appowners: + - suanand + - { role: openshift/object, app: transtats, template: secret.yml } + - { role: openshift/object, app: transtats, file: imagestream.yml } + - { role: openshift/object, app: transtats, file: buildconfig.yml } + - { role: openshift/start-build, app: transtats, name: transtats-docker-build } + - { role: openshift/object, app: transtats, file: service.yml } + - { role: openshift/object, app: transtats, file: route.yml } + - { role: openshift/object, app: transtats, file: deploymentconfig.yml } + - { role: openshift/rollout, app: transtats, name: transtats-web } diff --git a/roles/openshift-apps/transtats/files/buildconfig.yml b/roles/openshift-apps/transtats/files/buildconfig.yml new file mode 100644 index 0000000000..53286af4fd --- /dev/null +++ b/roles/openshift-apps/transtats/files/buildconfig.yml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: BuildConfig +metadata: + name: "transtats-build" + labels: + environment: "transtats" +spec: + runPolicy: Serial + source: + git: + ref: master + uri: https://github.com/transtats/transtats.git + secrets: null + type: Git + strategy: + sourceStrategy: + env: + - name: PIP_INDEX_URL + - name: TS_AUTH_SYSTEM + value: fedora + - name: OIDC_RP_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: transtats-secret + key: oidc-client-secret + type: Source + output: + to: + kind: ImageStreamTag + name: transtats:latest diff --git a/roles/openshift-apps/transtats/files/deploymentconfig.yml b/roles/openshift-apps/transtats/files/deploymentconfig.yml new file mode 100644 index 0000000000..56b1db8c4d --- /dev/null +++ b/roles/openshift-apps/transtats/files/deploymentconfig.yml @@ -0,0 +1,74 @@ + +apiVersion: v1 +kind: DeploymentConfig +metadata: + name: transtats-web + labels: + app: transtats + service: web +spec: + replicas: 2 + selector: + app: transtats + service: web + template: + metadata: + labels: + app: transtats + service: web + spec: + containers: + - name: web + image: transtats + ports: + - containerPort: 8080 + env: + - name: DATABASE_SERVICE_NAME + valueFrom: + secretKeyRef: + name: transtats-secret + key: database-host + - name: DATABASE_ENGINE + value: postgresql + - name: DATABASE_NAME + value: transtats + - name: DATABASE_USER + valueFrom: + secretKeyRef: + name: transtats-secret + key: database-user + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: transtats-secret + key: database-password + - name: DJANGO_SECRET_KEY + valueFrom: + secretKeyRef: + name: transtats-secret + key: django-secret-key + readinessProbe: + timeoutSeconds: 1 + initialDelaySeconds: 5 + httpGet: + path: /health + port: 8080 + livenessProbe: + timeoutSeconds: 1 + initialDelaySeconds: 30 + httpGet: + path: /health + port: 8080 + resources: + limits: + memory: 384Mi + triggers: + - type: ImageChange + imageChangeParams: + automatic: true + containerNames: + - web + from: + kind: ImageStreamTag + name: transtats:latest + - type: ConfigChange diff --git a/roles/openshift-apps/transtats/files/imagestream.yml b/roles/openshift-apps/transtats/files/imagestream.yml new file mode 100644 index 0000000000..c2b22a5e23 --- /dev/null +++ b/roles/openshift-apps/transtats/files/imagestream.yml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ImageStream +metadata: + name: transtats +spec: + tags: + - name: latest +--- +apiVersion: v1 +kind: ImageStream +metadata: + name: transtats diff --git a/roles/openshift-apps/transtats/files/route.yml b/roles/openshift-apps/transtats/files/route.yml new file mode 100644 index 0000000000..7e01795b29 --- /dev/null +++ b/roles/openshift-apps/transtats/files/route.yml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Route +metadata: + name: transtats-web + labels: + app: transtats +spec: + #host: transtats.stg.fedoraproject.org + port: + targetPort: web + to: + kind: Service + name: transtats-web + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect diff --git a/roles/openshift-apps/transtats/files/service.yml b/roles/openshift-apps/transtats/files/service.yml new file mode 100644 index 0000000000..ca5a770c4e --- /dev/null +++ b/roles/openshift-apps/transtats/files/service.yml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: transtats-web + labels: + app: transtats +spec: + selector: + app: transtats + service: web + ports: + - name: web + port: 8080 + targetPort: 8080 diff --git a/roles/openshift-apps/transtats/templates/secret.yml b/roles/openshift-apps/transtats/templates/secret.yml new file mode 100644 index 0000000000..0d8989960a --- /dev/null +++ b/roles/openshift-apps/transtats/templates/secret.yml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "transtats-secret" + labels: + app: "transtats" +stringData: +{% if env == 'staging' %} + djang-secret-key: "{{transtats_stg_django_secret_key}}" + database-password: "{{transtats_stg_database_password}}" +{% endif %} From ec3a9f5ff46cf715fe57fb6ddf38ef89ca75a6c0 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Fri, 16 Feb 2018 13:22:15 +0100 Subject: [PATCH 233/242] Add db host and user Signed-off-by: Patrick Uiterwijk --- roles/openshift-apps/transtats/templates/secret.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/openshift-apps/transtats/templates/secret.yml b/roles/openshift-apps/transtats/templates/secret.yml index 0d8989960a..63e7f5afae 100644 --- a/roles/openshift-apps/transtats/templates/secret.yml +++ b/roles/openshift-apps/transtats/templates/secret.yml @@ -8,4 +8,6 @@ stringData: {% if env == 'staging' %} djang-secret-key: "{{transtats_stg_django_secret_key}}" database-password: "{{transtats_stg_database_password}}" + database-host: db01.stg.phx2.fedoraproject.org + database-user: transtats {% endif %} From 9247b47176464429ae85467b009bb10f1850b232 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Fri, 16 Feb 2018 12:26:01 +0000 Subject: [PATCH 234/242] Define the from image in buildconfig Signed-off-by: Patrick Uiterwijk --- roles/openshift-apps/transtats/files/buildconfig.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/openshift-apps/transtats/files/buildconfig.yml b/roles/openshift-apps/transtats/files/buildconfig.yml index 53286af4fd..25d92b416c 100644 --- a/roles/openshift-apps/transtats/files/buildconfig.yml +++ b/roles/openshift-apps/transtats/files/buildconfig.yml @@ -14,6 +14,10 @@ spec: type: Git strategy: sourceStrategy: + from: + kind: ImageStreamTag + name: python:3.5 + namespace: openshift env: - name: PIP_INDEX_URL - name: TS_AUTH_SYSTEM From 75b913c96cdc4b40df719b3ba6a39bcb0909ae77 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Fri, 16 Feb 2018 12:27:19 +0000 Subject: [PATCH 235/242] Different name Signed-off-by: Patrick Uiterwijk --- playbooks/openshift-apps/transtats.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/openshift-apps/transtats.yml b/playbooks/openshift-apps/transtats.yml index ddd64fcd02..237ca2839a 100644 --- a/playbooks/openshift-apps/transtats.yml +++ b/playbooks/openshift-apps/transtats.yml @@ -17,7 +17,7 @@ - { role: openshift/object, app: transtats, template: secret.yml } - { role: openshift/object, app: transtats, file: imagestream.yml } - { role: openshift/object, app: transtats, file: buildconfig.yml } - - { role: openshift/start-build, app: transtats, name: transtats-docker-build } + - { role: openshift/start-build, app: transtats, name: transtats-build } - { role: openshift/object, app: transtats, file: service.yml } - { role: openshift/object, app: transtats, file: route.yml } - { role: openshift/object, app: transtats, file: deploymentconfig.yml } From 52a5988a713a4e954a5d47095e882d54a9816362 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Fri, 16 Feb 2018 12:28:40 +0000 Subject: [PATCH 236/242] Add blank oidc secret for now Signed-off-by: Patrick Uiterwijk --- roles/openshift-apps/transtats/templates/secret.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/openshift-apps/transtats/templates/secret.yml b/roles/openshift-apps/transtats/templates/secret.yml index 63e7f5afae..d904b14ff7 100644 --- a/roles/openshift-apps/transtats/templates/secret.yml +++ b/roles/openshift-apps/transtats/templates/secret.yml @@ -8,6 +8,7 @@ stringData: {% if env == 'staging' %} djang-secret-key: "{{transtats_stg_django_secret_key}}" database-password: "{{transtats_stg_database_password}}" + oidc-client-secret: "" database-host: db01.stg.phx2.fedoraproject.org database-user: transtats {% endif %} From 8ca6a2453f3d64c5fee46d45b2eaf6e8489e4241 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Fri, 16 Feb 2018 12:32:40 +0000 Subject: [PATCH 237/242] Update pip due to bug in building cryptography Signed-off-by: Patrick Uiterwijk --- roles/openshift-apps/transtats/files/buildconfig.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/openshift-apps/transtats/files/buildconfig.yml b/roles/openshift-apps/transtats/files/buildconfig.yml index 25d92b416c..db01af043a 100644 --- a/roles/openshift-apps/transtats/files/buildconfig.yml +++ b/roles/openshift-apps/transtats/files/buildconfig.yml @@ -19,6 +19,8 @@ spec: name: python:3.5 namespace: openshift env: + - name: UPGRADE_PIP_TO_LATEST + value: true - name: PIP_INDEX_URL - name: TS_AUTH_SYSTEM value: fedora From 69cffcc8c9dc4e754460e0ba9fb299511e874885 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Fri, 16 Feb 2018 12:37:05 +0000 Subject: [PATCH 238/242] true is special and must be escaped Signed-off-by: Patrick Uiterwijk --- roles/openshift-apps/transtats/files/buildconfig.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/openshift-apps/transtats/files/buildconfig.yml b/roles/openshift-apps/transtats/files/buildconfig.yml index db01af043a..ae14813c24 100644 --- a/roles/openshift-apps/transtats/files/buildconfig.yml +++ b/roles/openshift-apps/transtats/files/buildconfig.yml @@ -20,7 +20,7 @@ spec: namespace: openshift env: - name: UPGRADE_PIP_TO_LATEST - value: true + value: "true" - name: PIP_INDEX_URL - name: TS_AUTH_SYSTEM value: fedora From cb56517d44e1171dcd268fe8ed5f6b4506718ca8 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Fri, 16 Feb 2018 12:51:32 +0000 Subject: [PATCH 239/242] s/djang/django/ Signed-off-by: Patrick Uiterwijk --- roles/openshift-apps/transtats/templates/secret.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/openshift-apps/transtats/templates/secret.yml b/roles/openshift-apps/transtats/templates/secret.yml index d904b14ff7..3c2c9f8e6a 100644 --- a/roles/openshift-apps/transtats/templates/secret.yml +++ b/roles/openshift-apps/transtats/templates/secret.yml @@ -6,7 +6,7 @@ metadata: app: "transtats" stringData: {% if env == 'staging' %} - djang-secret-key: "{{transtats_stg_django_secret_key}}" + django-secret-key: "{{transtats_stg_django_secret_key}}" database-password: "{{transtats_stg_database_password}}" oidc-client-secret: "" database-host: db01.stg.phx2.fedoraproject.org From 23409243947f249762ef304086ad14398699904d Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Fri, 16 Feb 2018 15:05:00 +0100 Subject: [PATCH 240/242] Create the basic structure for hubs --- inventory/group_vars/hubs-stg | 12 +++++++ .../hubs01.stg.phx2.fedoraproject.org | 12 +++++++ inventory/inventory | 4 +++ playbooks/groups/hubs.yml | 33 +++++++++++++++++++ 4 files changed, 61 insertions(+) create mode 100644 inventory/group_vars/hubs-stg create mode 100644 inventory/host_vars/hubs01.stg.phx2.fedoraproject.org create mode 100644 playbooks/groups/hubs.yml diff --git a/inventory/group_vars/hubs-stg b/inventory/group_vars/hubs-stg new file mode 100644 index 0000000000..f20d3d4134 --- /dev/null +++ b/inventory/group_vars/hubs-stg @@ -0,0 +1,12 @@ +--- +# Define resources for this group of hosts here. +lvm_size: 20000 +mem_size: 4096 +num_cpus: 2 + +# for systems that do not match the above - specify the same parameter in +# the host_vars/$hostname file + +tcp_ports: [ 80 ] + +fas_client_groups: sysadmin-noc,sysadmin-web,sysadmin-hubs,sysadmin-veteran diff --git a/inventory/host_vars/hubs01.stg.phx2.fedoraproject.org b/inventory/host_vars/hubs01.stg.phx2.fedoraproject.org new file mode 100644 index 0000000000..e242461907 --- /dev/null +++ b/inventory/host_vars/hubs01.stg.phx2.fedoraproject.org @@ -0,0 +1,12 @@ +--- +nm: 255.255.255.0 +gw: 10.5.128.254 +dns: 10.5.126.21 + +ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-27 +ks_repo: http://10.5.126.23/pub/fedora/linux/releases/27/Server/x86_64/os/ + +volgroup: /dev/vg_guests +eth0_ip: 10.5.128.190 +vmhost: virthost05.phx2.fedoraproject.org +datacenter: phx2 diff --git a/inventory/inventory b/inventory/inventory index 696e28728d..b289429fc7 100644 --- a/inventory/inventory +++ b/inventory/inventory @@ -708,6 +708,9 @@ smtp-mm-ib01.fedoraproject.org smtp-mm-osuosl01.fedoraproject.org smtp-mm-tummy01.fedoraproject.org +[hubs-stg] +hubs01.stg.phx2.fedoraproject.org + [spare] # # All staging hosts should be in this group too. @@ -761,6 +764,7 @@ freshmaker-frontend01.stg.phx2.fedoraproject.org freshmaker-backend01.stg.phx2.fedoraproject.org github2fedmsg01.stg.phx2.fedoraproject.org hotness01.stg.phx2.fedoraproject.org +hubs01.stg.phx2.fedoraproject.org kerneltest01.stg.phx2.fedoraproject.org koji01.stg.phx2.fedoraproject.org koschei-backend01.stg.phx2.fedoraproject.org diff --git a/playbooks/groups/hubs.yml b/playbooks/groups/hubs.yml new file mode 100644 index 0000000000..5838e8545e --- /dev/null +++ b/playbooks/groups/hubs.yml @@ -0,0 +1,33 @@ +# create the hubs server +# NOTE: should be used with --limit most of the time +- import_playbook: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=hubs-stg" + +- name: make the box be real + hosts: hubs-stg + user: root + gather_facts: True + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - base + - rkhunter + - nagios_client + - hosts + - fas_client + - collectd/base + - sudo + - { role: openvpn/client, + when: env != "staging" } + - mod_wsgi + + tasks: + - import_tasks: "{{ tasks_path }}/yumrepos.yml" + - import_tasks: "{{ tasks_path }}/2fa_client.yml" + - import_tasks: "{{ tasks_path }}/motd.yml" + + handlers: + - import_tasks: "{{ handlers_path }}/restart_services.yml" From ef740c2147f591f39124d3774d53359de3a39cf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20P=C3=A1ral?= Date: Fri, 16 Feb 2018 16:55:10 +0100 Subject: [PATCH 241/242] fix typo in taskotron DBs and testdays configs --- roles/taskotron/execdb/templates/settings.py.j2 | 2 +- roles/taskotron/resultsdb-backend/templates/settings.py.j2 | 2 +- roles/taskotron/resultsdb-frontend/templates/settings.py.j2 | 2 +- roles/testdays/templates/settings.py.j2 | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/taskotron/execdb/templates/settings.py.j2 b/roles/taskotron/execdb/templates/settings.py.j2 index eeb44775c3..688dc451b5 100644 --- a/roles/taskotron/execdb/templates/settings.py.j2 +++ b/roles/taskotron/execdb/templates/settings.py.j2 @@ -2,7 +2,7 @@ SECRET_KEY = '{{ execdb_secret_key }}' SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://{{ execdb_db_user }}:{{ execdb_db_password }}@{{ execdb_db_host }}:{{ execdb_db_port }}/{{ execdb_db_name }}' FILE_LOGGING = False -LOGFILR = '/var/log/execdb/execdb.log' +LOGFILE = '/var/log/execdb/execdb.log' SYSLOG_LOGGING = False STREAM_LOGGING = True diff --git a/roles/taskotron/resultsdb-backend/templates/settings.py.j2 b/roles/taskotron/resultsdb-backend/templates/settings.py.j2 index daa6e02a98..8da290f859 100644 --- a/roles/taskotron/resultsdb-backend/templates/settings.py.j2 +++ b/roles/taskotron/resultsdb-backend/templates/settings.py.j2 @@ -1,7 +1,7 @@ SECRET_KEY = '{{ resultsdb_secret_key }}' SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://{{ resultsdb_db_user }}:{{ resultsdb_db_password }}@{{ resultsdb_db_host }}:{{ resultsdb_db_port }}/{{ resultsdb_db_name }}' FILE_LOGGING = False -LOGFILR = '/var/log/resultsdb/resultsdb.log' +LOGFILE = '/var/log/resultsdb/resultsdb.log' SYSLOG_LOGGING = False STREAM_LOGGING = True diff --git a/roles/taskotron/resultsdb-frontend/templates/settings.py.j2 b/roles/taskotron/resultsdb-frontend/templates/settings.py.j2 index 6c6042e071..29b6df89cc 100644 --- a/roles/taskotron/resultsdb-frontend/templates/settings.py.j2 +++ b/roles/taskotron/resultsdb-frontend/templates/settings.py.j2 @@ -4,6 +4,6 @@ RDB_URL = 'http://127.0.0.1/{{ resultsdb_endpoint }}/api/v2.0' SECRET_KEY = '{{ resultsdb_frontend_secret_key }}' FILE_LOGGING = False -LOGFILR = '/var/log/resultsdb_frontend/resultsdb_frontend.log' +LOGFILE = '/var/log/resultsdb_frontend/resultsdb_frontend.log' SYSLOG_LOGGING = False STREAM_LOGGING = True diff --git a/roles/testdays/templates/settings.py.j2 b/roles/testdays/templates/settings.py.j2 index 6f3b2354de..fd7d2cefe6 100644 --- a/roles/testdays/templates/settings.py.j2 +++ b/roles/testdays/templates/settings.py.j2 @@ -6,6 +6,6 @@ SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://{{ testdays_db_user }}:{{ testd SHOW_DB_URI = False PRODUCTION = True FILE_LOGGING = False -LOGFILR = '/var/log/testdays/testdays.log' +LOGFILE = '/var/log/testdays/testdays.log' SYSLOG_LOGGING = False STREAM_LOGGING = True From 29dbecbd014954ed59ec163f48d903a15149badd Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Fri, 16 Feb 2018 18:27:39 +0000 Subject: [PATCH 242/242] disable and enable require the backend name --- .../files/restart-mirrorlist-containers | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/mirrormanager/mirrorlist_proxy/files/restart-mirrorlist-containers b/roles/mirrormanager/mirrorlist_proxy/files/restart-mirrorlist-containers index fdf5385412..5ef79ccf1f 100644 --- a/roles/mirrormanager/mirrorlist_proxy/files/restart-mirrorlist-containers +++ b/roles/mirrormanager/mirrorlist_proxy/files/restart-mirrorlist-containers @@ -28,7 +28,7 @@ fi # start mirrorlist2 (old pkl and see that it's processing ok) systemctl start mirrorlist2 -echo "enable server mirror-lists/mirrorlist-local2" | nc -U /var/run/haproxy-admin >& /dev/null +echo "enable server mirror-lists-backend/mirrorlist-local2" | nc -U /var/run/haproxy-admin >& /dev/null sleep 5 curl -q -H mirrors.fedoraproject.org "http://localhost:18082/metalink?repo=rawhide&arch=x86_64" -o/dev/null -s -f --retry 50 --retry-delay 10 --retry-connrefused --retry-max-time 180 @@ -38,13 +38,13 @@ if [ $? != 0 ]; then fi # Drain mirrorlist1. This is safe since we assured that local2 is serving -echo "disable server mirror-lists/mirrorlist-local1" | nc -U /var/run/haproxy-admin >& /dev/null +echo "disable server mirror-lists-backend/mirrorlist-local1" | nc -U /var/run/haproxy-admin >& /dev/null sleep 1 # restart mirrorlist1 (new pkl and make sure it's processing ok) systemctl restart mirrorlist1 sleep 1 -echo "enable server mirror-lists/mirrorlist-local1" | nc -U /var/run/haproxy-admin >& /dev/null +echo "enable server mirror-lists-backend/mirrorlist-local1" | nc -U /var/run/haproxy-admin >& /dev/null sleep 5 curl -q -H mirrors.fedoraproject.org "http://localhost:18081/metalink?repo=rawhide&arch=x86_64" -o/dev/null -s -f --retry 50 --retry-delay 10 --retry-connrefused --retry-max-time 180 @@ -57,7 +57,7 @@ fi cp -a /srv/mirrorlist/data/mirrorlist1/* /srv/mirrorlist/data/mirrorlist2/ # Drain mirrorlist2 -echo "disable server mirror-lists/mirrorlist-local2" | nc -U /var/run/haproxy-admin >& /dev/null +echo "disable server mirror-lists-backend/mirrorlist-local2" | nc -U /var/run/haproxy-admin >& /dev/null sleep 1 # stop mirrorlist2 @@ -65,4 +65,4 @@ systemctl stop mirrorlist2 # Now that it's stopped, we can re-enable it. That makes sure that if anything went wrong, we # still have it enabled -echo "enable server mirror-lists/mirrorlist-local2" | nc -U /var/run/haproxy-admin >& /dev/null +echo "enable server mirror-lists-backend/mirrorlist-local2" | nc -U /var/run/haproxy-admin >& /dev/null