From 9a20193464e2745a3b1756386088442997325c1e Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Thu, 5 Apr 2018 17:20:35 +0200 Subject: [PATCH 01/43] Give sudo before collectd Signed-off-by: Clement Verna --- playbooks/groups/osbs-cluster.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/groups/osbs-cluster.yml b/playbooks/groups/osbs-cluster.yml index 19180bd823..9ea4f06857 100644 --- a/playbooks/groups/osbs-cluster.yml +++ b/playbooks/groups/osbs-cluster.yml @@ -23,9 +23,9 @@ - nagios_client - hosts - fas_client + - sudo - collectd/base - rsyncd - - sudo tasks: - import_tasks: "{{ tasks_path }}/2fa_client.yml" From c9c32b806da8583efd3c2995355b3ebaf8e8a2e0 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Thu, 5 Apr 2018 17:29:14 +0200 Subject: [PATCH 02/43] Make osbs-master01.stg an f27 box Signed-off-by: Clement Verna --- inventory/host_vars/osbs-master01.stg.phx2.fedoraproject.org | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inventory/host_vars/osbs-master01.stg.phx2.fedoraproject.org b/inventory/host_vars/osbs-master01.stg.phx2.fedoraproject.org index 61a1be5850..b4d7894be5 100644 --- a/inventory/host_vars/osbs-master01.stg.phx2.fedoraproject.org +++ b/inventory/host_vars/osbs-master01.stg.phx2.fedoraproject.org @@ -2,8 +2,8 @@ nm: 255.255.255.0 gw: 10.5.128.254 dns: 10.5.126.21 -ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-26 -ks_repo: http://10.5.126.23/pub/fedora/linux/releases/26/Server/x86_64/os/ +ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-27 +ks_repo: http://10.5.126.23/pub/fedora/linux/releases/27/Server/x86_64/os/ volgroup: /dev/vg_guests eth0_ip: 10.5.128.161 vmhost: virthost20.phx2.fedoraproject.org From c7133fefadf9ee5eb61a504146d614d7b057dc29 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 5 Apr 2018 19:02:04 +0200 Subject: [PATCH 03/43] Let's disable the cron on staging dist-git until they are fixed Signed-off-by: Pierre-Yves Chibon --- roles/distgit/pagure/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/distgit/pagure/tasks/main.yml b/roles/distgit/pagure/tasks/main.yml index bb5935cf82..da7867b711 100644 --- a/roles/distgit/pagure/tasks/main.yml +++ b/roles/distgit/pagure/tasks/main.yml @@ -267,6 +267,7 @@ - pagure - name: Configure cron job for a hourly pagure_poc + when: env != 'staging' cron: name: pagure-poc user: root @@ -278,6 +279,7 @@ - pagure - name: Configure cron job for a hourly pagure_bz + when: env != 'staging' cron: name: pagure-poc user: root @@ -289,6 +291,7 @@ - pagure - name: Configure cron job for a hourly pagure_owner_alias + when: env != 'staging' cron: name: pagure-poc user: root From caf130a42273e00550aff99ea2231906074aaa34 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 5 Apr 2018 18:08:24 +0000 Subject: [PATCH 04/43] lets see if the problem here is the name keyword --- playbooks/groups/ask.yml | 8 ++++---- playbooks/groups/docker-registry.yml | 4 ++-- playbooks/groups/nuancier.yml | 8 ++++---- roles/gluster/client/tasks/main.yml | 4 ++-- roles/gluster/server/tasks/main.yml | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/playbooks/groups/ask.yml b/playbooks/groups/ask.yml index d5cdd3c5e1..79c332a678 100644 --- a/playbooks/groups/ask.yml +++ b/playbooks/groups/ask.yml @@ -45,7 +45,7 @@ roles: - role: gluster/server - name: gluster + glusterservername: gluster username: "{{ ask_gluster_username }}" password: "{{ ask_gluster_password }}" owner: root @@ -53,7 +53,7 @@ datadir: /srv/glusterfs/ask-stg - role: gluster/client - name: gluster + glusterservername: gluster servers: - ask01.stg.phx2.fedoraproject.org username: "{{ ask_gluster_username }}" @@ -74,7 +74,7 @@ roles: - role: gluster/server - name: gluster + glusterservername: gluster username: "{{ ask_gluster_username }}" password: "{{ ask_gluster_password }}" owner: root @@ -82,7 +82,7 @@ datadir: /srv/glusterfs/ask - role: gluster/client - name: gluster + glusterservername: gluster servers: - ask01.phx2.fedoraproject.org - ask02.phx2.fedoraproject.org diff --git a/playbooks/groups/docker-registry.yml b/playbooks/groups/docker-registry.yml index 5e227d3ab7..c4f60ae24e 100644 --- a/playbooks/groups/docker-registry.yml +++ b/playbooks/groups/docker-registry.yml @@ -67,7 +67,7 @@ roles: - role: gluster/server - name: gluster + glusterservername: gluster username: "{{ registry_gluster_username_prod }}" password: "{{ registry_gluster_password_prod }}" owner: root @@ -75,7 +75,7 @@ datadir: /srv/glusterfs/registry - role: gluster/client - name: gluster + glusterservername: gluster servers: - docker-registry02.phx2.fedoraproject.org - docker-registry03.phx2.fedoraproject.org diff --git a/playbooks/groups/nuancier.yml b/playbooks/groups/nuancier.yml index acf8c66a0a..32e5a2e0f7 100644 --- a/playbooks/groups/nuancier.yml +++ b/playbooks/groups/nuancier.yml @@ -63,7 +63,7 @@ roles: - role: gluster/server - name: gluster + glusterservername: gluster username: "{{ nuancier_gluster_username }}" password: "{{ nuancier_gluster_password }}" owner: root @@ -71,7 +71,7 @@ datadir: /srv/glusterfs/nuancier-stg - role: gluster/client - name: gluster + glusterservername: gluster servers: - nuancier01.stg.phx2.fedoraproject.org - nuancier02.stg.phx2.fedoraproject.org @@ -93,7 +93,7 @@ roles: - role: gluster/server - name: gluster + glusterservername: gluster username: "{{ nuancier_gluster_username }}" password: "{{ nuancier_gluster_password }}" owner: root @@ -101,7 +101,7 @@ datadir: /srv/glusterfs/nuancier - role: gluster/client - name: gluster + glusterservername: gluster servers: - nuancier01.phx2.fedoraproject.org - nuancier02.phx2.fedoraproject.org diff --git a/roles/gluster/client/tasks/main.yml b/roles/gluster/client/tasks/main.yml index b596f19021..ff2a5d94d4 100644 --- a/roles/gluster/client/tasks/main.yml +++ b/roles/gluster/client/tasks/main.yml @@ -19,14 +19,14 @@ - name: copy over the client config template: src: client.config - dest: /etc/glusterfs/glusterfs.{{name}}.vol + dest: /etc/glusterfs/glusterfs.{{glusterservername}}.vol mode: 0640 #notify: #- remount? no idea... - name: mount it up mount: - src: /etc/glusterfs/glusterfs.{{name}}.vol + src: /etc/glusterfs/glusterfs.{{glusterservername}}.vol state: mounted fstype: glusterfs name: "{{mountdir}}" diff --git a/roles/gluster/server/tasks/main.yml b/roles/gluster/server/tasks/main.yml index a530aa6175..47b9b85a50 100644 --- a/roles/gluster/server/tasks/main.yml +++ b/roles/gluster/server/tasks/main.yml @@ -13,7 +13,7 @@ - restart glusterd - name: make the datapath - file: dest={{ datadir }}/{{ name }} state=directory + file: dest={{ datadir }}/{{ glusterservername }} state=directory notify: - restart glusterd From c12a30acceeb352b8dd75fb05e2bc8f99a446a97 Mon Sep 17 00:00:00 2001 From: Jeremy Cline Date: Thu, 5 Apr 2018 18:19:49 +0000 Subject: [PATCH 05/43] Add the release-monitoring stage db to the config Signed-off-by: Jeremy Cline --- roles/openshift-apps/release-monitoring/templates/configmap.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/openshift-apps/release-monitoring/templates/configmap.yml b/roles/openshift-apps/release-monitoring/templates/configmap.yml index 753aa06f03..2392fe8294 100644 --- a/roles/openshift-apps/release-monitoring/templates/configmap.yml +++ b/roles/openshift-apps/release-monitoring/templates/configmap.yml @@ -20,7 +20,7 @@ data: permanent_session_lifetime = 3600 {% if env == 'staging' %} - db_url = "sqlite:////var/tmp/anitya-dev.sqlite" + db_url = "postgresql://{{ anitya_stg_db_user }}:{{ anitya_stg_db_pass }}@{{ anitya_stg_db_host }}/{{ anitya_stg_db_name }}" {% else %} db_url = "postgresql://{{ anitya_db_user }}:{{ anitya_db_pass }}@{{ anitya_db_host }}/{{ anitya_db_name }}" {% endif %} From 81edf503cd0c75dc80653a3484105e794b2a9918 Mon Sep 17 00:00:00 2001 From: Jeremy Cline Date: Thu, 5 Apr 2018 18:23:14 +0000 Subject: [PATCH 06/43] Install python-social-auth for release-monitoring Signed-off-by: Jeremy Cline --- .../release-monitoring/files/buildconfig.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/roles/openshift-apps/release-monitoring/files/buildconfig.yml b/roles/openshift-apps/release-monitoring/files/buildconfig.yml index 436c5d0a2e..e9b007e31e 100644 --- a/roles/openshift-apps/release-monitoring/files/buildconfig.yml +++ b/roles/openshift-apps/release-monitoring/files/buildconfig.yml @@ -15,7 +15,7 @@ items: name="release-monitoring-web" \ vendor="Fedora Infrastructure" \ license="MIT" - RUN dnf install -y \ + RUN dnf install -y --enable-repo=updates-testing \ git \ python3-blinker \ python3-dateutil \ @@ -30,18 +30,17 @@ items: python3-pip \ python3-psycopg2 \ python3-setuptools \ + python3-social-auth-app-flask-sqlalchemy \ python3-straight-plugin \ python3-sqlalchemy \ python3-wtforms && \ dnf autoremove -y && \ dnf clean all -y - RUN pip-3 install social-auth-app-flask social-auth-app-flask-sqlalchemy RUN pip-3 install git+https://github.com/release-monitoring/anitya.git ENV USER=anitya EXPOSE 8080 EXPOSE 9940 - ENTRYPOINT python3 -c "from anitya.config import config; from anitya.lib import utilities; utilities.init('sqlite:////var/tmp/anitya-dev.sqlite', None, debug=True, create=True)" \ - && python3-gunicorn --bind 0.0.0.0:8080 --access-logfile=- anitya.wsgi:application + ENTRYPOINT python3-gunicorn --bind 0.0.0.0:8080 --access-logfile=- anitya.wsgi:application type: Dockerfile strategy: type: Docker From 0dbe0cf95d24fe42bf0084367b527618b1873c0d Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 5 Apr 2018 18:29:31 +0000 Subject: [PATCH 07/43] also change vars here --- roles/gluster/client/templates/client.config | 2 +- roles/gluster/server/templates/server.config | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/gluster/client/templates/client.config b/roles/gluster/client/templates/client.config index ffd059e6e3..f5614a3eec 100644 --- a/roles/gluster/client/templates/client.config +++ b/roles/gluster/client/templates/client.config @@ -1,4 +1,4 @@ -# Config for {{ name }} +# Config for {{ glusterservername }} # Generated by ansible {% for server in servers %} diff --git a/roles/gluster/server/templates/server.config b/roles/gluster/server/templates/server.config index 787494c235..c74091be55 100644 --- a/roles/gluster/server/templates/server.config +++ b/roles/gluster/server/templates/server.config @@ -1,9 +1,9 @@ -# Config for {{ name }} +# Config for {{ glusterservername }} # Generated by ansible volume posix type storage/posix - option directory {{ datadir }}/{{ name }} + option directory {{ datadir }}/{{ glusterservername }} end-volume volume locks @@ -22,8 +22,8 @@ volume server-tcp type protocol/server subvolumes iothreads option transport-type tcp - option auth.login.iothreads.allow {{ username }} - option auth.login.{{ username }}.password {{ password }} + option auth.login.iothreads.allow {{ userglusterservername }} + option auth.login.{{ userglusterservername }}.password {{ password }} option transport.socket.listen-port 6996 option transport.socket.nodelay on end-volume From 75d7bbc738416fa5a70698dbde00f33ce5f7f1f7 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 5 Apr 2018 18:45:27 +0000 Subject: [PATCH 08/43] bad global replace --- roles/gluster/server/templates/server.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/gluster/server/templates/server.config b/roles/gluster/server/templates/server.config index c74091be55..7b74b56af4 100644 --- a/roles/gluster/server/templates/server.config +++ b/roles/gluster/server/templates/server.config @@ -22,8 +22,8 @@ volume server-tcp type protocol/server subvolumes iothreads option transport-type tcp - option auth.login.iothreads.allow {{ userglusterservername }} - option auth.login.{{ userglusterservername }}.password {{ password }} + option auth.login.iothreads.allow {{ username }} + option auth.login.{{ username }}.password {{ password }} option transport.socket.listen-port 6996 option transport.socket.nodelay on end-volume From 066c97690efd915aa3e6977ab85ee83cd9a032a7 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 5 Apr 2018 19:58:10 +0000 Subject: [PATCH 09/43] tell ansible these are vars --- playbooks/openshift-apps/greenwave.yml | 16 ++++++++-------- .../openshift-apps/librariesio2fedmsg.yml | 12 ++++++------ playbooks/openshift-apps/modernpaste.yml | 18 +++++++++--------- .../openshift-apps/release-monitoring.yml | 16 ++++++++-------- playbooks/openshift-apps/transtats.yml | 16 ++++++++-------- playbooks/openshift-apps/waiverdb.yml | 16 ++++++++-------- 6 files changed, 47 insertions(+), 47 deletions(-) diff --git a/playbooks/openshift-apps/greenwave.yml b/playbooks/openshift-apps/greenwave.yml index 2c828d3db0..7539272de7 100644 --- a/playbooks/openshift-apps/greenwave.yml +++ b/playbooks/openshift-apps/greenwave.yml @@ -42,11 +42,11 @@ key: fedmsg-greenwave.crt privatefile: fedmsg-certs/keys/greenwave-greenwave-web-greenwave.app.os.fedoraproject.org.crt when: env != "staging" - - { role: openshift/object, app: greenwave, file: imagestream.yml } - - { role: openshift/object, app: greenwave, template: buildconfig.yml } - - { role: openshift/start-build, app: greenwave, name: greenwave-docker-build } - - { role: openshift/object, app: greenwave, template: configmap.yml } - - { role: openshift/object, app: greenwave, file: service.yml } - - { role: openshift/object, app: greenwave, file: route.yml } - - { role: openshift/object, app: greenwave, file: deploymentconfig.yml } - - { role: openshift/rollout, app: greenwave, name: greenwave-web } + - { role: openshift/object, vars: {app: greenwave, file: imagestream.yml }} + - { role: openshift/object, vars: {app: greenwave, template: buildconfig.yml }} + - { role: openshift/start-build, vars: {app: greenwave, name: greenwave-docker-build }} + - { role: openshift/object, vars: {app: greenwave, template: configmap.yml }} + - { role: openshift/object, vars: {app: greenwave, file: service.yml }} + - { role: openshift/object, vars: {app: greenwave, file: route.yml }} + - { role: openshift/object, vars: {app: greenwave, file: deploymentconfig.yml }} + - { role: openshift/rollout, vars: {app: greenwave, name: greenwave-web }} diff --git a/playbooks/openshift-apps/librariesio2fedmsg.yml b/playbooks/openshift-apps/librariesio2fedmsg.yml index fedac41ada..667199150f 100644 --- a/playbooks/openshift-apps/librariesio2fedmsg.yml +++ b/playbooks/openshift-apps/librariesio2fedmsg.yml @@ -24,9 +24,9 @@ secret_name: librariesio2fedmsg-fedmsg-crt key: fedmsg-librariesio2fedmsg.crt privatefile: fedmsg-certs/keys/librariesio2fedmsg-librariesio2fedmsg.app.os.fedoraproject.org.crt - - { role: openshift/object, app: librariesio2fedmsg, file: imagestream.yml } - - { role: openshift/object, app: librariesio2fedmsg, file: buildconfig.yml } - - { role: openshift/start-build, app: librariesio2fedmsg, name: sse2fedmsg-docker-build } - - { role: openshift/object, app: librariesio2fedmsg, template: configmap.yml } - - { role: openshift/object, app: librariesio2fedmsg, file: deploymentconfig.yml } - - { role: openshift/rollout, app: librariesio2fedmsg, name: librariesio2fedmsg } + - { role: openshift/object, vars: {app: librariesio2fedmsg, file: imagestream.yml }} + - { role: openshift/object, vars: {app: librariesio2fedmsg, file: buildconfig.yml }} + - { role: openshift/start-build, vars: {app: librariesio2fedmsg, name: sse2fedmsg-docker-build }} + - { role: openshift/object, vars: {app: librariesio2fedmsg, template: configmap.yml }} + - { role: openshift/object, vars: {app: librariesio2fedmsg, file: deploymentconfig.yml }} + - { role: openshift/rollout, vars: {app: librariesio2fedmsg, name: librariesio2fedmsg }} diff --git a/playbooks/openshift-apps/modernpaste.yml b/playbooks/openshift-apps/modernpaste.yml index 260281eb2a..69d46755ae 100644 --- a/playbooks/openshift-apps/modernpaste.yml +++ b/playbooks/openshift-apps/modernpaste.yml @@ -14,12 +14,12 @@ description: modernpaste appowners: - codeblock - - { role: openshift/object, app: modernpaste, file: imagestream.yml } - - { role: openshift/object, app: modernpaste, template: secret.yml } - - { role: openshift/object, app: modernpaste, file: buildconfig.yml } - - { role: openshift/start-build, app: modernpaste, name: modernpaste-docker-build } - - { role: openshift/object, app: modernpaste, template: configmap.yml } - - { role: openshift/object, app: modernpaste, file: service.yml } - - { role: openshift/object, app: modernpaste, file: route.yml } - - { role: openshift/object, app: modernpaste, file: deploymentconfig.yml } - - { role: openshift/rollout, app: modernpaste, name: modernpaste-web } + - { role: openshift/object, vars: {app: modernpaste, file: imagestream.yml }} + - { role: openshift/object, vars: {app: modernpaste, template: secret.yml }} + - { role: openshift/object, vars: {app: modernpaste, file: buildconfig.yml }} + - { role: openshift/start-build, vars: {app: modernpaste, name: modernpaste-docker-build }} + - { role: openshift/object, vars: {app: modernpaste, template: configmap.yml }} + - { role: openshift/object, vars: {app: modernpaste, file: service.yml }} + - { role: openshift/object, vars: {app: modernpaste, file: route.yml }} + - { role: openshift/object, vars: {app: modernpaste, file: deploymentconfig.yml }} + - { role: openshift/rollout, vars: {app: modernpaste, name: modernpaste-web }} diff --git a/playbooks/openshift-apps/release-monitoring.yml b/playbooks/openshift-apps/release-monitoring.yml index 7f4e866352..c7d591792a 100644 --- a/playbooks/openshift-apps/release-monitoring.yml +++ b/playbooks/openshift-apps/release-monitoring.yml @@ -14,11 +14,11 @@ description: release-monitoring appowners: - jcline - - { role: openshift/object, app: release-monitoring, file: imagestream.yml } - - { role: openshift/object, app: release-monitoring, file: buildconfig.yml } - - { role: openshift/start-build, app: release-monitoring, name: release-monitoring-web-build } - - { role: openshift/object, app: release-monitoring, template: configmap.yml } - - { role: openshift/object, app: release-monitoring, file: service.yml } - - { role: openshift/object, app: release-monitoring, file: route.yml } - - { role: openshift/object, app: release-monitoring, file: deploymentconfig.yml } - - { role: openshift/rollout, app: release-monitoring, name: release-monitoring-web } + - { role: openshift/object, vars: {app: release-monitoring, file: imagestream.yml }} + - { role: openshift/object, vars: {app: release-monitoring, file: buildconfig.yml }} + - { role: openshift/start-build, vars: {app: release-monitoring, name: release-monitoring-web-build }} + - { role: openshift/object, vars: {app: release-monitoring, template: configmap.yml }} + - { role: openshift/object, vars: {app: release-monitoring, file: service.yml }} + - { role: openshift/object, vars: {app: release-monitoring, file: route.yml }} + - { role: openshift/object, vars: {app: release-monitoring, file: deploymentconfig.yml }} + - { role: openshift/rollout, vars: {app: release-monitoring, name: release-monitoring-web }} diff --git a/playbooks/openshift-apps/transtats.yml b/playbooks/openshift-apps/transtats.yml index 237ca2839a..f18bc14384 100644 --- a/playbooks/openshift-apps/transtats.yml +++ b/playbooks/openshift-apps/transtats.yml @@ -14,11 +14,11 @@ description: transtats appowners: - suanand - - { role: openshift/object, app: transtats, template: secret.yml } - - { role: openshift/object, app: transtats, file: imagestream.yml } - - { role: openshift/object, app: transtats, file: buildconfig.yml } - - { role: openshift/start-build, app: transtats, name: transtats-build } - - { role: openshift/object, app: transtats, file: service.yml } - - { role: openshift/object, app: transtats, file: route.yml } - - { role: openshift/object, app: transtats, file: deploymentconfig.yml } - - { role: openshift/rollout, app: transtats, name: transtats-web } + - { role: openshift/object, vars: {app: transtats, template: secret.yml }} + - { role: openshift/object, vars: {app: transtats, file: imagestream.yml }} + - { role: openshift/object, vars: {app: transtats, file: buildconfig.yml }} + - { role: openshift/start-build, vars: {app: transtats, name: transtats-build }} + - { role: openshift/object, vars: {app: transtats, file: service.yml }} + - { role: openshift/object, vars: {app: transtats, file: route.yml }} + - { role: openshift/object, vars: {app: transtats, file: deploymentconfig.yml }} + - { role: openshift/rollout, vars: {app: transtats, name: transtats-web }} diff --git a/playbooks/openshift-apps/waiverdb.yml b/playbooks/openshift-apps/waiverdb.yml index cc8ff823f5..2f7af5f085 100644 --- a/playbooks/openshift-apps/waiverdb.yml +++ b/playbooks/openshift-apps/waiverdb.yml @@ -49,11 +49,11 @@ key: fedmsg-waiverdb.crt privatefile: fedmsg-certs/keys/waiverdb-waiverdb-web-waiverdb.app.os.fedoraproject.org.crt when: env != "staging" - - { role: openshift/object, app: waiverdb, file: imagestream.yml } - - { role: openshift/object, app: waiverdb, file: buildconfig.yml } - - { role: openshift/start-build, app: waiverdb, name: waiverdb-docker-build } - - { role: openshift/object, app: waiverdb, template: configmap.yml } - - { role: openshift/object, app: waiverdb, file: service.yml } - - { role: openshift/object, app: waiverdb, file: route.yml } - - { role: openshift/object, app: waiverdb, template: deploymentconfig.yml } - - { role: openshift/rollout, app: waiverdb, name: waiverdb-web } + - { role: openshift/object, vars: {app: waiverdb, file: imagestream.yml }} + - { role: openshift/object, vars: {app: waiverdb, file: buildconfig.yml }} + - { role: openshift/start-build, vars: {app: waiverdb, name: waiverdb-docker-build }} + - { role: openshift/object, vars: {app: waiverdb, template: configmap.yml }} + - { role: openshift/object, vars: {app: waiverdb, file: service.yml }} + - { role: openshift/object, vars: {app: waiverdb, file: route.yml }} + - { role: openshift/object, vars: {app: waiverdb, template: deploymentconfig.yml }} + - { role: openshift/rollout, vars: {app: waiverdb, name: waiverdb-web }} From c7f95e7c9e36debe3b6d798188a57b15a072bb19 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 5 Apr 2018 20:48:29 +0000 Subject: [PATCH 10/43] try and deal with name scoping some more --- playbooks/groups/batcave.yml | 4 ++- playbooks/groups/people.yml | 2 +- playbooks/groups/secondary.yml | 11 ++++---- playbooks/groups/torrent.yml | 9 +++---- playbooks/include/proxies-certificates.yml | 30 +++++++++++----------- roles/httpd/certificate/tasks/main.yml | 8 +++--- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/playbooks/groups/batcave.yml b/playbooks/groups/batcave.yml index 2444497020..85c06ce1e2 100644 --- a/playbooks/groups/batcave.yml +++ b/playbooks/groups/batcave.yml @@ -26,7 +26,9 @@ - rsyncd - apache - httpd/mod_ssl - - { role: httpd/certificate, name: "{{wildcard_cert_name}}", SSLCertificateChainFile: "{{wildcard_int_file}}" } + - role: httpd/certificate + certname: "{{wildcard_cert_name}}" + SSLCertificateChainFile: "{{wildcard_int_file}}" - openvpn/client - batcave diff --git a/playbooks/groups/people.yml b/playbooks/groups/people.yml index 8dbaa957b6..e7661b4b41 100644 --- a/playbooks/groups/people.yml +++ b/playbooks/groups/people.yml @@ -75,7 +75,7 @@ - role: apache - role: httpd/certificate - name: wildcard-2017.fedorapeople.org + certname: wildcard-2017.fedorapeople.org SSLCertificateChainFile: wildcard-2017.fedorapeople.org.intermediate.cert - people diff --git a/playbooks/groups/secondary.yml b/playbooks/groups/secondary.yml index d01b35d9f4..05df30fe35 100644 --- a/playbooks/groups/secondary.yml +++ b/playbooks/groups/secondary.yml @@ -37,15 +37,16 @@ - role: httpd/mod_ssl - role: httpd/certificate - name: "{{wildcard_cert_name}}" + certname: "{{wildcard_cert_name}}" SSLCertificateChainFile: "{{wildcard_int_file}}" - - role: httpd/website - name: secondary.fedoraproject.org - cert_name: "{{wildcard_cert_name}}" + - { role: httpd/website + vars: + - name: secondary.fedoraproject.org + - cert_name: "{{wildcard_cert_name}}" server_aliases: - archive.fedoraproject.org - - archives.fedoraproject.org + - archives.fedoraproject.org } tasks: - import_tasks: "{{ tasks_path }}/yumrepos.yml" - import_tasks: "{{ tasks_path }}/2fa_client.yml" diff --git a/playbooks/groups/torrent.yml b/playbooks/groups/torrent.yml index f80e989edb..f0bb95844f 100644 --- a/playbooks/groups/torrent.yml +++ b/playbooks/groups/torrent.yml @@ -26,13 +26,10 @@ - role: httpd/mod_ssl - role: httpd/certificate - name: "{{wildcard_cert_name}}" - SSLCertificateChainFile: "{{wildcard_int_file}}" + certname: "{{wildcard_cert_name}}" + SSLCertificateChainFile: "{{wildcard_int_file}}"}} - - role: httpd/website - name: torrent.fedoraproject.org - cert_name: "{{wildcard_cert_name}}" - sslonly: true + - {role: httpd/website, vars: {name: torrent.fedoraproject.org, cert_name: "{{wildcard_cert_name}}", sslonly: true}} tasks: - import_tasks: "{{ tasks_path }}/yumrepos.yml" diff --git a/playbooks/include/proxies-certificates.yml b/playbooks/include/proxies-certificates.yml index 65d86804d0..9a68eb7e5e 100644 --- a/playbooks/include/proxies-certificates.yml +++ b/playbooks/include/proxies-certificates.yml @@ -16,72 +16,72 @@ - role: httpd/mod_ssl - role: httpd/certificate - name: wildcard-2017.fedoraproject.org + certname: wildcard-2017.fedoraproject.org SSLCertificateChainFile: wildcard-2017.fedoraproject.org.intermediate.cert - role: httpd/certificate - name: wildcard-2017.fedorahosted.org + certname: wildcard-2017.fedorahosted.org SSLCertificateChainFile: wildcard-2017.fedorahosted.org.intermediate.cert - role: httpd/certificate - name: wildcard-2017.id.fedoraproject.org + certname: wildcard-2017.id.fedoraproject.org SSLCertificateChainFile: wildcard-2017.id.fedoraproject.org.intermediate.cert - role: httpd/certificate - name: wildcard-2017.stg.fedoraproject.org + certname: wildcard-2017.stg.fedoraproject.org SSLCertificateChainFile: wildcard-2017.stg.fedoraproject.org.intermediate.cert when: env == "staging" - role: httpd/certificate - name: wildcard-2017.app.os.stg.fedoraproject.org + certname: wildcard-2017.app.os.stg.fedoraproject.org SSLCertificateChainFile: wildcard-2017.app.os.stg.fedoraproject.org.intermediate.cert when: env == "staging" tags: - app.os.fedoraproject.org - role: httpd/certificate - name: wildcard-2017.app.os.fedoraproject.org + certname: wildcard-2017.app.os.fedoraproject.org SSLCertificateChainFile: wildcard-2017.app.os.fedoraproject.org.intermediate.cert tags: - app.os.fedoraproject.org - role: httpd/certificate - name: fedoramagazine.org + certname: fedoramagazine.org SSLCertificateChainFile: fedoramagazine.org.intermediate.cert - role: httpd/certificate - name: fpaste.org + certname: fpaste.org SSLCertificateChainFile: fpaste.org.intermediate.cert - role: httpd/certificate - name: getfedora.org + certname: getfedora.org SSLCertificateChainFile: getfedora.org.intermediate.cert - role: httpd/certificate - name: flocktofedora.org + certname: flocktofedora.org SSLCertificateChainFile: flocktofedora.org.intermediate.cert - role: httpd/certificate - name: qa.stg.fedoraproject.org + certname: qa.stg.fedoraproject.org SSLCertificateChainFile: qa.stg.fedoraproject.org.intermediate.cert when: env == "staging" - role: httpd/certificate - name: qa.fedoraproject.org + certname: qa.fedoraproject.org SSLCertificateChainFile: qa.fedoraproject.org.intermediate.cert - role: httpd/certificate - name: secondary.koji.fedoraproject.org.letsencrypt + certname: secondary.koji.fedoraproject.org.letsencrypt SSLCertificateChainFile: secondary.koji.fedoraproject.org.letsencrypt.intermediate.crt - role: httpd/certificate - name: whatcanidoforfedora.org + certname: whatcanidoforfedora.org SSLCertificateChainFile: whatcanidoforfedora.org.intermediate.crt tags: - whatcanidoforfedora.org - role: httpd/certificate - name: fedoracommunity.org + certname: fedoracommunity.org SSLCertificateChainFile: fedoracommunity.org.intermediate.cert tags: - fedoracommunity.org diff --git a/roles/httpd/certificate/tasks/main.yml b/roles/httpd/certificate/tasks/main.yml index afae3243eb..afad02c105 100644 --- a/roles/httpd/certificate/tasks/main.yml +++ b/roles/httpd/certificate/tasks/main.yml @@ -18,7 +18,7 @@ - httpd - httpd/certificate -- name: Copy {{name}}.cert +- name: Copy {{certname}}.cert copy: > src={{item}} dest=/etc/pki/tls/certs/{{item | basename}} @@ -27,14 +27,14 @@ mode=0644 with_first_found: - "{{private}}/files/httpd/{{cert}}.cert" - - "{{private}}/files/httpd/{{name}}.cert" + - "{{private}}/files/httpd/{{certname}}.cert" notify: - reload proxyhttpd tags: - httpd - httpd/certificate -- name: Copy {{name}}.key +- name: Copy {{certname}}.key copy: > src={{item}} dest=/etc/pki/tls/private/{{item | basename}} @@ -43,7 +43,7 @@ mode=0600 with_first_found: - "{{private}}/files/httpd/{{key}}.key" - - "{{private}}/files/httpd/{{name}}.key" + - "{{private}}/files/httpd/{{certname}}.key" notify: - reload proxyhttpd tags: From 874887227248febb65333b45a2ad3789a5e7391e Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 5 Apr 2018 20:50:24 +0000 Subject: [PATCH 11/43] clean up some leftover }s --- playbooks/groups/secondary.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/groups/secondary.yml b/playbooks/groups/secondary.yml index 05df30fe35..56a62e2f26 100644 --- a/playbooks/groups/secondary.yml +++ b/playbooks/groups/secondary.yml @@ -40,13 +40,13 @@ certname: "{{wildcard_cert_name}}" SSLCertificateChainFile: "{{wildcard_int_file}}" - - { role: httpd/website + - role: httpd/website vars: - name: secondary.fedoraproject.org - cert_name: "{{wildcard_cert_name}}" server_aliases: - archive.fedoraproject.org - - archives.fedoraproject.org } + - archives.fedoraproject.org tasks: - import_tasks: "{{ tasks_path }}/yumrepos.yml" - import_tasks: "{{ tasks_path }}/2fa_client.yml" From 109a1fd2431f8b20228903db7697274d6df27d1f Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 5 Apr 2018 20:52:19 +0000 Subject: [PATCH 12/43] drop stray }}s --- playbooks/groups/torrent.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/groups/torrent.yml b/playbooks/groups/torrent.yml index f0bb95844f..85be8e054d 100644 --- a/playbooks/groups/torrent.yml +++ b/playbooks/groups/torrent.yml @@ -27,7 +27,7 @@ - role: httpd/certificate certname: "{{wildcard_cert_name}}" - SSLCertificateChainFile: "{{wildcard_int_file}}"}} + SSLCertificateChainFile: "{{wildcard_int_file}}" - {role: httpd/website, vars: {name: torrent.fedoraproject.org, cert_name: "{{wildcard_cert_name}}", sslonly: true}} From 2a709b0387b067e2cd466ad482e5cf7d60e63cce Mon Sep 17 00:00:00 2001 From: Ricky Elrod Date: Thu, 5 Apr 2018 21:19:51 +0000 Subject: [PATCH 13/43] f27 this box Signed-off-by: Ricky Elrod --- inventory/host_vars/modernpaste02.phx2.fedoraproject.org | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inventory/host_vars/modernpaste02.phx2.fedoraproject.org b/inventory/host_vars/modernpaste02.phx2.fedoraproject.org index 46cfadf301..2d90fa43f2 100644 --- a/inventory/host_vars/modernpaste02.phx2.fedoraproject.org +++ b/inventory/host_vars/modernpaste02.phx2.fedoraproject.org @@ -3,8 +3,8 @@ nm: 255.255.255.0 gw: 10.5.126.254 dns: 10.5.126.21 -ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-25 -ks_repo: http://10.5.126.23/pub/fedora/linux/releases/25/Server/x86_64/os/ +ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-27 +ks_repo: http://10.5.126.23/pub/fedora/linux/releases/27/Server/x86_64/os/ volgroup: /dev/vg_virthost01 eth0_ip: 10.5.126.238 From 57a98d07db15d5461507da96b2b1899fc013306e Mon Sep 17 00:00:00 2001 From: Ricky Elrod Date: Thu, 5 Apr 2018 22:02:59 +0000 Subject: [PATCH 14/43] Toggle another selinux bool so sn2mp works Signed-off-by: Ricky Elrod --- roles/modernpaste/tasks/main.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/roles/modernpaste/tasks/main.yml b/roles/modernpaste/tasks/main.yml index 05c7937abf..c6e691d92d 100644 --- a/roles/modernpaste/tasks/main.yml +++ b/roles/modernpaste/tasks/main.yml @@ -80,8 +80,11 @@ tags: - modernpaste -- name: set sebooleans so paste can talk to the db - seboolean: name=httpd_can_network_connect_db state=true persistent=true +- name: set sebooleans so paste can talk to the db and sn2mp can talk to paste + seboolean: name={{item}} state=true persistent=true + with_items: + - httpd_can_network_connect_db + - httpd_can_network_connect tags: - config - selinux From e70ba908416069b1f531e6236695a8e0ccc02df6 Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Fri, 6 Apr 2018 03:16:18 +0530 Subject: [PATCH 15/43] fedimg: Remove the configuration related to staging Signed-off-by: Sayan Chowdhury --- roles/fedimg/tasks/main.yml | 2 -- roles/fedimg/templates/fedmsg.d/fedimg.py | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/fedimg/tasks/main.yml b/roles/fedimg/tasks/main.yml index f36ec72d79..5ceacbabcb 100644 --- a/roles/fedimg/tasks/main.yml +++ b/roles/fedimg/tasks/main.yml @@ -17,7 +17,6 @@ package: name={{ item }} state=present with_items: - euca2ools - when: env == "staging" tags: - fedimg @@ -44,7 +43,6 @@ owner=fedmsg group=fedmsg mode=0700 notify: - restart fedmsg-hub - when: env == "staging" tags: - fedimg diff --git a/roles/fedimg/templates/fedmsg.d/fedimg.py b/roles/fedimg/templates/fedmsg.d/fedimg.py index f3aa4dac56..25401cd5d3 100644 --- a/roles/fedimg/templates/fedmsg.d/fedimg.py +++ b/roles/fedimg/templates/fedmsg.d/fedimg.py @@ -29,6 +29,8 @@ config = { } {% else %} config = { - 'fedimgconsumer': True, + 'fedimgconsumer.dev.enabled': False, + 'fedimgconsumer.prod.enabled': True, + 'fedimgconsumer.stg.enabled': False, } {% endif %} From 1eddae41706678c048bc33338001da6e5439f4da Mon Sep 17 00:00:00 2001 From: Ricky Elrod Date: Thu, 5 Apr 2018 22:12:00 +0000 Subject: [PATCH 16/43] conditionalize this for now Signed-off-by: Ricky Elrod --- roles/modernpaste/templates/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/modernpaste/templates/config.py b/roles/modernpaste/templates/config.py index 20ceb6ff6a..752d25acb4 100644 --- a/roles/modernpaste/templates/config.py +++ b/roles/modernpaste/templates/config.py @@ -52,7 +52,11 @@ REQUIRE_LOGIN_TO_PASTE = False # Authentication method # This selects between either local users or oidc (OpenID Connect) +{% if env == 'staging' %} AUTH_METHOD = 'oidc' +{% else %} +AUTH_METHOD = 'local' +{% endif %} # OpenID Connect client secrets file AUTH_OIDC_CLIENT_SECRETS = '/etc/modern-paste/client_secrets.json' From 1049a16bf0a84171da4bcd0a725fa62675448140 Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Fri, 6 Apr 2018 03:56:50 +0530 Subject: [PATCH 17/43] fedimg: Set testing False for fedimg group Signed-off-by: Sayan Chowdhury --- inventory/group_vars/fedimg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/inventory/group_vars/fedimg b/inventory/group_vars/fedimg index 08e55eab39..8401c2232b 100644 --- a/inventory/group_vars/fedimg +++ b/inventory/group_vars/fedimg @@ -3,6 +3,8 @@ lvm_size: 20000 mem_size: 6144 num_cpus: 2 +testing: False + # for systems that do not match the above - specify the same parameter in # the host_vars/$hostname file From 93ab6cdfc93adf9bc91345f2344673363bd752cb Mon Sep 17 00:00:00 2001 From: Ricky Elrod Date: Thu, 5 Apr 2018 22:32:59 +0000 Subject: [PATCH 18/43] add issuer Signed-off-by: Ricky Elrod --- roles/modernpaste/templates/client_secrets.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/roles/modernpaste/templates/client_secrets.json b/roles/modernpaste/templates/client_secrets.json index cd5802767c..b7ff49843b 100644 --- a/roles/modernpaste/templates/client_secrets.json +++ b/roles/modernpaste/templates/client_secrets.json @@ -6,7 +6,8 @@ "client_id": "modernpaste", "client_secret": "{{stg_modernpaste_oidc_secret}}", "userinfo_uri": "https://id.stg.fedoraproject.org/openidc/UserInfo", - "token_introspection_uri": "https://id.stg.fedoraproject.org/openidc/TokenInfo" + "token_introspection_uri": "https://id.stg.fedoraproject.org/openidc/TokenInfo", + "issuer": ["https://id.stg.fedoraproject.org/openidc/"] } } {% else %} @@ -17,7 +18,8 @@ "client_id": "modernpaste", "client_secret": "{{prod_modernpaste_oidc_secret}}", "userinfo_uri": "https://id.fedoraproject.org/openidc/UserInfo", - "token_introspection_uri": "https://id.fedoraproject.org/openidc/TokenInfo" + "token_introspection_uri": "https://id.fedoraproject.org/openidc/TokenInfo", + "issuer": ["https://id.stg.fedoraproject.org/openidc/"] } } {% endif %} From 8693bb0902bf6ec90694eeca466a2dbf90f4b4ff Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Fri, 6 Apr 2018 04:17:53 +0530 Subject: [PATCH 19/43] fedimg: Remove the fedimg hotfix Signed-off-by: Sayan Chowdhury --- files/hotfix/fedimg/consumers.py | 99 -------------------------------- roles/fedimg/tasks/main.yml | 8 --- 2 files changed, 107 deletions(-) delete mode 100644 files/hotfix/fedimg/consumers.py diff --git a/files/hotfix/fedimg/consumers.py b/files/hotfix/fedimg/consumers.py deleted file mode 100644 index 3ebd4cb0c2..0000000000 --- a/files/hotfix/fedimg/consumers.py +++ /dev/null @@ -1,99 +0,0 @@ -# This file is part of fedimg. -# Copyright (C) 2014 Red Hat, Inc. -# -# fedimg is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# fedimg is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public -# License along with fedimg; if not, see http://www.gnu.org/licenses, -# or write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -# -# Authors: David Gay -# - -import logging -log = logging.getLogger("fedmsg") - -import multiprocessing.pool - -import fedmsg.consumers -import fedmsg.encoding -import fedfind.release - -import fedimg.uploader -from fedimg.util import get_rawxz_urls, safeget - - -class FedimgConsumer(fedmsg.consumers.FedmsgConsumer): - """ Listens for image Koji task completion and sends image files - produced by the child createImage tasks to the uploader. """ - - # It used to be that all *image* builds appeared as scratch builds on the - # task.state.change topic. However, with the switch to pungi4, some of - # them (and all of them in the future) appear as full builds under the - # build.state.change topic. That means we have to handle both cases like - # this, at least for now. - topic = [ - 'org.fedoraproject.prod.pungi.compose.status.change', - ] - - config_key = 'fedimgconsumer' - - def __init__(self, *args, **kwargs): - super(FedimgConsumer, self).__init__(*args, **kwargs) - - # threadpool for upload jobs - self.upload_pool = multiprocessing.pool.ThreadPool(processes=4) - - log.info("Super happy fedimg ready and reporting for duty.") - - def consume(self, msg): - """ This is called when we receive a message matching our topics. """ - - log.info('Received %r %r' % (msg['topic'], msg['body']['msg_id'])) - - STATUS_F = ('FINISHED_INCOMPLETE', 'FINISHED',) - - msg_info = msg['body']['msg'] - if msg_info['status'] not in STATUS_F: - return - - location = msg_info['location'] - compose_id = msg_info['compose_id'] - cmetadata = fedfind.release.get_release_cid(compose_id).metadata - - # Till F27, both cloud-base and atomic images were available - # under variant CloudImages. With F28 and onward releases, - # cloud-base image compose moved to cloud variant and atomic images - # moved under atomic variant. - prev_rel = ['26', '27'] - if msg_info['release_version'] in prev_rel: - images_meta = safeget(cmetadata, 'images', 'payload', 'images', - 'CloudImages', 'x86_64') - else: - images_meta = safeget(cmetadata, 'images', 'payload', 'images', - 'Cloud', 'x86_64') - images_meta.extend(safeget(cmetadata, 'images', 'payload', - 'images', 'AtomicHost', 'x86_64')) - - if images_meta is None: - return - - self.upload_urls = get_rawxz_urls(location, images_meta) - compose_meta = { - 'compose_id': compose_id, - } - - if len(self.upload_urls) > 0: - log.info("Processing compose id: %s" % compose_id) - fedimg.uploader.upload(self.upload_pool, - self.upload_urls, - compose_meta) diff --git a/roles/fedimg/tasks/main.yml b/roles/fedimg/tasks/main.yml index 5ceacbabcb..4b98023843 100644 --- a/roles/fedimg/tasks/main.yml +++ b/roles/fedimg/tasks/main.yml @@ -134,11 +134,3 @@ tags: - cron - fedimg - -- name: hotfix - copy the consumers.py over to the site-packages - copy: src="{{ files }}/hotfix/fedimg/consumers.py" dest=/usr/lib/python2.7/site-packages/fedimg/consumers.py - notify: - - restart fedmsg-hub - tags: - - fedimg - - hotfix From 628906a9455326ed41c8f32be5bdeaad9af5eddd Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Fri, 6 Apr 2018 05:29:38 +0530 Subject: [PATCH 20/43] fedimg: Port fix for F28 compose messages to 1.0.1 Signed-off-by: Sayan Chowdhury --- files/hotfix/fedimg/consumers.py | 135 +++++++++++++++++++++++++++++++ roles/fedimg/tasks/main.yml | 8 ++ 2 files changed, 143 insertions(+) create mode 100644 files/hotfix/fedimg/consumers.py diff --git a/files/hotfix/fedimg/consumers.py b/files/hotfix/fedimg/consumers.py new file mode 100644 index 0000000000..9b5e4a0437 --- /dev/null +++ b/files/hotfix/fedimg/consumers.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- +# This file is part of fedimg. +# Copyright (C) 2014-2017 Red Hat, Inc. +# +# fedimg is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# fedimg is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with fedimg; if not, see http://www.gnu.org/licenses, +# or write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Authors: David Gay +# Sayan Chowdhury +""" +This is the `fedmsg consumer`_ that subscribes to the topic emitted after the +completion of the nightly and production compose. The consumer on receving the +message uploads the image using the API of the cloud providers. +""" + +import logging +import multiprocessing.pool + +import fedmsg.consumers +import fedmsg.encoding +import fedfind.release + +import fedimg.uploader + +from fedimg.config import PROCESS_COUNT, STATUS_FILTER +from fedimg.utils import get_rawxz_urls, get_value_from_dict + +LOG = logging.getLogger(__name__) + + +class FedimgConsumer(fedmsg.consumers.FedmsgConsumer): + """ + A `fedmsg consumer`_ that listens to the pungi compose topics and kicks + of the process to upload the images to various cloud providers. + + Attributes: + topic (str): The topics this consumer is subscribed to. Set to + ``org.fedoraproject.prod.pungi.compose.status.change``. + config_key (str): The key to set to ``True`` in the fedmsg config to + enable this consumer. The key is ``fedimgconsumer.prod.enabled``. + """ + topic = ['org.fedoraproject.prod.pungi.compose.status.change'] + config_key = "fedimgconsumer.prod.enabled" + + def __init__(self, *args, **kwargs): + LOG.info("FedimgConsumer initializing") + super(FedimgConsumer, self).__init__(*args, **kwargs) + + # Threadpool for upload jobs + LOG.info("Creating thread pool of %s process", PROCESS_COUNT) + self.upload_pool = multiprocessing.pool.ThreadPool( + processes=PROCESS_COUNT + ) + LOG.info("FedimgConsumer initialized") + + def consume(self, msg): + """ + This is called when we receive a message matching our topics. + + Args: + msg (dict): The raw message from fedmsg. + """ + LOG.info('Received %r %r', msg['topic'], msg['body']['msg_id']) + + msg_info = msg['body']['msg'] + if msg_info['status'] not in STATUS_FILTER: + return + + location = msg_info['location'] + compose_id = msg_info['compose_id'] + compose_metadata = fedfind.release.get_release_cid(compose_id).metadata + images_meta = get_value_from_dict( + compose_metadata, + 'images', + 'payload', + 'images', + 'CloudImages', + 'x86_64' + ) + + if images_meta is None: + LOG.debug('No compatible image found to process') + return + + upload_urls = get_rawxz_urls(location, images_meta) + if len(upload_urls) > 0: + LOG.info("Start processing compose id: %s", compose_id) + fedimg.uploader.upload( + pool=self.upload_pool, + urls=upload_urls, + compose_id=compose_id + ) + + +class FedimgStagingConsumer(FedimgConsumer): + """ + A `fedmsg consumer`_ that listens to the staging pungi compose topics and + kicks of the process to upload the images to various cloud providers. + + Attributes: + topic (str): The topics this consumer is subscribed to. Set to + ``org.fedoraproject.stg.pungi.compose.status.change``. + config_key (str): The key to set to ``True`` in the fedmsg config to + enable this consumer. The key is ``fedimgconsumer.stg.enabled``. + """ + topic = ['org.fedoraproject.stg.pungi.compose.status.change'] + config_key = "fedimgconsumer.stg.enabled" + + +class FedimgDevConsumer(FedimgConsumer): + """ + A `fedmsg consumer`_ that listens to the dev pungi compose topics and + kicks of the process to upload the images to various cloud providers. + + Attributes: + topic (str): The topics this consumer is subscribed to. Set to + ``org.fedoraproject.dev.pungi.compose.status.change``. + config_key (str): The key to set to ``True`` in the fedmsg config to + enable this consumer. The key is ``fedimgconsumer.dev.enabled``. + """ + topic = ['org.fedoraproject.dev.pungi.compose.status.change'] + config_key = "fedimgconsumer.dev.enabled" + diff --git a/roles/fedimg/tasks/main.yml b/roles/fedimg/tasks/main.yml index 4b98023843..5ceacbabcb 100644 --- a/roles/fedimg/tasks/main.yml +++ b/roles/fedimg/tasks/main.yml @@ -134,3 +134,11 @@ tags: - cron - fedimg + +- name: hotfix - copy the consumers.py over to the site-packages + copy: src="{{ files }}/hotfix/fedimg/consumers.py" dest=/usr/lib/python2.7/site-packages/fedimg/consumers.py + notify: + - restart fedmsg-hub + tags: + - fedimg + - hotfix From 586b46910deebbd7db92f74f69c9d7c95c5db3b1 Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Fri, 6 Apr 2018 05:34:02 +0530 Subject: [PATCH 21/43] fedimg: patch to process the F28+ messages Signed-off-by: Sayan Chowdhury --- files/hotfix/fedimg/consumers.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/files/hotfix/fedimg/consumers.py b/files/hotfix/fedimg/consumers.py index 9b5e4a0437..410b64190d 100644 --- a/files/hotfix/fedimg/consumers.py +++ b/files/hotfix/fedimg/consumers.py @@ -80,15 +80,24 @@ class FedimgConsumer(fedmsg.consumers.FedmsgConsumer): location = msg_info['location'] compose_id = msg_info['compose_id'] - compose_metadata = fedfind.release.get_release_cid(compose_id).metadata - images_meta = get_value_from_dict( - compose_metadata, - 'images', - 'payload', - 'images', - 'CloudImages', - 'x86_64' - ) + compose_metadata = fedfind.release.get_release(cid=compose_id).metadata + + # Till F27, both cloud-base and atomic images were available + # under variant CloudImages. With F28 and onward releases, + # cloud-base image compose moved to cloud variant and atomic images + # moved under atomic variant. + prev_rel = ['26', '27'] + if msg_info['release_version'] in prev_rel: + images_meta = get_value_from_dict( + compose_metadata, 'images', 'payload', 'images', 'CloudImages', + 'x86_64') + else: + images_meta = get_value_from_dict( + compose_metadata, 'images', 'payload', 'images', + 'Cloud', 'x86_64') + images_meta.extend(get_value_from_dict( + compose_metadata, 'images', 'payload', + 'images', 'AtomicHost', 'x86_64')) if images_meta is None: LOG.debug('No compatible image found to process') From 32ff935a55dba6991c63769231a8dcaef63da0ad Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Fri, 6 Apr 2018 02:14:14 +0000 Subject: [PATCH 22/43] fix typos in pdc-backend csi data. ticket 6775 --- inventory/group_vars/pdc-backend | 8 ++++---- inventory/group_vars/pdc-backend-stg | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/inventory/group_vars/pdc-backend b/inventory/group_vars/pdc-backend index 97720e7ac3..4509bc28b4 100644 --- a/inventory/group_vars/pdc-backend +++ b/inventory/group_vars/pdc-backend @@ -23,18 +23,18 @@ csi_relationship: | fedmsg-hub daemon that loads the pdc-updater consumer plugin. However, the pdc-updater plugin is configured to do different things in each place. - On pdc-updater01, the compose handler is enabled which listens for new pungi + On pdc-backend01, the compose handler is enabled which listens for new pungi composes, and stores them in PDC. Fedora QE uses this data. The consumer has only a single thread enabled to avoid OOMing itself with more than one compose at a time. - On pdc-updater02, the modularity handlers are enabled which listen for MBS - activity, and store that in PDC. pdc-updater02 also hosts the retirement + On pdc-backend02, the modularity handlers are enabled which listen for MBS + activity, and store that in PDC. pdc-backend02 also hosts the retirement handler which listens to dist-git for new dead.package files, and propagates the retirement to PDC (by prematurely EOLing the branch). Multiple threads are enabled so that it can work more efficiently on these smaller tasks. - On pdc-updater03, the dep chain handlers are enabled which listen for koji + On pdc-backend03, the dep chain handlers are enabled which listen for koji messages and store dep chain information in PDC, like what rpms depend on what other rpms at build time, and what containers depend on what rpms, etc.. Multiple threads are enabled so that it can work more efficiently on these diff --git a/inventory/group_vars/pdc-backend-stg b/inventory/group_vars/pdc-backend-stg index 425ca623ef..f07babb410 100644 --- a/inventory/group_vars/pdc-backend-stg +++ b/inventory/group_vars/pdc-backend-stg @@ -23,11 +23,11 @@ csi_relationship: | a fedmsg-hub daemon that loads the pdc-updater consumer plugin. However, the pdc-updater plugin is configured to do different things in each place. - On pdc-updater01, the compose handler is enabled which listens for new pungi + On pdc-backend01, the compose handler is enabled which listens for new pungi composes, and stores them in PDC. Fedora QE uses this data. The consumer has only a single thread enabled to avoid OOMing itself with more than one compose at a time. - On pdc-updater02, the dep chain and modularity handlers are enabled which + On pdc-backend02, the dep chain and modularity handlers are enabled which listen for koji and MBS activity, and store that in PDC. Multiple threads are enabled so that it can work more efficiently on these smaller tasks. From 0bb668f2dfd374cd992a45353dbc99943559938f Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Fri, 6 Apr 2018 04:09:44 +0000 Subject: [PATCH 23/43] some more fedmsg can_send --- inventory/group_vars/resultsdb-prod | 1 + inventory/group_vars/resultsdb-stg | 1 + 2 files changed, 2 insertions(+) diff --git a/inventory/group_vars/resultsdb-prod b/inventory/group_vars/resultsdb-prod index 1df269d999..74833e516b 100644 --- a/inventory/group_vars/resultsdb-prod +++ b/inventory/group_vars/resultsdb-prod @@ -76,3 +76,4 @@ fedmsg_certs: group: apache can_send: - taskotron.result.new + - resultsdb.result.new diff --git a/inventory/group_vars/resultsdb-stg b/inventory/group_vars/resultsdb-stg index d834f9c64a..ec0840caae 100644 --- a/inventory/group_vars/resultsdb-stg +++ b/inventory/group_vars/resultsdb-stg @@ -73,3 +73,4 @@ fedmsg_certs: group: apache can_send: - taskotron.result.new + - resultsdb.result.new From 52c43d21481a2ed53c6ef8a09819a8e258c25ccb Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Fri, 6 Apr 2018 05:26:20 +0000 Subject: [PATCH 24/43] adjust shm size check for postgres servers --- roles/rkhunter/templates/rkhunter.conf.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/rkhunter/templates/rkhunter.conf.j2 b/roles/rkhunter/templates/rkhunter.conf.j2 index 9bff3d0c09..7a521b848c 100644 --- a/roles/rkhunter/templates/rkhunter.conf.j2 +++ b/roles/rkhunter/templates/rkhunter.conf.j2 @@ -643,3 +643,7 @@ SCRIPTWHITELIST=/usr/bin/groups SCRIPTWHITELIST=/usr/bin/GET SCRIPTWHITELIST=/sbin/ifup SCRIPTWHITELIST=/sbin/ifdown +{% if inventory_hostname in groups['dbservers'] or ansible_hostname.startswith(('pagure', 'retrace', 'anitya', 'upstream')) %} +# Set this size very large on postgres running servers. +IPC_SEG_SIZE=100000000000 +{% endif %} From 180cc21c6ac8e4ccbe709efdf685a6c9fe76b67a Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Fri, 6 Apr 2018 05:31:38 +0000 Subject: [PATCH 25/43] fix typo --- roles/rkhunter/templates/rkhunter.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/rkhunter/templates/rkhunter.conf.j2 b/roles/rkhunter/templates/rkhunter.conf.j2 index 7a521b848c..4f823fbf1e 100644 --- a/roles/rkhunter/templates/rkhunter.conf.j2 +++ b/roles/rkhunter/templates/rkhunter.conf.j2 @@ -643,7 +643,7 @@ SCRIPTWHITELIST=/usr/bin/groups SCRIPTWHITELIST=/usr/bin/GET SCRIPTWHITELIST=/sbin/ifup SCRIPTWHITELIST=/sbin/ifdown -{% if inventory_hostname in groups['dbservers'] or ansible_hostname.startswith(('pagure', 'retrace', 'anitya', 'upstream')) %} +{% if inventory_hostname in groups['dbservers'] or inventory_hostname.startswith(('pagure', 'retrace', 'anitya', 'upstream')) %} # Set this size very large on postgres running servers. IPC_SEG_SIZE=100000000000 {% endif %} From 6699d4ed8e4332ddcdc6551001413ed887a47507 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Fri, 6 Apr 2018 05:34:04 +0000 Subject: [PATCH 26/43] fix space --- roles/rkhunter/templates/rkhunter.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/rkhunter/templates/rkhunter.conf.j2 b/roles/rkhunter/templates/rkhunter.conf.j2 index 4f823fbf1e..8667ddc561 100644 --- a/roles/rkhunter/templates/rkhunter.conf.j2 +++ b/roles/rkhunter/templates/rkhunter.conf.j2 @@ -643,7 +643,7 @@ SCRIPTWHITELIST=/usr/bin/groups SCRIPTWHITELIST=/usr/bin/GET SCRIPTWHITELIST=/sbin/ifup SCRIPTWHITELIST=/sbin/ifdown -{% if inventory_hostname in groups['dbservers'] or inventory_hostname.startswith(('pagure', 'retrace', 'anitya', 'upstream')) %} +{% if inventory_hostname in groups['dbservers'] or inventory_hostname.startswith(('pagure','retrace','anitya','upstream')) %} # Set this size very large on postgres running servers. IPC_SEG_SIZE=100000000000 {% endif %} From aabe4115b5d6c7925bb37cd00b2cc53b35565c00 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Fri, 6 Apr 2018 05:38:25 +0000 Subject: [PATCH 27/43] try and simplify --- roles/rkhunter/templates/rkhunter.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/rkhunter/templates/rkhunter.conf.j2 b/roles/rkhunter/templates/rkhunter.conf.j2 index 8667ddc561..28d08530ad 100644 --- a/roles/rkhunter/templates/rkhunter.conf.j2 +++ b/roles/rkhunter/templates/rkhunter.conf.j2 @@ -643,7 +643,7 @@ SCRIPTWHITELIST=/usr/bin/groups SCRIPTWHITELIST=/usr/bin/GET SCRIPTWHITELIST=/sbin/ifup SCRIPTWHITELIST=/sbin/ifdown -{% if inventory_hostname in groups['dbservers'] or inventory_hostname.startswith(('pagure','retrace','anitya','upstream')) %} +{% if inventory_hostname.startswith(('db','pagure','retrace','anitya','upstream')) %} # Set this size very large on postgres running servers. IPC_SEG_SIZE=100000000000 {% endif %} From 46a249f5ad08ae8cf6b198c3d46fdac0d9a81188 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Fri, 6 Apr 2018 17:03:50 +0200 Subject: [PATCH 28/43] Ensures /etc/dnsmasq.d/ dir exists Signed-off-by: Clement Verna --- playbooks/groups/osbs-cluster.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/playbooks/groups/osbs-cluster.yml b/playbooks/groups/osbs-cluster.yml index 9ea4f06857..bfed5887d6 100644 --- a/playbooks/groups/osbs-cluster.yml +++ b/playbooks/groups/osbs-cluster.yml @@ -305,7 +305,8 @@ state: restarted tasks: - + - name: Ensures /etc/dnsmasq.d/ dir exists + file: path="/etc/dnsmasq.d/" state=directory - name: install fedora dnsmasq specific config copy: src: "{{files}}/osbs/fedora-dnsmasq.conf.{{env}}" From 346d810a6e261400695c0b254904861e144f8fc6 Mon Sep 17 00:00:00 2001 From: Jeremy Cline Date: Fri, 6 Apr 2018 11:05:06 -0400 Subject: [PATCH 29/43] Run release-monitoring db migrations in a pre-deployment step This also adds the necessary alembic configuration. Signed-off-by: Jeremy Cline --- .../files/deploymentconfig.yml | 9 +++++ .../templates/configmap.yml | 35 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/roles/openshift-apps/release-monitoring/files/deploymentconfig.yml b/roles/openshift-apps/release-monitoring/files/deploymentconfig.yml index ba3ad2a846..1ff307439a 100644 --- a/roles/openshift-apps/release-monitoring/files/deploymentconfig.yml +++ b/roles/openshift-apps/release-monitoring/files/deploymentconfig.yml @@ -22,6 +22,15 @@ items: maxUnavailable: 25% timeoutSeconds: 600 updatePeriodSeconds: 1 +{% if env == 'staging' %} + pre: + failurePolicy: Abort + execNewPod: + containerName: release-monitoring-web + command: [ /bin/sh, -i, -c, "alembic -c /etc/anitya/alembic.ini upgrade head" ] + volumes: + - config-volume +{% endif %} type: Rolling template: metadata: diff --git a/roles/openshift-apps/release-monitoring/templates/configmap.yml b/roles/openshift-apps/release-monitoring/templates/configmap.yml index 2392fe8294..aa48fc8cf8 100644 --- a/roles/openshift-apps/release-monitoring/templates/configmap.yml +++ b/roles/openshift-apps/release-monitoring/templates/configmap.yml @@ -70,3 +70,38 @@ data: [anitya_log_config.root] level = "ERROR" handlers = ["console"] + alembic.ini: |- + [alembic] + script_location = anitya:db/migrations + sourceless = false +{% if env == 'staging' %} + sqlalchemy.url = "postgresql://{{ anitya_stg_db_user }}:{{ anitya_stg_db_pass }}@{{ anitya_stg_db_host }}/{{ anitya_stg_db_name }}" +{% else %} + sqlalchemy.url = "postgresql://{{ anitya_db_user }}:{{ anitya_db_pass }}@{{ anitya_db_host }}/{{ anitya_db_name }}" +{% endif %} + [loggers] + keys = root,sqlalchemy,alembic + [handlers] + keys = console + [formatters] + keys = generic + [logger_root] + level = WARN + handlers = console + qualname = + [logger_sqlalchemy] + level = WARN + handlers = + qualname = sqlalchemy.engine + [logger_alembic] + level = INFO + handlers = + qualname = alembic + [handler_console] + class = StreamHandler + args = (sys.stderr,) + level = NOTSET + formatter = generic + [formatter_generic] + format = %(levelname)-5.5s [%(name)s] %(message)s + datefmt = %H:%M:%S From 58633c9921907b85ff853c9dd002dbc1e23db40d Mon Sep 17 00:00:00 2001 From: Jeremy Cline Date: Fri, 6 Apr 2018 15:12:54 +0000 Subject: [PATCH 30/43] Drop if statement in deploymentconfig Signed-off-by: Jeremy Cline --- .../release-monitoring/files/deploymentconfig.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/openshift-apps/release-monitoring/files/deploymentconfig.yml b/roles/openshift-apps/release-monitoring/files/deploymentconfig.yml index 1ff307439a..652b9005e7 100644 --- a/roles/openshift-apps/release-monitoring/files/deploymentconfig.yml +++ b/roles/openshift-apps/release-monitoring/files/deploymentconfig.yml @@ -22,7 +22,6 @@ items: maxUnavailable: 25% timeoutSeconds: 600 updatePeriodSeconds: 1 -{% if env == 'staging' %} pre: failurePolicy: Abort execNewPod: @@ -30,7 +29,6 @@ items: command: [ /bin/sh, -i, -c, "alembic -c /etc/anitya/alembic.ini upgrade head" ] volumes: - config-volume -{% endif %} type: Rolling template: metadata: From 51444b5dc85dad8bc318dc3f025def32d7bc6e91 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Fri, 6 Apr 2018 17:13:16 +0200 Subject: [PATCH 31/43] Let's try without the user in stg Signed-off-by: Clement Verna --- playbooks/groups/osbs-cluster.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/playbooks/groups/osbs-cluster.yml b/playbooks/groups/osbs-cluster.yml index bfed5887d6..c832e28ef1 100644 --- a/playbooks/groups/osbs-cluster.yml +++ b/playbooks/groups/osbs-cluster.yml @@ -325,12 +325,6 @@ - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml tasks: - - name: set policy for koji builder in openshift for osbs - shell: "oadm policy add-role-to-user -n default edit htpasswd_provider: {{ osbs_koji_stg_username }} && touch /etc/origin/koji-builder-policy-added" - args: - creates: "/etc/origin/koji-builder-policy-added" - when: env == "staging" - - name: set policy for koji builder in openshift for osbs shell: "oadm policy add-role-to-user -n default edit htpasswd_provider: {{ osbs_koji_prod_username }} && touch /etc/origin/koji-builder-policy-added" args: @@ -341,6 +335,7 @@ shell: "oadm policy add-role-to-user -n default edit system:serviceaccount:default:builder && touch /etc/origin/atomic-reactor-policy-added" args: creates: "/etc/origin/atomic-reactor-policy-added" + when: env == "production" - name: Deploy OSBS on top of OpenShift hosts: osbs-masters-stg[0]:osbs-masters[0] From 53946acfd6d2469065d7e5b1b4fb293eda8bf71a Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Fri, 6 Apr 2018 17:27:46 +0200 Subject: [PATCH 32/43] Add the vars and KUBECONFIG Path Signed-off-by: Clement Verna --- playbooks/groups/osbs-cluster.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/playbooks/groups/osbs-cluster.yml b/playbooks/groups/osbs-cluster.yml index c832e28ef1..0ce6f74fc8 100644 --- a/playbooks/groups/osbs-cluster.yml +++ b/playbooks/groups/osbs-cluster.yml @@ -395,6 +395,16 @@ tags: - osbs-worker-namespace user: root + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + vars: + osbs_kubeconfig_path: /etc/origin/master/admin.kubeconfig + osbs_environment: + KUBECONFIG: "{{ osbs_kubeconfig_path }}" + roles: - role: osbs-namespace osbs_namespace: "{{ osbs_worker_namespace }}" From 0e4f2354439b2adbb49e29fe64568dc0736b584f Mon Sep 17 00:00:00 2001 From: Ricky Elrod Date: Fri, 6 Apr 2018 20:04:05 +0000 Subject: [PATCH 33/43] make this f27 Signed-off-by: Ricky Elrod --- inventory/host_vars/modernpaste01.phx2.fedoraproject.org | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inventory/host_vars/modernpaste01.phx2.fedoraproject.org b/inventory/host_vars/modernpaste01.phx2.fedoraproject.org index e56aee0a40..f65077e12c 100644 --- a/inventory/host_vars/modernpaste01.phx2.fedoraproject.org +++ b/inventory/host_vars/modernpaste01.phx2.fedoraproject.org @@ -3,8 +3,8 @@ nm: 255.255.255.0 gw: 10.5.126.254 dns: 10.5.126.21 -ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-25 -ks_repo: http://10.5.126.23/pub/fedora/linux/releases/25/Server/x86_64/os/ +ks_url: http://10.5.126.23/repo/rhel/ks/kvm-fedora-27 +ks_repo: http://10.5.126.23/pub/fedora/linux/releases/27/Server/x86_64/os/ volgroup: /dev/vg_virthost03 eth0_ip: 10.5.126.230 From c60591bebe472c009c18ec2fcf2a087212efddf0 Mon Sep 17 00:00:00 2001 From: Ricky Elrod Date: Fri, 6 Apr 2018 20:38:25 +0000 Subject: [PATCH 34/43] add nagios check for modernpaste Signed-off-by: Ricky Elrod --- .../templates/nagios/services/websites.cfg.j2 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/roles/nagios_server/templates/nagios/services/websites.cfg.j2 b/roles/nagios_server/templates/nagios/services/websites.cfg.j2 index 67e74c0fee..ce4744bfb5 100644 --- a/roles/nagios_server/templates/nagios/services/websites.cfg.j2 +++ b/roles/nagios_server/templates/nagios/services/websites.cfg.j2 @@ -114,6 +114,13 @@ define service { use websitetemplate } +define service { + hostgroup_name proxies + service_description http-modernpaste + check_command check_website_ssl!paste.fedoraproject.org!/archive!NEXT + max_check_attempts 8 + use websitetemplate +} ## ## Individual hosts From 41b2948b3c423a71bae4403ce0ef7f9d15eff32e Mon Sep 17 00:00:00 2001 From: Ricky Elrod Date: Fri, 6 Apr 2018 21:00:29 +0000 Subject: [PATCH 35/43] comment out control01.cloud for now Signed-off-by: Ricky Elrod --- inventory/inventory | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/inventory b/inventory/inventory index baa94f0eea..f269db67ee 100644 --- a/inventory/inventory +++ b/inventory/inventory @@ -1120,7 +1120,7 @@ newcloud-control newcloud-compute [newcloud-control] -control01.cloud.fedoraproject.org +#control01.cloud.fedoraproject.org [newcloud-compute] From 72bbaa220cb14cd880a69a2b3645ed3fa23e27a1 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Sat, 7 Apr 2018 01:11:37 +0000 Subject: [PATCH 36/43] add another fedmsg it is ok for bodhi03/04 to emit --- inventory/group_vars/bodhi2 | 1 + 1 file changed, 1 insertion(+) diff --git a/inventory/group_vars/bodhi2 b/inventory/group_vars/bodhi2 index d58091de32..1a111750cc 100644 --- a/inventory/group_vars/bodhi2 +++ b/inventory/group_vars/bodhi2 @@ -60,6 +60,7 @@ fedmsg_certs: - bodhi.update.request.revoke - bodhi.update.request.stable - bodhi.update.request.testing + - bodhi.update.request.batched - bodhi.update.request.unpush # Things that only the mash does - not the web UI From 887d7df729b261a3ca73b49cea305215f2021a8b Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Sat, 7 Apr 2018 14:03:01 +0200 Subject: [PATCH 37/43] Osbs readwrite users is not needed in the worker namespace Signed-off-by: Clement Verna --- playbooks/groups/osbs-cluster.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/playbooks/groups/osbs-cluster.yml b/playbooks/groups/osbs-cluster.yml index 0ce6f74fc8..1fa8e463e7 100644 --- a/playbooks/groups/osbs-cluster.yml +++ b/playbooks/groups/osbs-cluster.yml @@ -413,7 +413,6 @@ osbs_authoritative_registry: "{{ source_registry }}" osbs_sources_command: "{{ osbs_conf_sources_command }}" osbs_vendor: "{{ osbs_conf_vendor }}" - osbs_readwrite_users: "{{ osbs_conf_readwrite_users }}" when: env == "staging" - name: setup koji secret in worker namespace From ccdf42ccc66019dbc99e3b3dcfb3a856357a2708 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Sat, 7 Apr 2018 14:11:42 +0200 Subject: [PATCH 38/43] Fixing the syntax for osbs readwrite users Signed-off-by: Clement Verna --- inventory/group_vars/osbs-masters-stg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inventory/group_vars/osbs-masters-stg b/inventory/group_vars/osbs-masters-stg index 7f17ef3499..2f201aff87 100644 --- a/inventory/group_vars/osbs-masters-stg +++ b/inventory/group_vars/osbs-masters-stg @@ -40,8 +40,8 @@ osbs_conf_service_accounts: - koji osbs_conf_readwrite_users: - - system:serviceaccount:{{ osbs_namespace }}:default - - system:serviceaccount:{{ osbs_namespace }}:builder + - "system:serviceaccount:{{ osbs_namespace }}:default" + - "system:serviceaccount:{{ osbs_namespace }}:builder" osbs_worker_clusters: From e0d35a1b9ca6c115e8fd49f3723d86fd16f98922 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Sat, 7 Apr 2018 14:16:28 +0200 Subject: [PATCH 39/43] Fix the variable name Signed-off-by: Clement Verna --- inventory/group_vars/osbs-masters-stg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/osbs-masters-stg b/inventory/group_vars/osbs-masters-stg index 2f201aff87..93e9cf1bb0 100644 --- a/inventory/group_vars/osbs-masters-stg +++ b/inventory/group_vars/osbs-masters-stg @@ -44,7 +44,7 @@ osbs_conf_readwrite_users: - "system:serviceaccount:{{ osbs_namespace }}:builder" -osbs_worker_clusters: +osbs_conf_worker_clusters: x86_64: - name: x86_64-stg max_concurrent_builds: 2 From 1c861d231d6a2fe307c7399e0685d98f0abb6bb8 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Sat, 7 Apr 2018 17:41:20 +0000 Subject: [PATCH 40/43] add another stray fedmsg --- inventory/host_vars/branched-composer.phx2.fedoraproject.org | 1 + 1 file changed, 1 insertion(+) diff --git a/inventory/host_vars/branched-composer.phx2.fedoraproject.org b/inventory/host_vars/branched-composer.phx2.fedoraproject.org index a1848501dd..38c14764fe 100644 --- a/inventory/host_vars/branched-composer.phx2.fedoraproject.org +++ b/inventory/host_vars/branched-composer.phx2.fedoraproject.org @@ -23,6 +23,7 @@ fedmsg_certs: - pungi.compose.createiso.targets - pungi.compose.createiso.imagefail - pungi.compose.createiso.imagedone + - pungi.compose.ostree - compose.branched.complete - compose.branched.mash.complete - compose.branched.mash.start From dce218db1268d7078b95a05a252eac69994992a4 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Sat, 7 Apr 2018 19:59:04 +0000 Subject: [PATCH 41/43] moved pkgs02 over to bvirthost04 and ssds and off iscsi --- inventory/host_vars/pkgs02.phx2.fedoraproject.org | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inventory/host_vars/pkgs02.phx2.fedoraproject.org b/inventory/host_vars/pkgs02.phx2.fedoraproject.org index 68f14ce85e..28aa15bd99 100644 --- a/inventory/host_vars/pkgs02.phx2.fedoraproject.org +++ b/inventory/host_vars/pkgs02.phx2.fedoraproject.org @@ -6,8 +6,8 @@ gw: 10.5.125.254 dns: 10.5.126.21 ks_url: http://10.5.126.23/repo/rhel/ks/kvm-rhel-7 ks_repo: http://10.5.126.23/repo/rhel/RHEL7-x86_64/ -volgroup: /dev/xenGuests -vmhost: bvirthost11.phx2.fedoraproject.org +volgroup: /dev/vg_guests +vmhost: bvirthost04.phx2.fedoraproject.org datacenter: phx2 mem_size: 16384 From cf874ad5b732f95e9a61e3703e47c252003ea30d Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Mon, 9 Apr 2018 08:02:03 +0000 Subject: [PATCH 42/43] Install f28 maintainer-test in cloud --- inventory/cloud | 1 + .../host_vars/f28-test.fedorainfracloud.org | 17 +++++++++++++++++ inventory/inventory | 1 + 3 files changed, 19 insertions(+) create mode 100644 inventory/host_vars/f28-test.fedorainfracloud.org diff --git a/inventory/cloud b/inventory/cloud index e020e82c4a..e7fbd71718 100644 --- a/inventory/cloud +++ b/inventory/cloud @@ -24,6 +24,7 @@ el6-test.fedorainfracloud.org el7-test.fedorainfracloud.org f26-test.fedorainfracloud.org f27-test.fedorainfracloud.org +f28-test.fedorainfracloud.org faitout.fedorainfracloud.org fas2-dev.fedorainfracloud.org fas3-dev.fedorainfracloud.org diff --git a/inventory/host_vars/f28-test.fedorainfracloud.org b/inventory/host_vars/f28-test.fedorainfracloud.org new file mode 100644 index 0000000000..94959259be --- /dev/null +++ b/inventory/host_vars/f28-test.fedorainfracloud.org @@ -0,0 +1,17 @@ +--- +image: Fedora-Cloud-Base-28_Beta-1.3.x86_64 +instance_type: ms1.medium +keypair: fedora-admin-20130801 +security_group: ssh-anywhere-maintainertest,web-80-anywhere-maintainertest,default,web-443-anywhere-maintainertest,all-icmp-maintainertest +zone: nova +tcp_ports: [22] + +inventory_tenant: maintainertest +inventory_instance_name: f28-test +hostbase: f28-test +public_ip: 209.132.184.118 +description: f28-test instance + +cloud_networks: + # maintainertest-net + - net-id: "a512d096-3fa2-49cc-b989-d43ca687f91d" diff --git a/inventory/inventory b/inventory/inventory index f269db67ee..21a14263e6 100644 --- a/inventory/inventory +++ b/inventory/inventory @@ -1216,6 +1216,7 @@ testdays.fedorainfracloud.org [maintainer-test] f26-test.fedorainfracloud.org f27-test.fedorainfracloud.org +f28-test.fedorainfracloud.org rawhide-test.fedorainfracloud.org ppc64-test.fedorainfracloud.org ppc64le-test.fedorainfracloud.org From 3a591c2ad38de0c78de2e881584499d46ad7b74f Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Mon, 9 Apr 2018 09:21:49 +0000 Subject: [PATCH 43/43] Increase timeout for spinning UP VM using nova_compute --- tasks/persistent_cloud.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/persistent_cloud.yml b/tasks/persistent_cloud.yml index 88cd56a85b..c1b36483bf 100644 --- a/tasks/persistent_cloud.yml +++ b/tasks/persistent_cloud.yml @@ -17,7 +17,7 @@ login_tenant_name: "{{inventory_tenant}}" name: "{{inventory_instance_name}}" image_id: "{{ image|image_name_to_id('admin', ADMIN_PASS, inventory_tenant, os_auth_url) }}" - wait_for: 300 + wait_for: 600 flavor_id: "{{ instance_type|flavor_name_to_id('admin', ADMIN_PASS, inventory_tenant, os_auth_url) }}" security_groups: "{{security_group}}" key_name: "{{ keypair }}"