bodhi: Upgrade production to Bodhi 4.0.0.

Signed-off-by: Randy Barlow <randy@electronsweatshop.com>

inventory/group_vars/bodhi_backend

@@ -15,15 +15,6 @@ virt_install_command: "{{ virt_install_command_two_nic }}"
 # Do not use testing repositories on production
 testing: False
 
-# These are for fedmsg publication from the bodhi backend.
-# If you change these iptables rules, you also need to changes the endpoints
-# list in roles/fedmsg/base/templates/endpoints-bodhi.py
-tcp_ports: [
-    3000, 3001, 3002, 3003, 3004,
-    3005, 3006, 3007, 3008, 3009,
-    3010, 3011, 3012, 3013, 3014,
-    3015, 3016, 3017, 3018, 3019,
-]
 # Make connections from signing bridges stateless, they break sigul connections
 # https://bugzilla.redhat.com/show_bug.cgi?id=1283364
 custom_rules: ['-A INPUT --proto tcp --sport 44334 --source 10.5.125.71 -j ACCEPT']
@@ -34,9 +25,10 @@ nrpe_procs_crit: 1000
 
 host_group: bodhi2
 
-# These people get told when something goes wrong.
+bodhi_message_queue_name: "bodhi{{ env_suffix }}_composer"
-fedmsg_error_recipients:
+# Define the topics that our fedora-messaging queue should be subscribed to.
-- bodhiadmin-members@fedoraproject.org
+bodhi_message_routing_keys:
+  - "org.fedoraproject.*.bodhi.composer.start"
 
 ## XXX -- note that the fedmsg_certs declaration does not happen here, but
 # happens instead at the inventory/host_vars/ level since bodhi-backend03 and

playbooks/groups/bodhi-backend.yml

@@ -48,9 +48,6 @@
     nfs_src_dir: 'fedora_ftp/fedora.redhat.com/pub/'
 
   - bodhi2/backend
-  # We can eliminate this block once we deploy bodhi 4.0.0 to production
-  - role: fedmsg/base
-    when: env == "production"
   - role: collectd/fedmsg-service
     process: fedmsg-hub
     user: masher
@@ -74,16 +71,12 @@
     cert_src: "{{private}}/files/docker-registry/{{env}}/pki/issued/containerstable.crt"
     key_src: "{{private}}/files/docker-registry/{{env}}/pki/private/containerstable.key"
     certs_group: apache
-  # We can remove the when statement when Bodhi 4.0.0 is deployed to production.
   - role: rabbit/user
     username: "bodhi{{ env_suffix }}"
-    when: env == "staging"
-  # We can remove the when statement when Bodhi 4.0.0 is deployed to production.
   - role: rabbit/queue
     username: "bodhi{{ env_suffix }}"
     queue_name: "{{ bodhi_message_queue_name }}"
     routing_keys: "{{ bodhi_message_routing_keys }}"
-    when: env == "staging"
 
 
   tasks:

playbooks/manual/upgrade/bodhi.yml

@@ -101,15 +101,10 @@
   - bodhi2/backend
 
   tasks:
-  # The when condition on this block can be dropped once we deploy bodhi 4.0.0 to production.
   - name: Stop the fedora-messaging backend
     service:
         name: fm-consumer@config.service
         state: stopped
-    when: env == "staging"
-  # This block can be dropped once we deploy bodhi 4.0.0 to production.
-  - name: Stop the fedmsg-hub-3 backend
-    service: name="fedmsg-hub-3" state=stopped
 
   - name: Upgrade the database
     command: /usr/bin/alembic-3 -c /etc/bodhi/alembic.ini upgrade head
@@ -117,16 +112,10 @@
       chdir: /usr/share/bodhi/
     when: inventory_hostname.startswith(('bodhi-backend01.phx2', 'bodhi-backend01.stg.phx2'))
 
-  # The when condition on this block can be dropped once we deploy bodhi 4.0.0 to production.
   - name: Start the fedora-messaging backend
     service:
         name: fm-consumer@config.service
         state: started
-    when: env == "staging"
-  # This block can be dropped once we deploy bodhi 4.0.0 to production.
-  - name: And... start the backend again
-    service: name="fedmsg-hub-3" state=started
-    when: env == "production"
 
   post_tasks:
   - name: tell nagios to unshush w.r.t. the backend

playbooks/openshift-apps/bodhi.yml

@@ -19,16 +19,12 @@
   - include_vars: dir=/srv/web/infra/ansible/vars/all/ ignore_files=README
 
   roles:
-  # We can remove the when statement when Bodhi 4.0.0 is deployed to production.
   - role: rabbit/user
     username: "bodhi{{ env_suffix }}"
-    when: env == "staging"
-  # We can remove the when statement when Bodhi 4.0.0 is deployed to production.
   - role: rabbit/queue
     username: "bodhi{{ env_suffix }}"
     queue_name: "{{ bodhi_message_queue_name }}"
     routing_keys: "{{ bodhi_message_routing_keys }}"
-    when: env == "staging"
   - role: openshift/project
     app: bodhi
     description: bodhi
@@ -44,39 +40,21 @@
     secret_name: bodhi-keytab
     service: bodhi
     host: "bodhi{{ env_suffix }}.fedoraproject.org"
-  # When we deploy Bodhi 4.0.0 to production, we can drop the env staging statements on the
-  # fedora-messaging items below.
   - role: openshift/secret-file
     app: bodhi
     secret_name: bodhi-fedora-messaging-ca
     key: cacert.pem
     privatefile: "rabbitmq/{{env}}/pki/ca.crt"
-    when: env == "staging"
   - role: openshift/secret-file
     app: bodhi
     secret_name: bodhi-fedora-messaging-crt
     key: bodhi-cert.pem
     privatefile: "rabbitmq/{{env}}/pki/issued/bodhi{{env_suffix}}.crt"
-    when: env == "staging"
   - role: openshift/secret-file
     app: bodhi
     secret_name: bodhi-fedora-messaging-key
     key: bodhi-key.pem
     privatefile: "rabbitmq/{{env}}/pki/private/bodhi{{env_suffix}}.key"
-    when: env == "staging"
-  # When we deploy Bodhi 4.0.0 to production, we can drop these fedmsg secrets.
-  - role: openshift/secret-file
-    app: bodhi
-    secret_name: bodhi-fedmsg-key
-    key: fedmsg-bodhi.key
-    privatefile: fedmsg-certs/keys/bodhi-bodhi-web-temp-bodhi.app.os.fedoraproject.org.key
-    when: env != "staging"
-  - role: openshift/secret-file
-    app: bodhi
-    secret_name: bodhi-fedmsg-crt
-    key: fedmsg-bodhi.crt
-    privatefile: fedmsg-certs/keys/bodhi-bodhi-web-temp-bodhi.app.os.fedoraproject.org.crt
-    when: env != "staging"
   - role: openshift/imagestream
     app: bodhi
     imagename: bodhi-base
@@ -96,7 +74,7 @@
     app: bodhi
     template: buildconfig.yml
     objectname: buildconfig.yml
-    bodhi_version: 3.14.0-1.fc29.infra
+    bodhi_version: 4.0.0-1.fc29.infra
     when: env == "production"
   - role: openshift/start-build
     app: bodhi
@@ -127,10 +105,12 @@
     dcname: bodhi-consumer
 
   post_tasks:
+  - name: Scale up pods
+    command: oc -n bodhi scale dc/bodhi-consumer --replicas={{ hostvars[groups['bodhi2'][0]]['openshift_pods'] }}
+    when: env == "production"
   - name: Scale up pods
     command: oc -n bodhi scale dc/bodhi-web --replicas={{ hostvars[groups['bodhi2'][0]]['openshift_pods'] }}
     when: env == "production"
-  # Add one of these for production when we deploy 4.0.0
   - name: Scale up pods
     command: oc -n bodhi scale dc/bodhi-consumer --replicas={{ hostvars[groups['bodhi2_stg'][0]]['openshift_pods'] }}
     when: env == "staging"

roles/bodhi2/backend/tasks/main.yml

@@ -187,20 +187,6 @@
   - bodhi
   - cron
 
-# This task can be dropped once Bodhi 4.0.0 is deployed to production.
-- name: Install bodhi-dequeue-stable cron job that moves updates from batched to stable
-  cron:
-    name: "bodhi-dequeue-stable"
-    minute: "45"
-    hour: "23"
-    weekday: "*"
-    job: "/usr/bin/bodhi-dequeue-stable && touch /var/lib/bodhi/dequeue-stable"
-    user: "apache"
-  when: inventory_hostname.startswith('bodhi-backend01.phx2') and env == "production"
-  tags:
-  - bodhi
-  - cron
-
 - name: put the koji sync listener script in place
   copy:
       src: koji-sync-listener.py
@@ -356,7 +342,6 @@
   - config
   - bodhi
 
-# Remove the env == "staging" condition when we deploy Bodhi 4.0.0 to production.
 - name: Install fedora-messaging config
   template: >
     src="{{  roles_path  }}/bodhi2/base/templates/fedora-messaging.toml.j2"
@@ -364,7 +349,7 @@
     owner=apache
     group=apache
     mode=0600
-  when: inventory_hostname.startswith('bodhi-backend') and env == "staging"
+  when: inventory_hostname.startswith('bodhi-backend')
   notify:
   - restart fedora-messaging
   tags:
@@ -376,31 +361,6 @@
   tags:
   - bodhi
 
-# We can drop this once we deploy Bodhi 4.0.0 to production
-- name: create the /usr/lib/systemd/system/fedmsg-hub-3.service.d drop-in directory
-  file: path=/usr/lib/systemd/system/fedmsg-hub-3.service.d state=directory mode=0755
-  when: env == "production"
-  tags:
-  - config
-  - bodhi
-
-# We can drop this once we deploy Bodhi 4.0.0 to production
-- name: install a femdsg-hub-3.service drop-in to run it as the apache
-  copy: >
-    src="fedmsg-hub.conf"
-    dest="/usr/lib/systemd/system/fedmsg-hub-3.service.d/fedmsg-hub-3.conf"
-    owner=root
-    group=root
-    mode=0644
-  when: env == "production"
-  notify:
-  - reload systemd
-  - restart fedmsg-hub-3
-  tags:
-  - bodhi
-  - config
-
-# We can drop the when condition when we deploy Bodhi 4.0.0 to production
 - name: Configure fm-consumer@.service to run as apache
   copy:
     src: fm-consumer@.service
@@ -408,7 +368,6 @@
     owner: root
     group: root
     mode: 0644
-  when: env == "staging"
   notify:
   - reload systemd
   - restart fedora-messaging
@@ -426,50 +385,11 @@
   - config
   - bodhi
 
-# We can drop this block when we upgrade production to bodhi 4.0.0
-- name: have the apache own /var/cache/fedmsg because of course..
-  file: >
-    path="/var/cache/fedmsg"
-    owner=apache
-    group=apache
-    state=directory
-  when: inventory_hostname.startswith('bodhi-backend') and env == "production"
-  tags:
-  - config
-  - bodhi
-
-# We can drop this block when we upgrade production to bodhi 4.0.0
-- name: ensure that nrpe has rights to monitor us (dir), including setgid bit.
-  file: >
-    dest=/var/run/fedmsg
-    mode=2775
-    owner=apache
-    group=nrpe
-    state=directory
-  when: env == "production"
-  tags:
-  - fedmsgmonitor
-  - bodhi
-
-# We can drop this block when we upgrade production to bodhi 4.0.0
-- name: ensure that nrpe has rights to monitor us (file)
-  file: >
-    dest=/var/run/fedmsg/monitoring-fedmsg-hub-3.socket
-    mode=0775
-    owner=apache
-    group=nrpe
-    state=file
-  when: env == "production"
-  tags:
-  - fedmsgmonitor
-  - bodhi
-
 - name: ensure apache is disabled on the backend
   service: name=httpd enabled=no state=stopped
   tags:
   - bodhi
 
-# The when statement should be dropped when we deploy Bodhi 4.0.0 to production.
 - name: Create /etc/pki/fedora-messaging
   file:
     dest: /etc/pki/fedora-messaging
@@ -480,7 +400,6 @@
   tags:
   - bodhi
 
-# The when staging should be dropped when we deploy Bodhi 4.0.0 to production.
 - name: Deploy the fedora-messaging CA
   copy:
     src: "{{ private }}/files/rabbitmq/{{env}}/pki/ca.crt"
@@ -488,13 +407,11 @@
     mode: 0644
     owner: apache
     group: apache
-  when: env == "staging"
   notify:
   - restart fedora-messaging
   tags:
   - bodhi
   
-# The when staging should be dropped when we deploy Bodhi 4.0.0 to production.
 - name: Deploy the fedora-messaging cert
   copy:
     src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/bodhi{{env_suffix}}.crt"
@@ -502,13 +419,11 @@
     mode: 0644
     owner: apache
     group: apache
-  when: env == "staging"
   notify:
   - restart fedora-messaging
   tags:
   - bodhi
   
-# The when staging should be dropped when we deploy Bodhi 4.0.0 to production.
 - name: Deploy the fedora-messaging key
   copy:
     src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/bodhi{{env_suffix}}.key"
@@ -516,23 +431,13 @@
     mode: 0600
     owner: apache
     group: apache
-  when: env == "staging"
   notify:
   - restart fedora-messaging
   tags:
   - bodhi
   
-# When we deploy Bodhi 4.0.0 to production, we can drop the when statement.
 - name: ensure fedora-messaging is enabled and started on the backend
   service: name=fm-consumer@config.service enabled=yes state=started
-  when: env == "staging"
-  tags:
-  - bodhi
-
-# When we deploy Bodhi 4.0.0 to production, we can drop this block.
-- name: ensure fedmsg-hub-3 is enabled and started on the backend
-  service: name=fedmsg-hub-3 enabled=yes state=started
-  when: env == "production"
   tags:
   - bodhi
 

roles/bodhi2/base/tasks/main.yml

@@ -28,28 +28,6 @@
   - config
   - bodhi
 
-# When Bodhi 4.0.0 is deployed to production, we should drop this.
-- name: Copy some fedmsg configuration of our own for fedmsg-hub-3
-  template: >
-      src={{item}}
-      dest=/etc/fedmsg.d/{{item}}
-      owner=root
-      group=root
-      mode=0644
-  with_items:
-  - bodhi-logging.py
-  - bodhi-threading.py
-  - bodhi.py
-  - masher.py
-  - updates_handler.py
-  - signed_handler.py
-  when: env == 'production'
-  notify:
-  - restart fedmsg-hub-3
-  tags:
-  - config
-  - bodhi
-
 - name: setup /etc/pki/bodhi directory
   file: path=/etc/pki/bodhi owner=root group=root mode=0755 state=directory
   tags:

roles/bodhi2/base/templates/bodhi-logging.py

@@ -1,23 +0,0 @@
-# This should get merged smartly with the site-wide fedmsg.d/logging.py we have
-# installed everywhere.
-config = dict(
-    logging=dict(
-        loggers=dict(
-            bodhi={
-                "level": "DEBUG",
-                "propagate": False,
-                "handlers": ["console", "mailer"],
-            },
-            sqlalchemy={
-                "level": "WARN",
-                "propagate": False,
-                "handlers": ["console", "mailer"],
-            },
-            root={
-                "level": "INFO",
-                "propagate": False,
-                "handlers": ["console", "mailer"],
-            },
-        ),
-    ),
-)

roles/bodhi2/base/templates/bodhi-threading.py

@@ -1,15 +0,0 @@
-config = {
-    # This sets uptwo master threads to handle incoming messages.
-    # Each of those master consumer threads then can fork off many masher
-    # threads to mash different repos.
-    # If you crank up this number, you should also crank up:
-    # - the iptables rules in inventory/group_vars/bodhi-backend
-    # - the fedmsg endpoints in roles/fedmsg/base/templates/endpoints-bodhi.py
-    {% if inventory_hostname.startswith('bodhi-backend02') %}
-    # https://github.com/fedora-infra/bodhi/issues/795
-    "moksha.workers_per_consumer": 1,
-    {% else %}
-    "moksha.workers_per_consumer": 2,
-    {% endif %}
-    "moksha.threadpool_size": 22,
-}

roles/bodhi2/base/templates/bodhi.py

@@ -1,6 +0,0 @@
-# This file is empty on purpose.
-#
-# The bodhi-server rpm ships an /etc/fedmsg.d/bodhi.py file by default that has
-# some nice local settings for testing, but which don't make sense for our
-# infrastructure.  So... we overwrite it with this empty file.
-config = {}

roles/bodhi2/base/templates/configmap.yml

@@ -110,8 +110,6 @@ data:
     {{ load_file('logging.yaml') | indent }}
   production.ini: |-
     {{ load_file('production.ini.j2') | indent }}
-# We can remove this if statement once we deploy Bodhi 4.0.0 to production.
-{% if env == "staging" %}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -122,35 +120,3 @@ metadata:
 data:
   config.toml: |-
     {{ load_file('fedora-messaging.toml.j2') | indent }}
-{% endif %}
-# We can drop the next ConfigMap once Bodhi 4.0.0 is deployed to production.
-{% if env == "production" %}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: fedmsg-configmap
-  labels:
-    app: bodhi
-data:
-  bodhi-openshift-web.py: |-
-    import socket
-    hostname = socket.gethostname().split('.', 1)[0]
-    config = {
-        'sign_messages': True,
-        'active': True,
-        'cert_prefix': 'bodhi',
-        'certnames': {'bodhi.{}'.format(socket.gethostname()): 'bodhi'},
-        'relay_inbound': ['tcp://busgateway01{{ env_suffix }}.phx2.fedoraproject.org:9941'],
-        'bodhi.%s' % hostname: ['tcp://busgateway01{{ env_suffix }}.phx2.fedoraproject.org:9941'],
-{% if env == 'staging' %}
-        'environment': 'stg',
-        # stg should listen to the stg bus
-        'endpoints': {
-            'staging_gateway': ['tcp://stg.fedoraproject.org:9940']
-        }
-{% else %}
-        'environment': 'prod',
-{% endif %}
-    }
-{% endif %}

roles/bodhi2/base/templates/masher.py

@@ -1,17 +0,0 @@
-{% if env == 'staging' %}
-suffix = 'stg.phx2.fedoraproject.org'
-{% else %}
-suffix = 'phx2.fedoraproject.org'
-{% endif %}
-
-config = dict(
-    # Note, the masher runs on bodhi-backend01, while other consumers will run
-    # on bodhi-backend02.
-    masher={{bodhi_masher_enabled}},
-    masher_topic='bodhi.masher.start',
-{% if ansible_hostname == 'bodhi-backend01' %}
-    releng_fedmsg_certname='shell-bodhi-backend01.%s' % suffix,
-{% else %}
-    releng_fedmsg_certname='shell-bodhi-backend03.%s' % suffix,
-{% endif %}
-)

roles/bodhi2/base/templates/production.ini.j2

@@ -41,13 +41,6 @@ filter-with = proxy-prefix
 # libravatar.libravatar_url()'s https setting. Defaults to True.
 # libravatar_prefer_tls =
 
-# We can remove this block once Bodhi 4.0.0 is deployed to production.
-{% if env != "staging" %}
-# Set this to True in order to send fedmsg messages.
-# fedmsg_enabled = False
-fedmsg_enabled = True
-{% endif %}
-
 
 ##
 ### Legal
@@ -145,13 +138,7 @@ message_id_email_domain = admin{{env_suffix}}.fedoraproject.org
 ##
 ## Masher settings
 ##
-# We can remove this block once Bodhi 4.0.0 is deployed to production.
-{% if env != "staging" %}
-releng_fedmsg_certname = shell-bodhi-backend01{{env_suffix}}.phx2.fedoraproject.org
-{% endif %}
 
-# We can remove this if/else once Bodhi 4.0.0 is deployed to production.
-{% if env == "staging" %}
 # Where to initially mash repositories. You can use %(here)s to reference the location of this file.
 # compose_dir =
 {% if inventory_hostname.startswith('bodhi-backend') %}
@@ -160,33 +147,14 @@ compose_dir = /mnt/koji/compose/updates/
 # do not use on frontends as bodhi will check the mount and refuse to run without it.
 # compose_dir = /mnt/koji/compose/updates/
 {% endif %}
-{% else %}
-# Where to initially mash repositories. You can use %(here)s to reference the location of this file.
-# mash_dir =
-{% if inventory_hostname.startswith('bodhi-backend') %}
-mash_dir = /mnt/koji/compose/updates/
-{% else %}
-# do not use on frontends as bodhi will check the mount and refuse to run without it.
-#mash_dir = /mnt/koji/compose/updates/
-{% endif %}
-{% endif %}
 
-# We can remove this if/else once Bodhi 4.0.0 is deployed to production.
-{% if env == "staging" %}
 # The max number of composer threads running at the same time
 # max_concurrent_composes = 2
 max_concurrent_composes = 3
-{% else %}
-# The max number of mash threads running at the same time
-# max_concurrent_mashes = 2
-max_concurrent_mashes = 3
-{% endif %}
 
 # Whether to clean old composes at the end of each run.
 clean_old_composes = false
 
-# We can remove this if/else once Bodhi 4.0.0 is deployed to production.
-{% if env == "staging" %}
 # Where to symlink the latest repos by their tag name. You can use %(here)s to reference the
 # location of this file.
 # compose_stage_dir =
@@ -196,17 +164,6 @@ compose_stage_dir = /mnt/koji/compose/updates/
 # do not use on frontends as bodhi will check the mount and refuse to run without it.
 # compose_stage_dir = /mnt/koji/compose/updates/
 {% endif %}
-{% else %}
-# Where to symlink the latest repos by their tag name. You can use %(here)s to reference the
-# location of this file.
-# mash_stage_dir =
-{% if inventory_hostname.startswith('bodhi-backend') %}
-mash_stage_dir = /mnt/koji/compose/updates/
-{% else %}
-# do not use on frontends as bodhi will check the mount and refuse to run without it.
-#mash_stage_dir = /mnt/koji/compose/updates/
-{% endif %}
-{% endif %}
 
 # The following jinja2 template variables are available for use to customize the Pungi configs and
 # variants files to the Release and Updates:
@@ -720,12 +677,9 @@ pyramid.debug_notfound = false
 pyramid.debug_routematch = false
 pyramid.default_locale_name = en
 
-# We should drop this if statement once we deploy Bodhi 4.0.0 to production.
-{% if env == "staging" %}
 pyramid.includes =
     pyramid_sawing
 pyramid_sawing.file = /etc/bodhi/logging.yaml
-{% endif %}
 
 debugtoolbar.hosts = 127.0.0.1 ::1
 
@@ -797,46 +751,3 @@ port = 6543
 [pshell]
 m = bodhi.server.models
 #db = bodhi.server.util.pshell_db
-
-# We should remove the logging here once Bodhi 4.0.0 is deployed to production - it is configured in
-# /etc/bodhi/logging.yaml instead.
-{% if env == "production" %}
-# Begin logging configuration
-
-[loggers]
-keys = root, bodhi, sqlalchemy
-
-[handlers]
-keys = console
-
-[formatters]
-keys = generic
-
-[logger_root]
-level = INFO
-handlers = console
-
-[logger_bodhi]
-level = DEBUG
-handlers =
-qualname = bodhi
-
-[logger_sqlalchemy]
-level = WARN
-handlers =
-qualname = sqlalchemy.engine
-# "level = INFO" logs SQL queries.
-# "level = DEBUG" logs SQL queries and results.
-# "level = WARN" logs neither.  (Recommended for production systems.)
-
-[handler_console]
-class = StreamHandler
-args = (sys.stderr,)
-level = NOTSET
-formatter = generic
-
-[formatter_generic]
-format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
-
-# End logging configuration
-{% endif %}

roles/bodhi2/base/templates/signed_handler.py

@@ -1,3 +0,0 @@
-config = dict(
-    signed_handler={{bodhi_signed_handler_enabled}},
-)

roles/bodhi2/base/templates/updates_handler.py

@@ -1,5 +0,0 @@
-config = dict(
-    # Note, the masher runs on bodhi-backend03, but this
-    # runs on bodhi-backend02 (separation of concerns).
-    updates_handler={{bodhi_updates_handler_enabled}},
-)

roles/nagios_client/templates/check_fedmsg_consumers.cfg.j2

@@ -10,8 +10,6 @@ command[check_fedmsg_cp_notifs_backend]={{libdir}}/nagios/plugins/check_fedmsg_p
 command[check_fedmsg_cp_bugzilla2fedmsg]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py moksha-hub BugzillaConsumer MonitoringProducer
 command[check_fedmsg_cp_fedimg_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FedimgConsumer MonitoringProducer
 command[check_fedmsg_cp_hotness_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub BugzillaTicketFiler MonitoringProducer
-command[check_fedmsg_cp_bodhi_backend01_hub]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub-3 Masher MonitoringProducer
-command[check_fedmsg_cp_bodhi_backend02_hub]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub-3 UpdatesHandler MonitoringProducer
 command[check_fedmsg_cp_autocloud_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub AutoCloudConsumer MonitoringProducer
 command[check_fedmsg_cp_packages_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub CacheInvalidator MonitoringProducer
 command[check_fedmsg_cp_bugyou_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub BugyouConsumer MonitoringProducer
@@ -29,8 +27,6 @@ command[check_fedmsg_cexceptions_notifs_backend]={{libdir}}/nagios/plugins/check
 command[check_fedmsg_cexceptions_bugzilla2fedmsg]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py moksha-hub BugzillaConsumer 1 10
 command[check_fedmsg_cexceptions_fedimg_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub FedimgConsumer 1 10
 command[check_fedmsg_cexceptions_hotness_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub BugzillaTicketFiler 1 10
-command[check_fedmsg_cexceptions_bodhi_backend01_hub]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub-3 Masher 1 10
-command[check_fedmsg_cexceptions_bodhi_backend02_hub]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub-3 UpdatesHandler 1 10
 command[check_fedmsg_cexceptions_autocloud_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub AutoCloudConsumer 1 10
 command[check_fedmsg_cexceptions_packages_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub CacheInvalidator 1 10
 command[check_fedmsg_cexceptions_bugyou_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub BugyouConsumer 1 10
@@ -48,8 +44,6 @@ command[check_fedmsg_cbacklog_notifs_backend]={{libdir}}/nagios/plugins/check_fe
 command[check_fedmsg_cbacklog_bugzilla2fedmsg]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py moksha-hub BugzillaConsumer 10 100
 command[check_fedmsg_cbacklog_fedimg_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub FedimgConsumer 2000 5000
 command[check_fedmsg_cbacklog_hotness_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub BugzillaTicketFiler 1000 5000
-command[check_fedmsg_cbacklog_bodhi_backend01_hub]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub-3 Masher 500 1000
-command[check_fedmsg_cbacklog_bodhi_backend02_hub]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub-3 UpdatesHandler 500 1000
 command[check_fedmsg_cbacklog_autocloud_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub AutoCloudConsumer 100 500
 command[check_fedmsg_cbacklog_packages_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub CacheInvalidator 30000 40000
 command[check_fedmsg_cbacklog_bugyou_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub BugyouConsumer 5000 10000

roles/nagios_server/files/nagios/services/fedmsg.cfg

@@ -109,20 +109,6 @@ define service {
   use                   defaulttemplate
 }
 
-define service {
-  host_name             bodhi-backend01.phx2.fedoraproject.org
-  service_description   Check for fedmsg-hub proc
-  check_command         check_by_nrpe!check_fedmsg_composer_proc
-  use                   defaulttemplate
-}
-
-define service {
-  host_name             bodhi-backend02.phx2.fedoraproject.org
-  service_description   Check for fedmsg-hub proc
-  check_command         check_by_nrpe!check_fedmsg_composer_proc
-  use                   defaulttemplate
-}
-
 define service {
   host_name             packages03.phx2.fedoraproject.org
   service_description   Check for fedmsg-hub proc
@@ -371,20 +357,6 @@ define service {
   use                   defaulttemplate
 }
 
-define service {
-  host_name             bodhi-backend01.phx2.fedoraproject.org
-  service_description   Check fedmsg consumers and producers hub
-  check_command         check_by_nrpe!check_fedmsg_cp_bodhi_backend01_hub
-  use                   defaulttemplate
-}
-
-define service {
-  host_name             bodhi-backend02.phx2.fedoraproject.org
-  service_description   Check fedmsg consumers and producers hub
-  check_command         check_by_nrpe!check_fedmsg_cp_bodhi_backend02_hub
-  use                   defaulttemplate
-}
-
 define service {
   host_name             packages03.phx2.fedoraproject.org
   service_description   Check fedmsg consumers and producers hub
@@ -468,20 +440,6 @@ define service {
   use                   defaulttemplate
 }
 
-define service {
-  host_name             bodhi-backend01.phx2.fedoraproject.org
-  service_description   Check fedmsg-hub consumers exceptions
-  check_command         check_by_nrpe!check_fedmsg_cexceptions_bodhi_backend01_hub
-  use                   defaulttemplate
-}
-
-define service {
-  host_name             bodhi-backend02.phx2.fedoraproject.org
-  service_description   Check fedmsg-hub consumers exceptions
-  check_command         check_by_nrpe!check_fedmsg_cexceptions_bodhi_backend02_hub
-  use                   defaulttemplate
-}
-
 define service {
   host_name             packages03.phx2.fedoraproject.org
   service_description   Check fedmsg-hub consumers exceptions
@@ -568,20 +526,6 @@ define service {
   use                   defaulttemplate
 }
 
-define service {
-  host_name             bodhi-backend01.phx2.fedoraproject.org
-  service_description   Check fedmsg-hub consumers backlog
-  check_command         check_by_nrpe!check_fedmsg_cbacklog_bodhi_backend01_hub
-  use                   defaulttemplate
-}
-
-define service {
-  host_name             bodhi-backend02.phx2.fedoraproject.org
-  service_description   Check fedmsg-hub consumers backlog
-  check_command         check_by_nrpe!check_fedmsg_cbacklog_bodhi_backend02_hub
-  use                   defaulttemplate
-}
-
 define service {
   host_name             packages03.phx2.fedoraproject.org
   service_description   Check fedmsg-hub consumers backlog

roles/nagios_server/files/nrpe/nrpe.cfg

@@ -400,8 +400,6 @@ command[check_fedmsg_cp_notifs_backend]=/usr/lib64/nagios/plugins/check_fedmsg_p
 command[check_fedmsg_cp_bugzilla2fedmsg]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py moksha-hub BugzillaConsumer MonitoringProducer
 command[check_fedmsg_cp_fedimg_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FedimgConsumer MonitoringProducer
 command[check_fedmsg_cp_hotness_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub BugzillaTicketFiler MonitoringProducer
-command[check_fedmsg_cp_bodhi_backend01_hub]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub-3 Masher MonitoringProducer
-command[check_fedmsg_cp_bodhi_backend02_hub]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub-3 UpdatesHandler MonitoringProducer
 command[check_fedmsg_cp_autocloud_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub AutoCloudConsumer MonitoringProducer
 command[check_fedmsg_cp_packages_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub CacheInvalidator MonitoringProducer
 command[check_fedmsg_cp_bugyou_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub BugyouConsumer MonitoringProducer
@@ -418,8 +416,6 @@ command[check_fedmsg_cexceptions_notifs_backend]=/usr/lib64/nagios/plugins/check
 command[check_fedmsg_cexceptions_bugzilla2fedmsg]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py moksha-hub BugzillaConsumer 1 10
 command[check_fedmsg_cexceptions_fedimg_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub FedimgConsumer 1 10
 command[check_fedmsg_cexceptions_hotness_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub BugzillaTicketFiler 1 10
-command[check_fedmsg_cexceptions_bodhi_backend01_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub-3 Masher 1 10
-command[check_fedmsg_cexceptions_bodhi_backend02_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub-3 UpdatesHandler 1 10
 command[check_fedmsg_cexceptions_autocloud_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub AutoCloudConsumer 1 10
 command[check_fedmsg_cexceptions_packages_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub CacheInvalidator 1 10
 command[check_fedmsg_cexceptions_bugyou_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub BugyouConsumer 1 10
@@ -436,8 +432,6 @@ command[check_fedmsg_cbacklog_notifs_backend]=/usr/lib64/nagios/plugins/check_fe
 command[check_fedmsg_cbacklog_bugzilla2fedmsg]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py moksha-hub BugzillaConsumer 10 100
 command[check_fedmsg_cbacklog_fedimg_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub FedimgConsumer 2000 5000
 command[check_fedmsg_cbacklog_hotness_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub BugzillaTicketFiler 100 500
-command[check_fedmsg_cbacklog_bodhi_backend01_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub-3 Masher 500 1000
-command[check_fedmsg_cbacklog_bodhi_backend02_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub-3 UpdatesHandler 500 1000
 command[check_fedmsg_cbacklog_autocloud_backend_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub AutoCloudConsumer 500 1000
 command[check_fedmsg_cbacklog_packages_backend_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub CacheInvalidator 30000 40000
 command[check_fedmsg_cbacklog_bugyou_backend_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub BugyouConsumer 5000 10000

roles/openshift-apps/bodhi/templates/buildconfig.yml

@@ -22,17 +22,6 @@ items:
         # While dnf has a --nodocs, it doesen't have a --docs...
         RUN sed -i '/nodocs/d' /etc/dnf/dnf.conf
         RUN dnf install -y bodhi-server-{{bodhi_version}}
-        # We can drop this block once Bodhi 4.0.0 is deployed to production.
-{% if env == "production" %}
-        # Put the fedmsg config in place
-        RUN rm -f /etc/fedmsg.d/bodhi.py && \
-        # Let's name it starting with zz so it sorts last.
-            ln -sf /etc/bodhi-fedmsg.d/bodhi-openshift-web.py /etc/fedmsg.d/zzbodhi.py && \
-        # Put the fedmsg cert and key in place
-            mkdir -p /etc/pki/fedmsg/ && \
-            ln -sf /etc/pki/fedmsg/key/fedmsg-bodhi.key /etc/pki/fedmsg/bodhi.key && \
-            ln -sf /etc/pki/fedmsg/crt/fedmsg-bodhi.crt /etc/pki/fedmsg/bodhi.crt
-{% endif %}
         # Set up krb5
         RUN rm -f /etc/krb5.conf && \
             ln -sf /etc/bodhi/krb5.conf /etc/krb5.conf && \
@@ -68,7 +57,6 @@ items:
           vendor="Fedora Infrastructure" \
           license="MIT"
         RUN dnf install -y bodhi-docs-{{bodhi_version}} python3-pyramid_sawing
-        # We can drop this block once Bodhi 4.0.0 is deployed to production.
         EXPOSE 8080
         ENTRYPOINT bash /etc/bodhi/start.sh
       type: Dockerfile
@@ -88,8 +76,6 @@ items:
         name: bodhi-web:latest
 kind: List
 metadata: {}
-{# We can drop the if statement here once we deploy 4.0.0 to production #}
-{% if env == "staging" %}
 ---
 apiVersion: v1
 items:
@@ -126,5 +112,3 @@ items:
         name: bodhi-consumer:latest
 kind: List
 metadata: {}
-{# We can drop this once we deploy 4.0.0 to production #}
-{% endif %}

roles/openshift-apps/bodhi/templates/deploymentconfig.yml

@@ -49,18 +49,6 @@ items:
             readOnly: true
           - name: httpdir-volume
             mountPath: /httpdir
-# We can remove this if/else once we deploy Bodhi 4.0.0 to production.
-{% if env == "production" %}
-          - name: fedmsg-config-volume
-            mountPath: /etc/bodhi-fedmsg.d
-            readOnly: true
-          - name: fedmsg-key-volume
-            mountPath: /etc/pki/fedmsg/key
-            readOnly: true
-          - name: fedmsg-crt-volume
-            mountPath: /etc/pki/fedmsg/crt
-            readOnly: true
-{% else %}
           - name: fedora-messaging-config-volume
             mountPath: /etc/fedora-messaging
             readOnly: true
@@ -76,7 +64,6 @@ items:
             mountPath: /etc/pki/fedora-messaging/bodhi-key.pem
             subPath: bodhi-key.pem
             readOnly: true
-{% endif %}
           readinessProbe:
             timeoutSeconds: 10
             initialDelaySeconds: 5
@@ -98,8 +85,6 @@ items:
             secretName: bodhi-keytab
         - name: httpdir-volume
           emptyDir: {}
-# We can remove this if/else once we deploy Bodhi 4.0.0 to production.
-{% if env == "staging" %}
         - name: fedora-messaging-config-volume
           configMap:
             name: fedora-messaging-configmap
@@ -112,17 +97,6 @@ items:
         - name: fedora-messaging-key-volume
           secret:
             secretName: bodhi-fedora-messaging-key
-{% else %}
-        - name: fedmsg-config-volume
-          configMap:
-            name: fedmsg-configmap
-        - name: fedmsg-key-volume
-          secret:
-            secretName: bodhi-fedmsg-key
-        - name: fedmsg-crt-volume
-          secret:
-            secretName: bodhi-fedmsg-crt
-{% endif %}
     triggers:
     - imageChangeParams:
         automatic: true
@@ -136,8 +110,6 @@ items:
     - type: ConfigChange
 kind: List
 metadata: {}
-# We can remove this if once we deploy Bodhi 4.0.0 to production.
-{% if env == "staging" %}
 ---
 apiVersion: v1
 items:
@@ -233,5 +205,3 @@ items:
     - type: ConfigChange
 kind: List
 metadata: {}
-{# End of the if staging block for this consumer pod #}
-{% endif %}