bodhi: Deploy bodhi-4.0.0b0 to staging.

A lot of things have changed in Bodhi 4.0.0, so I'm sure this
commit isn't quite correct (too bad we can't easily test Ansible
code before committing!)

This adjusts the playbook to switch staging to Bodhi 4.0.0, which
includes switching to fedora-messaging and a new logging config
(Bodhi can now send rate-limited error e-mails!)

Signed-off-by: Randy Barlow <randy@electronsweatshop.com>

playbooks/openshift-apps/bodhi.yml

@@ -27,24 +27,33 @@
     secret_name: bodhi-keytab
     service: bodhi
     host: "bodhi{{ env_suffix }}.fedoraproject.org"
+  # When we deploy Bodhi 4.0.0 to production, we can drop the env staging statements on the
+  # fedora-messaging items below.
   - role: openshift/secret-file
     app: bodhi
-    secret_name: bodhi-fedmsg-key
+    secret_name: bodhi-fedora-messaging-ca
-    key: fedmsg-bodhi.key
+    key: cacert.pem
-    privatefile: fedmsg-certs/keys/bodhi-bodhi01.stg.phx2.fedoraproject.org.key
+    privatefile: "rabbitmq/{{env}}/pki/ca.crt"
     when: env == "staging"
+  - role: openshift/secret-file
+    app: bodhi
+    secret_name: bodhi-fedora-messaging-crt
+    key: bodhi-cert.pem
+    privatefile: "rabbitmq/{{env}}/pki/issued/<app>.crt"
+    when: env == "staging"
+  - role: openshift/secret-file
+    app: bodhi
+    secret_name: bodhi-fedora-messaging-key
+    key: bodhi-key.pem
+    privatefile: "rabbitmq/{{env}}/pki/private/bodhi.key"
+    when: env == "staging"
+  # When we deploy Bodhi 4.0.0 to production, we can drop these fedmsg secrets.
   - role: openshift/secret-file
     app: bodhi
     secret_name: bodhi-fedmsg-key
     key: fedmsg-bodhi.key
     privatefile: fedmsg-certs/keys/bodhi-bodhi-web-temp-bodhi.app.os.fedoraproject.org.key
     when: env != "staging"
-  - role: openshift/secret-file
-    app: bodhi
-    secret_name: bodhi-fedmsg-crt
-    key: fedmsg-bodhi.crt
-    privatefile: fedmsg-certs/keys/bodhi-bodhi01.stg.phx2.fedoraproject.org.crt
-    when: env == "staging"
   - role: openshift/secret-file
     app: bodhi
     secret_name: bodhi-fedmsg-crt
@@ -58,14 +67,14 @@
     app: bodhi
     template: buildconfig.yml
     objectname: buildconfig.yml
-    bodhi_version: 3.14.0-1.fc29.infra
+    bodhi_version: 4.0.0-0.0.beta.fc29.infra
     when: env == "staging"
   - role: openshift/object
     app: bodhi
     template: buildconfig.yml
     objectname: buildconfig.yml
     bodhi_version: 3.14.0-1.fc29.infra
-    when: env != "staging"
+    when: env == "production"
   - role: openshift/start-build
     app: bodhi
     buildname: bodhi-web
@@ -94,7 +103,7 @@
   post_tasks:
   - name: Scale up pods
     command: oc -n bodhi scale dc/bodhi-web --replicas={{ hostvars[groups['bodhi2'][0]]['openshift_pods'] }}
-    when: env != "staging"
+    when: env == "production"
   - name: Scale up pods
     command: oc -n bodhi scale dc/bodhi-web --replicas={{ hostvars[groups['bodhi2-stg'][0]]['openshift_pods'] }}
     when: env == "staging"

roles/bodhi2/backend/tasks/main.yml

@@ -1,6 +1,6 @@
 ---
-# tasklist for setting up bodhi/masher (requires bodhi/base)
+# tasklist for setting up bodhi/composer (requires bodhi/base)
-# This is the base set of files needed for bodhi/masher
+# This is the base set of files needed for bodhi/composer
 
 - name: add ftpsync group
   group: name=ftpsync gid=263 system=yes state=present
@@ -185,6 +185,7 @@
   - bodhi
   - cron
 
+# This task can be dropped once Bodhi 4.0.0 is deployed to production.
 - name: Install bodhi-dequeue-stable cron job that moves updates from batched to stable
   cron:
     name: "bodhi-dequeue-stable"
@@ -193,7 +194,7 @@
     weekday: "*"
     job: "/usr/bin/bodhi-dequeue-stable && touch /var/lib/bodhi/dequeue-stable"
     user: "apache"
-  when: inventory_hostname.startswith('bodhi-backend01.phx2')
+  when: inventory_hostname.startswith('bodhi-backend01.phx2') and env == "production"
   tags:
   - bodhi
   - cron
@@ -324,7 +325,21 @@
   - bodhi
   - cron
 
-- name: setup basic /etc/bodhi/ contents
+- name: Install logging.yaml
+  template: >
+    src="{{  roles_path  }}/bodhi2/base/templates/logging.yaml"
+    dest="/etc/bodhi/logging.yaml"
+    owner=apache
+    group=apache
+    mode=0600
+  when: inventory_hostname.startswith('bodhi-backend')
+  notify:
+  - reload bodhi httpd
+  tags:
+  - config
+  - bodhi
+
+- name: Install production.ini
   template: >
     src="{{  roles_path  }}/bodhi2/base/templates/production.ini.j2"
     dest="/etc/bodhi/production.ini"
@@ -338,6 +353,19 @@
   - config
   - bodhi
 
+# Remove the env == "staging" condition when we deploy Bodhi 4.0.0 to production.
+- name: Install fedora-messaging config
+  template: >
+    src="{{  roles_path  }}/bodhi2/base/templates/fedora-messaging.toml.j2"
+    dest="/etc/fedora-messaging/config.toml"
+    owner=apache
+    group=apache
+    mode=0600
+  when: inventory_hostname.startswith('bodhi-backend') and env == "staging"
+  tags:
+  - config
+  - bodhi
+
 - name: make a mnt/koji link
   file: state=link src=/mnt/fedora_koji/koji dest=/mnt/koji
   tags:

roles/bodhi2/base/tasks/main.yml

@@ -28,6 +28,7 @@
   - config
   - bodhi
 
+# When Bodhi 4.0.0 is deployed to production, we should drop this.
 - name: Copy some fedmsg configuration of our own for fedmsg-hub-3
   template: >
       src={{item}}
@@ -42,6 +43,7 @@
   - masher.py
   - updates_handler.py
   - signed_handler.py
+  when: env == 'production'
   notify:
   - restart fedmsg-hub-3
   tags:

roles/bodhi2/base/templates/configmap.yml

@@ -1,3 +1,4 @@
+{%- macro load_file(filename) %}    {% include filename %}{%- endmacro -%}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -105,9 +106,26 @@ data:
     WSGIRestrictSignal Off
     WSGIPythonOptimize 1
     WSGIScriptAlias / /usr/share/bodhi/bodhi.wsgi
+  logging.yaml: |-
+    {{ load_file('logging.yaml') | indent }}
   production.ini: |-
-    {%- macro production_ini() %}    {% include "production.ini.j2" %}{%- endmacro -%}
+    {{ load_file('production.ini.j2') | indent }}
-    {{ production_ini() | indent() }}
+# We can remove this if statement once we deploy Bodhi 4.0.0 to production.
+{% if env == "staging" %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fedora-messaging-configmap
+  labels:
+    app: bodhi
+data:
+  fedora-messaging.toml: |-
+  config.toml: |-
+    {{ load_file('fedora-messaging.toml.j2') | indent }}
+{% endif %}
+# We can drop the next ConfigMap once Bodhi 4.0.0 is deployed to production.
+{% if env == "production" %}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -136,3 +154,4 @@ data:
         'environment': 'prod',
 {% endif %}
     }
+{% endif %}

roles/bodhi2/base/templates/fedora-messaging.toml.j2

@@ -0,0 +1,107 @@
+amqp_url = "amqps://bodhi{{ env_suffix }}:@rabbitmq{{ env_suffix }}.fedoraproject.org/%2Fpubsub"
+
+{% if env == "staging" %}
+topic_prefix = "org.fedoraproject.stg"
+{% else %}
+topic_prefix = "org.fedoraproject.prod"
+{% endif %}
+
+publish_exchange = "amq.topic"
+passive_declares = false
+
+callback = "bodhi.server.consumers:messaging_callback"
+
+
+[tls]
+ca_cert = "/etc/fedora-messaging/cacert.pem"
+keyfile = "/etc/fedora-messaging/bodhi-key.pem"
+certfile = "/etc/fedora-messaging/bodhi-cert.pem"
+
+
+[client_properties]
+app = "bodhi"
+
+
+[queues."bodhi{{ env_suffix }}"]
+durable = true
+auto_delete = false
+exclusive = false
+arguments = {}
+
+
+[[bindings]]
+queue = "bodhi{{ env_suffix }}"
+exchange = "amq.topic"
+routing_keys = [
+    "org.fedoraproject.*.bodhi.composer.start",
+    "org.fedoraproject.*.bodhi.update.request.testing",
+    "org.fedoraproject.*.buildsys.tag",
+]
+
+
+[log_config]
+version = 1
+disable_existing_loggers = true
+
+
+[log_config.filters.rate_limit]
+() = bodhi.server.logging.RateLimiter
+rate = 3600
+
+
+[log_config.formatters.simple]
+format = "[%(levelname)s %(name)s] %(message)s"
+
+
+[log_config.handlers.console]
+class = "logging.StreamHandler"
+formatter = "simple"
+stream = "ext://sys.stdout"
+
+
+[log_config.handlers.email]
+class = "logging.handlers.SMTPHandler"
+formatter = "simple"
+level = "ERROR"
+filters = ["rate_limit"]
+mailhost = "bastion"
+{% if env == "staging" %}
+fromaddr = "updates@stg.fedoraproject.org"
+{% else %}
+fromaddr = "updates@fedoraproject.org"
+{% endif %}
+toaddrs = ["bodhiadmin-members@fedoraproject.org"]
+subject = "Bodhi error"
+
+
+[log_config.loggers.bodhi]
+level = "INFO"
+propagate = false
+handlers = ["console", "email"]
+
+
+[log_config.loggers.fedora_messaging]
+level = "INFO"
+propagate = false
+handlers = ["console", "email"]
+
+
+# Twisted is the asynchronous framework that manages the TCP/TLS connection, as well
+# as the consumer event loop. When debugging you may want to lower this log level.
+[log_config.loggers.twisted]
+level = "INFO"
+propagate = false
+handlers = ["console", "email"]
+
+
+# Pika is the underlying AMQP client library. When debugging you may want to
+# lower this log level.
+[log_config.loggers.pika]
+level = "WARNING"
+propagate = false
+handlers = ["console", "email"]
+
+
+[log_config.root]
+level = "ERROR"
+handlers = ["console", "email"]

roles/bodhi2/base/templates/logging.yaml

@@ -0,0 +1,40 @@
+---
+version: 1
+
+formatters:
+  generic:
+    format: '%(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s'
+filters:
+  rate_limit:
+    (): bodhi.server.logging.RateLimiter
+    rate: 3600
+handlers:
+  console:
+    class: StreamHandler
+    level: NOTSET
+    formatter: generic
+  smtp:
+    class: logging.handlers.SMTPHandler
+    level: ERROR
+    formatter: generic
+    filters: [rate_limit]
+    mailhost: "bastion"
+{% if env == "production" %}
+    fromaddr: "updates@fedoraproject.org"
+{% else %}
+    fromaddr: "updates@stg.fedoraproject.org"
+{% endif %}
+    toaddrs:
+        - "bodhiadmin-members@fedoraproject.org"
+    subject: "Bodhi error"
+loggers:
+  bodhi:
+    level: INFO
+    handlers: [console, smtp]
+    propagate: 0
+  root:
+    level: INFO
+    handlers: [console, smtp]
+root:
+  level: NOTSET
+  handlers: []

roles/bodhi2/base/templates/production.ini.j2

@@ -37,14 +37,17 @@ filter-with = proxy-prefix
 # Set this to true if you want to do federated dns libravatar lookup
 # libravatar_dns = False
 
-# If libravatar_dns is True, prefer_ssl will define what gets handed to
+# If libravatar_dns is True, libravatar_prefer_tls will define what gets handed to
-# libravatar.libravatar_url()'s https setting. It may be set to True or False, but defaults to None,
+# libravatar.libravatar_url()'s https setting. Defaults to True.
-# which is effectively False.
+# libravatar_prefer_tls =
-# prefer_ssl =
 
+# We can remove this block once Bodhi 4.0.0 is deployed to production.
+{% if env != "staging" %}
 # Set this to True in order to send fedmsg messages.
 # fedmsg_enabled = False
 fedmsg_enabled = True
+{% endif %}
+
 
 ##
 ### Legal
@@ -142,8 +145,22 @@ message_id_email_domain = admin{{env_suffix}}.fedoraproject.org
 ##
 ## Masher settings
 ##
+# We can remove this block once Bodhi 4.0.0 is deployed to production.
+{% if env != "staging" %}
 releng_fedmsg_certname = shell-bodhi-backend01{{env_suffix}}.phx2.fedoraproject.org
+{% endif %}
 
+# We can remove this if/else once Bodhi 4.0.0 is deployed to production.
+{% if env == "staging" %}
+# Where to initially mash repositories. You can use %(here)s to reference the location of this file.
+# compose_dir =
+{% if inventory_hostname.startswith('bodhi-backend') %}
+compose_dir = /mnt/koji/compose/updates/
+{% else %}
+# do not use on frontends as bodhi will check the mount and refuse to run without it.
+# compose_dir = /mnt/koji/compose/updates/
+{% endif %}
+{% else %}
 # Where to initially mash repositories. You can use %(here)s to reference the location of this file.
 # mash_dir =
 {% if inventory_hostname.startswith('bodhi-backend') %}
@@ -152,14 +169,34 @@ mash_dir = /mnt/koji/compose/updates/
 # do not use on frontends as bodhi will check the mount and refuse to run without it.
 #mash_dir = /mnt/koji/compose/updates/
 {% endif %}
+{% endif %}
 
+# We can remove this if/else once Bodhi 4.0.0 is deployed to production.
+{% if env == "staging" %}
+# The max number of composer threads running at the same time
+# max_concurrent_composes = 2
+max_concurrent_composes = 3
+{% else %}
 # The max number of mash threads running at the same time
 # max_concurrent_mashes = 2
 max_concurrent_mashes = 3
+{% endif %}
 
 # Whether to clean old composes at the end of each run.
 clean_old_composes = false
 
+# We can remove this if/else once Bodhi 4.0.0 is deployed to production.
+{% if env == "staging" %}
+# Where to symlink the latest repos by their tag name. You can use %(here)s to reference the
+# location of this file.
+# compose_stage_dir =
+{% if inventory_hostname.startswith('bodhi-backend') %}
+compose_stage_dir = /mnt/koji/compose/updates/
+{% else %}
+# do not use on frontends as bodhi will check the mount and refuse to run without it.
+# compose_stage_dir = /mnt/koji/compose/updates/
+{% endif %}
+{% else %}
 # Where to symlink the latest repos by their tag name. You can use %(here)s to reference the
 # location of this file.
 # mash_stage_dir =
@@ -169,6 +206,7 @@ mash_stage_dir = /mnt/koji/compose/updates/
 # do not use on frontends as bodhi will check the mount and refuse to run without it.
 #mash_stage_dir = /mnt/koji/compose/updates/
 {% endif %}
+{% endif %}
 
 # The following jinja2 template variables are available for use to customize the Pungi configs and
 # variants files to the Release and Updates:
@@ -679,6 +717,13 @@ pyramid.debug_notfound = false
 pyramid.debug_routematch = false
 pyramid.default_locale_name = en
 
+# We should drop this if statement once we deploy Bodhi 4.0.0 to production.
+{% if env == "staging" %}
+pyramid.includes =
+    pyramid_sawing
+pyramid_sawing.file = /etc/bodhi/logging.yaml
+{% endif %}
+
 debugtoolbar.hosts = 127.0.0.1 ::1
 
 ##
@@ -750,6 +795,9 @@ port = 6543
 m = bodhi.server.models
 #db = bodhi.server.util.pshell_db
 
+# We should remove the logging here once Bodhi 4.0.0 is deployed to production - it is configured in
+# /etc/bodhi/logging.yaml instead.
+{% if env == "production" %}
 # Begin logging configuration
 
 [loggers]
@@ -788,3 +836,4 @@ formatter = generic
 format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
 
 # End logging configuration
+{% endif %}

roles/openshift-apps/bodhi/templates/buildconfig.yml

@@ -21,7 +21,7 @@ items:
 {% endif %}
         # While dnf has a --nodocs, it doesen't have a --docs...
         RUN sed -i '/nodocs/d' /etc/dnf/dnf.conf
-        RUN dnf install -y bodhi-server-{{bodhi_version}} bodhi-docs-{{bodhi_version}}
+        RUN dnf install -y bodhi-server-{{bodhi_version}} bodhi-docs-{{bodhi_version}} python3-pyramid_sawing
         # Put the fedmsg config in place
         RUN rm -f /etc/fedmsg.d/bodhi.py && \
         # Let's name it starting with zz so it sorts last.

roles/openshift-apps/bodhi/templates/deploymentconfig.yml

@@ -49,6 +49,8 @@ items:
             readOnly: true
           - name: httpdir-volume
             mountPath: /httpdir
+# We can remove this if/else once we deploy Bodhi 4.0.0 to production.
+{% if env == "production" %}
           - name: fedmsg-config-volume
             mountPath: /etc/bodhi-fedmsg.d
             readOnly: true
@@ -58,6 +60,20 @@ items:
           - name: fedmsg-crt-volume
             mountPath: /etc/pki/fedmsg/crt
             readOnly: true
+{% else %}
+          - name: fedora-messaging-config-volume
+            mountPath: /etc/fedora-messaging
+            readOnly: true
+          - name: fedora-messaging-ca-volume
+            mountPath: /etc/fedora-messaging/cacert.pem
+            readOnly: true
+          - name: fedora-messaging-crt-volume
+            mountPath: /etc/fedora-messaging/bodhi-cert.pem
+            readOnly: true
+          - name: fedora-messaging-key-volume
+            mountPath: /etc/fedora-messaging/bodhi-key.pem
+            readOnly: true
+{% endif %}
           readinessProbe:
             timeoutSeconds: 10
             initialDelaySeconds: 5
@@ -79,6 +95,21 @@ items:
             secretName: bodhi-keytab
         - name: httpdir-volume
           emptyDir: {}
+# We can remove this if/else once we deploy Bodhi 4.0.0 to production.
+{% if env == "staging" %}
+        - name: fedora-messaging-config-volume
+          configMap:
+            name: fedora-messaging-configmap
+        - name: fedora-messaging-ca-volume
+          secret:
+            secretName: bodhi-fedora-messaging-ca
+        - name: fedora-messaging-crt-volume
+          secret:
+            secretName: bodhi-fedora-messaging-crt
+        - name: fedora-messaging-key-volume
+          secret:
+            secretName: bodhi-fedora-messaging-key
+{% else %}
         - name: fedmsg-config-volume
           configMap:
             name: fedmsg-configmap
@@ -88,6 +119,7 @@ items:
         - name: fedmsg-crt-volume
           secret:
             secretName: bodhi-fedmsg-crt
+{% endif %}
     triggers:
     - imageChangeParams:
         automatic: true