From b3a4f6fa2f42fcf92dd41cd8b53f75643f17ad6f Mon Sep 17 00:00:00 2001 From: Randy Barlow Date: Thu, 9 May 2019 21:44:44 +0000 Subject: [PATCH] bodhi: Deploy bodhi-4.0.0b0 to staging. A lot of things have changed in Bodhi 4.0.0, so I'm sure this commit isn't quite correct (too bad we can't easily test Ansible code before committing!) This adjusts the playbook to switch staging to Bodhi 4.0.0, which includes switching to fedora-messaging and a new logging config (Bodhi can now send rate-limited error e-mails!) Signed-off-by: Randy Barlow --- playbooks/openshift-apps/bodhi.yml | 33 ++++-- roles/bodhi2/backend/tasks/main.yml | 36 +++++- roles/bodhi2/base/tasks/main.yml | 2 + roles/bodhi2/base/templates/configmap.yml | 23 +++- .../base/templates/fedora-messaging.toml.j2 | 107 ++++++++++++++++++ roles/bodhi2/base/templates/logging.yaml | 40 +++++++ roles/bodhi2/base/templates/production.ini.j2 | 57 +++++++++- .../bodhi/templates/buildconfig.yml | 2 +- .../bodhi/templates/deploymentconfig.yml | 32 ++++++ 9 files changed, 309 insertions(+), 23 deletions(-) create mode 100644 roles/bodhi2/base/templates/fedora-messaging.toml.j2 create mode 100644 roles/bodhi2/base/templates/logging.yaml diff --git a/playbooks/openshift-apps/bodhi.yml b/playbooks/openshift-apps/bodhi.yml index d87ac72120..644738d6fd 100644 --- a/playbooks/openshift-apps/bodhi.yml +++ b/playbooks/openshift-apps/bodhi.yml @@ -27,24 +27,33 @@ secret_name: bodhi-keytab service: bodhi host: "bodhi{{ env_suffix }}.fedoraproject.org" + # When we deploy Bodhi 4.0.0 to production, we can drop the env staging statements on the + # fedora-messaging items below. - role: openshift/secret-file app: bodhi - secret_name: bodhi-fedmsg-key - key: fedmsg-bodhi.key - privatefile: fedmsg-certs/keys/bodhi-bodhi01.stg.phx2.fedoraproject.org.key + secret_name: bodhi-fedora-messaging-ca + key: cacert.pem + privatefile: "rabbitmq/{{env}}/pki/ca.crt" when: env == "staging" + - role: openshift/secret-file + app: bodhi + secret_name: bodhi-fedora-messaging-crt + key: bodhi-cert.pem + privatefile: "rabbitmq/{{env}}/pki/issued/.crt" + when: env == "staging" + - role: openshift/secret-file + app: bodhi + secret_name: bodhi-fedora-messaging-key + key: bodhi-key.pem + privatefile: "rabbitmq/{{env}}/pki/private/bodhi.key" + when: env == "staging" + # When we deploy Bodhi 4.0.0 to production, we can drop these fedmsg secrets. - role: openshift/secret-file app: bodhi secret_name: bodhi-fedmsg-key key: fedmsg-bodhi.key privatefile: fedmsg-certs/keys/bodhi-bodhi-web-temp-bodhi.app.os.fedoraproject.org.key when: env != "staging" - - role: openshift/secret-file - app: bodhi - secret_name: bodhi-fedmsg-crt - key: fedmsg-bodhi.crt - privatefile: fedmsg-certs/keys/bodhi-bodhi01.stg.phx2.fedoraproject.org.crt - when: env == "staging" - role: openshift/secret-file app: bodhi secret_name: bodhi-fedmsg-crt @@ -58,14 +67,14 @@ app: bodhi template: buildconfig.yml objectname: buildconfig.yml - bodhi_version: 3.14.0-1.fc29.infra + bodhi_version: 4.0.0-0.0.beta.fc29.infra when: env == "staging" - role: openshift/object app: bodhi template: buildconfig.yml objectname: buildconfig.yml bodhi_version: 3.14.0-1.fc29.infra - when: env != "staging" + when: env == "production" - role: openshift/start-build app: bodhi buildname: bodhi-web @@ -94,7 +103,7 @@ post_tasks: - name: Scale up pods command: oc -n bodhi scale dc/bodhi-web --replicas={{ hostvars[groups['bodhi2'][0]]['openshift_pods'] }} - when: env != "staging" + when: env == "production" - name: Scale up pods command: oc -n bodhi scale dc/bodhi-web --replicas={{ hostvars[groups['bodhi2-stg'][0]]['openshift_pods'] }} when: env == "staging" diff --git a/roles/bodhi2/backend/tasks/main.yml b/roles/bodhi2/backend/tasks/main.yml index cec79476c7..f571942886 100644 --- a/roles/bodhi2/backend/tasks/main.yml +++ b/roles/bodhi2/backend/tasks/main.yml @@ -1,6 +1,6 @@ --- -# tasklist for setting up bodhi/masher (requires bodhi/base) -# This is the base set of files needed for bodhi/masher +# tasklist for setting up bodhi/composer (requires bodhi/base) +# This is the base set of files needed for bodhi/composer - name: add ftpsync group group: name=ftpsync gid=263 system=yes state=present @@ -185,6 +185,7 @@ - bodhi - cron +# This task can be dropped once Bodhi 4.0.0 is deployed to production. - name: Install bodhi-dequeue-stable cron job that moves updates from batched to stable cron: name: "bodhi-dequeue-stable" @@ -193,7 +194,7 @@ weekday: "*" job: "/usr/bin/bodhi-dequeue-stable && touch /var/lib/bodhi/dequeue-stable" user: "apache" - when: inventory_hostname.startswith('bodhi-backend01.phx2') + when: inventory_hostname.startswith('bodhi-backend01.phx2') and env == "production" tags: - bodhi - cron @@ -324,7 +325,21 @@ - bodhi - cron -- name: setup basic /etc/bodhi/ contents +- name: Install logging.yaml + template: > + src="{{ roles_path }}/bodhi2/base/templates/logging.yaml" + dest="/etc/bodhi/logging.yaml" + owner=apache + group=apache + mode=0600 + when: inventory_hostname.startswith('bodhi-backend') + notify: + - reload bodhi httpd + tags: + - config + - bodhi + +- name: Install production.ini template: > src="{{ roles_path }}/bodhi2/base/templates/production.ini.j2" dest="/etc/bodhi/production.ini" @@ -338,6 +353,19 @@ - config - bodhi +# Remove the env == "staging" condition when we deploy Bodhi 4.0.0 to production. +- name: Install fedora-messaging config + template: > + src="{{ roles_path }}/bodhi2/base/templates/fedora-messaging.toml.j2" + dest="/etc/fedora-messaging/config.toml" + owner=apache + group=apache + mode=0600 + when: inventory_hostname.startswith('bodhi-backend') and env == "staging" + tags: + - config + - bodhi + - name: make a mnt/koji link file: state=link src=/mnt/fedora_koji/koji dest=/mnt/koji tags: diff --git a/roles/bodhi2/base/tasks/main.yml b/roles/bodhi2/base/tasks/main.yml index bc20f89f8e..75d7bfdf6a 100644 --- a/roles/bodhi2/base/tasks/main.yml +++ b/roles/bodhi2/base/tasks/main.yml @@ -28,6 +28,7 @@ - config - bodhi +# When Bodhi 4.0.0 is deployed to production, we should drop this. - name: Copy some fedmsg configuration of our own for fedmsg-hub-3 template: > src={{item}} @@ -42,6 +43,7 @@ - masher.py - updates_handler.py - signed_handler.py + when: env == 'production' notify: - restart fedmsg-hub-3 tags: diff --git a/roles/bodhi2/base/templates/configmap.yml b/roles/bodhi2/base/templates/configmap.yml index 66d716d77c..7d0ed0bad2 100644 --- a/roles/bodhi2/base/templates/configmap.yml +++ b/roles/bodhi2/base/templates/configmap.yml @@ -1,3 +1,4 @@ +{%- macro load_file(filename) %} {% include filename %}{%- endmacro -%} --- apiVersion: v1 kind: ConfigMap @@ -105,9 +106,26 @@ data: WSGIRestrictSignal Off WSGIPythonOptimize 1 WSGIScriptAlias / /usr/share/bodhi/bodhi.wsgi + logging.yaml: |- + {{ load_file('logging.yaml') | indent }} production.ini: |- - {%- macro production_ini() %} {% include "production.ini.j2" %}{%- endmacro -%} - {{ production_ini() | indent() }} + {{ load_file('production.ini.j2') | indent }} +# We can remove this if statement once we deploy Bodhi 4.0.0 to production. +{% if env == "staging" %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: fedora-messaging-configmap + labels: + app: bodhi +data: + fedora-messaging.toml: |- + config.toml: |- + {{ load_file('fedora-messaging.toml.j2') | indent }} +{% endif %} +# We can drop the next ConfigMap once Bodhi 4.0.0 is deployed to production. +{% if env == "production" %} --- apiVersion: v1 kind: ConfigMap @@ -136,3 +154,4 @@ data: 'environment': 'prod', {% endif %} } +{% endif %} diff --git a/roles/bodhi2/base/templates/fedora-messaging.toml.j2 b/roles/bodhi2/base/templates/fedora-messaging.toml.j2 new file mode 100644 index 0000000000..5e85384ab3 --- /dev/null +++ b/roles/bodhi2/base/templates/fedora-messaging.toml.j2 @@ -0,0 +1,107 @@ +amqp_url = "amqps://bodhi{{ env_suffix }}:@rabbitmq{{ env_suffix }}.fedoraproject.org/%2Fpubsub" + +{% if env == "staging" %} +topic_prefix = "org.fedoraproject.stg" +{% else %} +topic_prefix = "org.fedoraproject.prod" +{% endif %} + +publish_exchange = "amq.topic" +passive_declares = false + +callback = "bodhi.server.consumers:messaging_callback" + + +[tls] +ca_cert = "/etc/fedora-messaging/cacert.pem" +keyfile = "/etc/fedora-messaging/bodhi-key.pem" +certfile = "/etc/fedora-messaging/bodhi-cert.pem" + + +[client_properties] +app = "bodhi" + + +[queues."bodhi{{ env_suffix }}"] +durable = true +auto_delete = false +exclusive = false +arguments = {} + + +[[bindings]] +queue = "bodhi{{ env_suffix }}" +exchange = "amq.topic" +routing_keys = [ + "org.fedoraproject.*.bodhi.composer.start", + "org.fedoraproject.*.bodhi.update.request.testing", + "org.fedoraproject.*.buildsys.tag", +] + + +[log_config] +version = 1 +disable_existing_loggers = true + + +[log_config.filters.rate_limit] +() = bodhi.server.logging.RateLimiter +rate = 3600 + + +[log_config.formatters.simple] +format = "[%(levelname)s %(name)s] %(message)s" + + +[log_config.handlers.console] +class = "logging.StreamHandler" +formatter = "simple" +stream = "ext://sys.stdout" + + +[log_config.handlers.email] +class = "logging.handlers.SMTPHandler" +formatter = "simple" +level = "ERROR" +filters = ["rate_limit"] +mailhost = "bastion" +{% if env == "staging" %} +fromaddr = "updates@stg.fedoraproject.org" +{% else %} +fromaddr = "updates@fedoraproject.org" +{% endif %} +toaddrs = ["bodhiadmin-members@fedoraproject.org"] +subject = "Bodhi error" + + +[log_config.loggers.bodhi] +level = "INFO" +propagate = false +handlers = ["console", "email"] + + +[log_config.loggers.fedora_messaging] +level = "INFO" +propagate = false +handlers = ["console", "email"] + + +# Twisted is the asynchronous framework that manages the TCP/TLS connection, as well +# as the consumer event loop. When debugging you may want to lower this log level. +[log_config.loggers.twisted] +level = "INFO" +propagate = false +handlers = ["console", "email"] + + +# Pika is the underlying AMQP client library. When debugging you may want to +# lower this log level. +[log_config.loggers.pika] +level = "WARNING" +propagate = false +handlers = ["console", "email"] + + +[log_config.root] +level = "ERROR" +handlers = ["console", "email"] diff --git a/roles/bodhi2/base/templates/logging.yaml b/roles/bodhi2/base/templates/logging.yaml new file mode 100644 index 0000000000..0b8d276bff --- /dev/null +++ b/roles/bodhi2/base/templates/logging.yaml @@ -0,0 +1,40 @@ +--- +version: 1 + +formatters: + generic: + format: '%(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s' +filters: + rate_limit: + (): bodhi.server.logging.RateLimiter + rate: 3600 +handlers: + console: + class: StreamHandler + level: NOTSET + formatter: generic + smtp: + class: logging.handlers.SMTPHandler + level: ERROR + formatter: generic + filters: [rate_limit] + mailhost: "bastion" +{% if env == "production" %} + fromaddr: "updates@fedoraproject.org" +{% else %} + fromaddr: "updates@stg.fedoraproject.org" +{% endif %} + toaddrs: + - "bodhiadmin-members@fedoraproject.org" + subject: "Bodhi error" +loggers: + bodhi: + level: INFO + handlers: [console, smtp] + propagate: 0 + root: + level: INFO + handlers: [console, smtp] +root: + level: NOTSET + handlers: [] diff --git a/roles/bodhi2/base/templates/production.ini.j2 b/roles/bodhi2/base/templates/production.ini.j2 index 7b75154156..b28f420836 100644 --- a/roles/bodhi2/base/templates/production.ini.j2 +++ b/roles/bodhi2/base/templates/production.ini.j2 @@ -37,14 +37,17 @@ filter-with = proxy-prefix # Set this to true if you want to do federated dns libravatar lookup # libravatar_dns = False -# If libravatar_dns is True, prefer_ssl will define what gets handed to -# libravatar.libravatar_url()'s https setting. It may be set to True or False, but defaults to None, -# which is effectively False. -# prefer_ssl = +# If libravatar_dns is True, libravatar_prefer_tls will define what gets handed to +# libravatar.libravatar_url()'s https setting. Defaults to True. +# libravatar_prefer_tls = +# We can remove this block once Bodhi 4.0.0 is deployed to production. +{% if env != "staging" %} # Set this to True in order to send fedmsg messages. # fedmsg_enabled = False fedmsg_enabled = True +{% endif %} + ## ### Legal @@ -142,8 +145,22 @@ message_id_email_domain = admin{{env_suffix}}.fedoraproject.org ## ## Masher settings ## +# We can remove this block once Bodhi 4.0.0 is deployed to production. +{% if env != "staging" %} releng_fedmsg_certname = shell-bodhi-backend01{{env_suffix}}.phx2.fedoraproject.org +{% endif %} +# We can remove this if/else once Bodhi 4.0.0 is deployed to production. +{% if env == "staging" %} +# Where to initially mash repositories. You can use %(here)s to reference the location of this file. +# compose_dir = +{% if inventory_hostname.startswith('bodhi-backend') %} +compose_dir = /mnt/koji/compose/updates/ +{% else %} +# do not use on frontends as bodhi will check the mount and refuse to run without it. +# compose_dir = /mnt/koji/compose/updates/ +{% endif %} +{% else %} # Where to initially mash repositories. You can use %(here)s to reference the location of this file. # mash_dir = {% if inventory_hostname.startswith('bodhi-backend') %} @@ -152,14 +169,34 @@ mash_dir = /mnt/koji/compose/updates/ # do not use on frontends as bodhi will check the mount and refuse to run without it. #mash_dir = /mnt/koji/compose/updates/ {% endif %} +{% endif %} +# We can remove this if/else once Bodhi 4.0.0 is deployed to production. +{% if env == "staging" %} +# The max number of composer threads running at the same time +# max_concurrent_composes = 2 +max_concurrent_composes = 3 +{% else %} # The max number of mash threads running at the same time # max_concurrent_mashes = 2 max_concurrent_mashes = 3 +{% endif %} # Whether to clean old composes at the end of each run. clean_old_composes = false +# We can remove this if/else once Bodhi 4.0.0 is deployed to production. +{% if env == "staging" %} +# Where to symlink the latest repos by their tag name. You can use %(here)s to reference the +# location of this file. +# compose_stage_dir = +{% if inventory_hostname.startswith('bodhi-backend') %} +compose_stage_dir = /mnt/koji/compose/updates/ +{% else %} +# do not use on frontends as bodhi will check the mount and refuse to run without it. +# compose_stage_dir = /mnt/koji/compose/updates/ +{% endif %} +{% else %} # Where to symlink the latest repos by their tag name. You can use %(here)s to reference the # location of this file. # mash_stage_dir = @@ -169,6 +206,7 @@ mash_stage_dir = /mnt/koji/compose/updates/ # do not use on frontends as bodhi will check the mount and refuse to run without it. #mash_stage_dir = /mnt/koji/compose/updates/ {% endif %} +{% endif %} # The following jinja2 template variables are available for use to customize the Pungi configs and # variants files to the Release and Updates: @@ -679,6 +717,13 @@ pyramid.debug_notfound = false pyramid.debug_routematch = false pyramid.default_locale_name = en +# We should drop this if statement once we deploy Bodhi 4.0.0 to production. +{% if env == "staging" %} +pyramid.includes = + pyramid_sawing +pyramid_sawing.file = /etc/bodhi/logging.yaml +{% endif %} + debugtoolbar.hosts = 127.0.0.1 ::1 ## @@ -750,6 +795,9 @@ port = 6543 m = bodhi.server.models #db = bodhi.server.util.pshell_db +# We should remove the logging here once Bodhi 4.0.0 is deployed to production - it is configured in +# /etc/bodhi/logging.yaml instead. +{% if env == "production" %} # Begin logging configuration [loggers] @@ -788,3 +836,4 @@ formatter = generic format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s # End logging configuration +{% endif %} diff --git a/roles/openshift-apps/bodhi/templates/buildconfig.yml b/roles/openshift-apps/bodhi/templates/buildconfig.yml index 013798e732..3e83ba6ba1 100644 --- a/roles/openshift-apps/bodhi/templates/buildconfig.yml +++ b/roles/openshift-apps/bodhi/templates/buildconfig.yml @@ -21,7 +21,7 @@ items: {% endif %} # While dnf has a --nodocs, it doesen't have a --docs... RUN sed -i '/nodocs/d' /etc/dnf/dnf.conf - RUN dnf install -y bodhi-server-{{bodhi_version}} bodhi-docs-{{bodhi_version}} + RUN dnf install -y bodhi-server-{{bodhi_version}} bodhi-docs-{{bodhi_version}} python3-pyramid_sawing # Put the fedmsg config in place RUN rm -f /etc/fedmsg.d/bodhi.py && \ # Let's name it starting with zz so it sorts last. diff --git a/roles/openshift-apps/bodhi/templates/deploymentconfig.yml b/roles/openshift-apps/bodhi/templates/deploymentconfig.yml index 0eaa3fb248..8b35bd3fb6 100644 --- a/roles/openshift-apps/bodhi/templates/deploymentconfig.yml +++ b/roles/openshift-apps/bodhi/templates/deploymentconfig.yml @@ -49,6 +49,8 @@ items: readOnly: true - name: httpdir-volume mountPath: /httpdir +# We can remove this if/else once we deploy Bodhi 4.0.0 to production. +{% if env == "production" %} - name: fedmsg-config-volume mountPath: /etc/bodhi-fedmsg.d readOnly: true @@ -58,6 +60,20 @@ items: - name: fedmsg-crt-volume mountPath: /etc/pki/fedmsg/crt readOnly: true +{% else %} + - name: fedora-messaging-config-volume + mountPath: /etc/fedora-messaging + readOnly: true + - name: fedora-messaging-ca-volume + mountPath: /etc/fedora-messaging/cacert.pem + readOnly: true + - name: fedora-messaging-crt-volume + mountPath: /etc/fedora-messaging/bodhi-cert.pem + readOnly: true + - name: fedora-messaging-key-volume + mountPath: /etc/fedora-messaging/bodhi-key.pem + readOnly: true +{% endif %} readinessProbe: timeoutSeconds: 10 initialDelaySeconds: 5 @@ -79,6 +95,21 @@ items: secretName: bodhi-keytab - name: httpdir-volume emptyDir: {} +# We can remove this if/else once we deploy Bodhi 4.0.0 to production. +{% if env == "staging" %} + - name: fedora-messaging-config-volume + configMap: + name: fedora-messaging-configmap + - name: fedora-messaging-ca-volume + secret: + secretName: bodhi-fedora-messaging-ca + - name: fedora-messaging-crt-volume + secret: + secretName: bodhi-fedora-messaging-crt + - name: fedora-messaging-key-volume + secret: + secretName: bodhi-fedora-messaging-key +{% else %} - name: fedmsg-config-volume configMap: name: fedmsg-configmap @@ -88,6 +119,7 @@ items: - name: fedmsg-crt-volume secret: secretName: bodhi-fedmsg-crt +{% endif %} triggers: - imageChangeParams: automatic: true