Update Bodhi to 5.0 in staging

Signed-off-by: Aurélien Bompard <aurelien@bompard.org>

playbooks/manual/staging-sync/bodhi.yml

@@ -23,6 +23,7 @@
   tasks:
   - command: oc -n bodhi scale dc/bodhi-web --replicas=0
   - command: oc -n bodhi scale dc/bodhi-consumer --replicas=0
+  - command: oc -n bodhi scale dc/bodhi-celery --replicas=0
 
 - name: bring staging services down (messaging)
   hosts: bodhi_backend_stg
@@ -74,6 +75,7 @@
   tasks:
   - command: oc -n bodhi scale dc/bodhi-web --replicas={{ hostvars[groups['bodhi2_stg'][0]]['openshift_pods'] }}
   - command: oc -n bodhi scale dc/bodhi-consumer --replicas={{ hostvars[groups['bodhi2_stg'][0]]['openshift_pods'] }}
+  - command: oc -n bodhi scale dc/bodhi-celery --replicas={{ hostvars[groups['bodhi2_stg'][0]]['openshift_pods'] }}
 
 - name: bring staging services up (httpd)
   hosts: bodhi2_stg

playbooks/openshift-apps/bodhi.yml

@@ -69,11 +69,14 @@
   - role: openshift/imagestream
     app: bodhi
     imagename: bodhi-consumer
+  - role: openshift/imagestream
+    app: bodhi
+    imagename: bodhi-celery
   - role: openshift/object
     app: bodhi
     template: buildconfig.yml
     objectname: buildconfig.yml
-    bodhi_version: 4.1.1-1.fc29.infra
+    bodhi_version: 5.0.0-0.beta.1.761b898.fc29.infra
     when: env == "staging"
   - role: openshift/object
     app: bodhi
@@ -108,17 +111,26 @@
   - role: openshift/rollout
     app: bodhi
     dcname: bodhi-consumer
+  - role: openshift/rollout
+    app: bodhi
+    dcname: bodhi-celery
 
   post_tasks:
   - name: Scale up pods
     command: oc -n bodhi scale dc/bodhi-consumer --replicas={{ hostvars[groups['bodhi2'][0]]['openshift_pods'] }}
     when: env == "production"
+  - name: Scale up pods
+    command: oc -n bodhi scale dc/bodhi-celery --replicas={{ hostvars[groups['bodhi2'][0]]['openshift_pods'] }}
+    when: env == "production"
   - name: Scale up pods
     command: oc -n bodhi scale dc/bodhi-web --replicas={{ hostvars[groups['bodhi2'][0]]['openshift_pods'] }}
     when: env == "production"
   - name: Scale up pods
     command: oc -n bodhi scale dc/bodhi-consumer --replicas={{ hostvars[groups['bodhi2_stg'][0]]['openshift_pods'] }}
     when: env == "staging"
+  - name: Scale up pods
+    command: oc -n bodhi scale dc/bodhi-celery --replicas={{ hostvars[groups['bodhi2_stg'][0]]['openshift_pods'] }}
+    when: env == "staging"
   - name: Scale up pods
     command: oc -n bodhi scale dc/bodhi-web --replicas={{ hostvars[groups['bodhi2_stg'][0]]['openshift_pods'] }}
     when: env == "staging"

roles/bodhi2/backend/files/bodhi-celery.service

@@ -0,0 +1,12 @@
+[Unit]
+Description = Bodhi's Celery worker
+After = network-online.target
+Wants = network-online.target
+
+[Service]
+User = apache
+Group = apache
+ExecStart = /usr/bin/celery-3 worker -A bodhi.server.tasks.app -l info -Q celery,has_koji_mount
+
+[Install]
+WantedBy = multi-user.target

roles/bodhi2/backend/tasks/main.yml

@@ -18,6 +18,10 @@
       - bodhi-composer
       - python3-pyramid_sawing
       - sigul
+      # This next one is not strictly needed since bodhi-composer depends on bodhi-server which
+      # depends on python3-celery, but since we're depoying a service file calling celery in this
+      # role I think it makes sense to require it here instead of assuming a dependency chain.
+      - python3-celery
       # This is used to generate zchunk data more efficiently
       - fedora-repo-zdicts
       # The new-updates-sync script uses this
@@ -217,23 +221,34 @@
   - bodhi
 
 - name: Install production.ini
-  template: >
+  template:
-    src="{{  roles_path  }}/bodhi2/base/templates/production.ini.j2"
+    src: "{{ roles_path }}/bodhi2/base/templates/production.ini.j2"
-    dest="/etc/bodhi/production.ini"
+    dest: /etc/bodhi/production.ini
-    owner=apache
+    owner: apache
-    group=apache
+    group: apache
-    mode=0600
+    mode: 0600
+  tags:
+  - config
+  - bodhi
+
+- name: Install celeryconfig.py
+  template:
+    src: "{{ roles_path }}/bodhi2/base/templates/celeryconfig.py.j2"
+    dest: /etc/bodhi/celeryconfig.py
+    owner: apache
+    group: apache
+    mode: 0600
   tags:
   - config
   - bodhi
 
 - name: Install fedora-messaging config
-  template: >
+  template:
-    src="{{  roles_path  }}/bodhi2/base/templates/fedora-messaging.toml.j2"
+    src: "{{ roles_path }}/bodhi2/base/templates/fedora-messaging.toml.j2"
-    dest="/etc/fedora-messaging/config.toml"
+    dest: /etc/fedora-messaging/config.toml
-    owner=apache
+    owner: apache
-    group=apache
+    group: apache
-    mode=0600
+    mode: 0600
   tags:
   - config
   - bodhi
@@ -267,6 +282,19 @@
   - bodhi
   - config
 
+- name: Setup the Celery service
+  copy:
+    src: bodhi-celery.service
+    dest: /etc/systemd/system/bodhi-celery.service
+    owner: root
+    group: root
+    mode: 0644
+  notify:
+  - reload systemd
+  tags:
+  - bodhi
+  - config
+
 - name: ensure apache is disabled on the backend
   service: name=httpd enabled=no state=stopped
   tags:
@@ -291,7 +319,7 @@
     group: apache
   tags:
   - bodhi
-  
+
 - name: Deploy the fedora-messaging cert
   copy:
     src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/bodhi{{env_suffix}}.crt"
@@ -301,7 +329,7 @@
     group: apache
   tags:
   - bodhi
-  
+
 - name: Deploy the fedora-messaging key
   copy:
     src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/bodhi{{env_suffix}}.key"
@@ -311,9 +339,15 @@
     group: apache
   tags:
   - bodhi
-  
+
-- name: ensure fedora-messaging is enabled and started on the backend
+- name: ensure fedora-messaging and celery are enabled and started on the backend
-  service: name=fm-consumer@config.service enabled=yes state=started
+  service:
+    name: "{{ item }}"
+    enabled: yes
+    state: started
+  with_items:
+  - fm-consumer@config
+  - bodhi-celery
   tags:
   - bodhi
 

roles/bodhi2/base/tasks/main.yml

@@ -11,3 +11,77 @@
   tags:
   - config
   - bodhi
+
+# Bodhi virtualhost in RabbitMQ
+
+- name: Configure the bodhi virtual host
+  run_once: true
+  delegate_to: "rabbitmq01{{ env_suffix }}.phx2.fedoraproject.org"
+  rabbitmq_vhost:
+    name: /bodhi
+    state: present
+  tags:
+  - rabbitmq_cluster
+  - config
+  - bodhi
+
+- name: Configure the HA policy for the bodhi queues
+  run_once: true
+  delegate_to: "rabbitmq01{{ env_suffix }}.phx2.fedoraproject.org"
+  rabbitmq_policy:
+    name: HA
+    apply_to: queues
+    pattern: .*
+    tags:
+      ha-mode: all
+      ha-sync-mode: automatic  # Auto sync queues to new cluster members
+      ha-sync-batch-size: 10000  # Larger is faster, but must finish in 1 net_ticktime
+    vhost: /bodhi
+  tags:
+  - rabbitmq_cluster
+  - config
+  - bodhi
+
+- name: Grant the admin user access to the bodhi vhost
+  run_once: true
+  delegate_to: "rabbitmq01{{ env_suffix }}.phx2.fedoraproject.org"
+  rabbitmq_user:
+    user: admin
+    vhost: /bodhi
+    configure_priv: .*
+    read_priv: .*
+    write_priv: .*
+  tags:
+  - rabbitmq_cluster
+  - config
+  - bodhi
+
+# Create a user for Celery
+# - name: Create a user for Celery usage
+#   run_once: true
+#   delegate_to: "rabbitmq01{{ env_suffix }}.phx2.fedoraproject.org"
+#   rabbitmq_user:
+#     user: "bodhi-celery{{ env_suffix }}"
+#     vhost: /bodhi
+#     configure_priv: .*
+#     write_priv: .*
+#     read_priv: .*
+#     state: present
+#   tags:
+#   - rabbitmq_cluster
+#   - config
+#   - bodhi
+
+- name: Grant the bodhi user access to the bodhi vhost
+  run_once: true
+  delegate_to: "rabbitmq01{{ env_suffix }}.phx2.fedoraproject.org"
+  rabbitmq_user:
+    user: "bodhi{{ env_suffix }}"
+    vhost: /bodhi
+    configure_priv: .*
+    read_priv: .*
+    write_priv: .*
+  tags:
+  - rabbitmq_cluster
+  - config
+  - bodhi

roles/bodhi2/base/templates/celeryconfig.py.j2

@@ -0,0 +1,29 @@
+#
+# Celery configuration file
+# See: docs.celeryproject.org/en/latest/userguide/configuration.html
+#
+
+# Broker URL
+# This might be more appropriate in prod:
+# broker_url = amqps://user:password@hostname:port//vhost
+# broker_use_ssl =
+#   keyfile=/var/ssl/private/worker-key.pem
+#   certfile=/var/ssl/amqp-server-cert.pem
+#   ca_certs=/var/ssl/myca.pem
+#   cert_reqs=ssl.CERT_REQUIRED
+broker_url = "amqps://bodhi{{ env_suffix }}:@rabbitmq{{ env_suffix }}.fedoraproject.org//bodhi"
+broker_use_ssl =
+    ca_certs=/etc/pki/fedora-messaging/cacert.pem
+    keyfile=/etc/pki/fedora-messaging/bodhi-key.pem
+    certfile=/etc/pki/fedora-messaging/bodhi-cert.pem
+    cert_reqs=ssl.CERT_REQUIRED
+
+# Where the tasks are defined
+imports = "bodhi.server.tasks"
+
+# Task routing
+task_routes = {
+    # Route the compose task to a specific queue that will only be run on hosts
+    # that have a Koji mount.
+    'compose': {'queue': 'has_koji_mount'},
+}

roles/bodhi2/base/templates/configmap.yml

@@ -110,6 +110,8 @@ data:
     {{ load_file('logging.yaml') | indent }}
   production.ini: |-
     {{ load_file('production.ini.j2') | indent }}
+  celeryconfig.py: |-
+    {{ load_file('celeryconfig.py.j2') | indent }}
 ---
 apiVersion: v1
 kind: ConfigMap

roles/bodhi2/base/templates/logging.yaml

@@ -54,3 +54,19 @@ loggers:
     level: DEBUG
 {% endif %}
     handlers: [console, smtp]
+  celery:
+{% if env == "production" %}
+    level: INFO
+{% else %}
+    level: DEBUG
+{% endif %}
+    handlers: [console, smtp]
+    propagate: 0
+  celery_worker_job:
+{% if env == "production" %}
+    level: ERROR
+{% else %}
+    level: INFO
+{% endif %}
+    handlers: [console, smtp]
+    propagate: 1

roles/bodhi2/base/templates/production.ini.j2

@@ -425,14 +425,6 @@ koji_url = https://koji{{env_suffix}}.fedoraproject.org
 # fmn_url = https://apps.fedoraproject.org/notifications/
 fmn_url = https://apps{{env_suffix}}.fedoraproject.org/notifications/
 
-# If this is defined, fedmenu's JS will be injected into the master template. Fedora's fedmenu URL
-# is https://apps.fedoraproject.org/fedmenu and its data_url is
-# https://apps.fedoraproject.org/js/data.js
-# fedmenu.url =
-# fedmenu.data_url =
-fedmenu.url = https://apps{{env_suffix}}.fedoraproject.org/fedmenu
-fedmenu.data_url = https://apps{{env_suffix}}.fedoraproject.org/js/data.js
-
 # Koji krb5
 # krb_principal =
 # krb_keytab =
@@ -517,10 +509,14 @@ bugtracker = bugzilla
 
 # A URL to a Bugzilla instance's xmlrpc.cgi script for Bodhi to use.
 # bz_server = https://bugzilla.redhat.com/xmlrpc.cgi
+# A URL to a Bugzilla instance's REST api for Bodhi to use.
+# bz_server_rest = https://bugzilla.redhat.com/rest/
 {% if env == 'production' %}
 bz_server = https://bugzilla.redhat.com/xmlrpc.cgi
+bz_server_rest = https://bugzilla.redhat.com/rest/
 {% elif env == 'staging' %}
 bz_server = https://partner-bugzilla.redhat.com/xmlrpc.cgi
+bz_server_rest = https://partner-bugzilla.redhat.com/rest/
 {% endif %}
 
 # Bodhi will avoid touching bugs that are not against the following comma-separated products.
@@ -657,7 +653,7 @@ admin_packager_groups = provenpackager releng-team security_respons
 openid.provider = https://id{{env_suffix}}.fedoraproject.org/openid/
 openid.url = https://id{{env_suffix}}.fedoraproject.org/
 openid_template = {username}.id{{env_suffix}}.fedoraproject.org
-openid.sreg_required = email
+openid.sreg_required = email nickname
 
 # CORS allowed origins for cornice services
 # This can be wide-open.  read-only, we don't care as much about.
@@ -747,6 +743,9 @@ cache.short_term.expire = 60
 cache.default_term.expire = 300
 cache.long_term.expire = 3600
 
+# Celery configuration file
+celery_config = /etc/bodhi/celeryconfig.py
+
 [server:main]
 use = egg:waitress#main
 host = 0.0.0.0

roles/openshift-apps/bodhi/templates/buildconfig.yml

@@ -112,3 +112,39 @@ items:
         name: bodhi-consumer:latest
 kind: List
 metadata: {}
+---
+apiVersion: v1
+items:
+- apiVersion: v1
+  kind: BuildConfig
+  metadata:
+    labels:
+      build: bodhi-celery
+    name: bodhi-celery
+  spec:
+    runPolicy: Serial
+    source:
+      dockerfile: |-
+        FROM bodhi-base
+        LABEL \
+          name="bodhi-celery" \
+          vendor="Fedora Infrastructure" \
+          license="MIT"
+        ENTRYPOINT /usr/bin/celery-3 worker -A bodhi.server.tasks.app -l info -Q celery
+      type: Dockerfile
+    strategy:
+      type: Docker
+      dockerStrategy:
+        from:
+          kind: "ImageStreamTag"
+          name: "bodhi-base:latest"
+        noCache: false
+    triggers:
+      - type: "imageChange"
+        imageChange: {}
+    output:
+      to:
+        kind: ImageStreamTag
+        name: bodhi-celery:latest
+kind: List
+metadata: {}

roles/openshift-apps/bodhi/templates/deploymentconfig.yml

@@ -205,3 +205,98 @@ items:
     - type: ConfigChange
 kind: List
 metadata: {}
+---
+apiVersion: v1
+items:
+- apiVersion: v1
+  kind: DeploymentConfig
+  metadata:
+    labels:
+      app: bodhi
+      service: celery
+    name: bodhi-celery
+  spec:
+{% if env == "staging" %}
+    replicas: {{ hostvars[groups['bodhi2_stg'][0]]['openshift_pods'] }}
+{% else %}
+    replicas: {{ hostvars[groups['bodhi2'][0]]['openshift_pods'] }}
+{% endif %}
+    selector:
+      deploymentconfig: bodhi-celery
+    strategy:
+      activeDeadlineSeconds: 21600
+      recreateParams:
+        timeoutSeconds: 600
+      resources: {}
+      rollingParams:
+        intervalSeconds: 1
+        maxSurge: 25%
+        maxUnavailable: 25%
+        timeoutSeconds: 600
+        updatePeriodSeconds: 1
+      type: Rolling
+    template:
+      metadata:
+        creationTimestamp: null
+        labels:
+          app: bodhi-celery
+          deploymentconfig: bodhi-celery
+      spec:
+        containers:
+        - name: bodhi-celery
+          image: bodhi-celery:latest
+          resources: {}
+          volumeMounts:
+          - name: config-volume
+            mountPath: /etc/bodhi
+            readOnly: true
+          - name: keytab-volume
+            mountPath: /etc/keytabs
+            readOnly: true
+          - name: fedora-messaging-config-volume
+            mountPath: /etc/fedora-messaging
+            readOnly: true
+          - name: fedora-messaging-ca-volume
+            mountPath: /etc/pki/fedora-messaging/cacert.pem
+            subPath: cacert.pem
+            readOnly: true
+          - name: fedora-messaging-crt-volume
+            mountPath: /etc/pki/fedora-messaging/bodhi-cert.pem
+            subPath: bodhi-cert.pem
+            readOnly: true
+          - name: fedora-messaging-key-volume
+            mountPath: /etc/pki/fedora-messaging/bodhi-key.pem
+            subPath: bodhi-key.pem
+            readOnly: true
+        volumes:
+        - name: config-volume
+          configMap:
+            name: bodhi-configmap
+        - name: keytab-volume
+          secret:
+            secretName: bodhi-keytab
+        - name: fedora-messaging-config-volume
+          configMap:
+            name: fedora-messaging-configmap
+        - name: fedora-messaging-ca-volume
+          secret:
+            secretName: bodhi-fedora-messaging-ca
+        - name: fedora-messaging-crt-volume
+          secret:
+            secretName: bodhi-fedora-messaging-crt
+        - name: fedora-messaging-key-volume
+          secret:
+            secretName: bodhi-fedora-messaging-key
+    triggers:
+    - imageChangeParams:
+        automatic: true
+        containerNames:
+        - bodhi-celery
+        from:
+          kind: ImageStreamTag
+          name: bodhi-celery:latest
+          namespace: bodhi
+      type: ImageChange
+    - type: ConfigChange
+kind: List
+metadata: {}