diff --git a/playbooks/manual/staging-sync/bodhi.yml b/playbooks/manual/staging-sync/bodhi.yml index 00486c7ed7..8763bdc463 100644 --- a/playbooks/manual/staging-sync/bodhi.yml +++ b/playbooks/manual/staging-sync/bodhi.yml @@ -23,6 +23,7 @@ tasks: - command: oc -n bodhi scale dc/bodhi-web --replicas=0 - command: oc -n bodhi scale dc/bodhi-consumer --replicas=0 + - command: oc -n bodhi scale dc/bodhi-celery --replicas=0 - name: bring staging services down (messaging) hosts: bodhi_backend_stg @@ -74,6 +75,7 @@ tasks: - command: oc -n bodhi scale dc/bodhi-web --replicas={{ hostvars[groups['bodhi2_stg'][0]]['openshift_pods'] }} - command: oc -n bodhi scale dc/bodhi-consumer --replicas={{ hostvars[groups['bodhi2_stg'][0]]['openshift_pods'] }} + - command: oc -n bodhi scale dc/bodhi-celery --replicas={{ hostvars[groups['bodhi2_stg'][0]]['openshift_pods'] }} - name: bring staging services up (httpd) hosts: bodhi2_stg diff --git a/playbooks/openshift-apps/bodhi.yml b/playbooks/openshift-apps/bodhi.yml index 9fd79303f1..699b8a98ea 100644 --- a/playbooks/openshift-apps/bodhi.yml +++ b/playbooks/openshift-apps/bodhi.yml @@ -69,11 +69,14 @@ - role: openshift/imagestream app: bodhi imagename: bodhi-consumer + - role: openshift/imagestream + app: bodhi + imagename: bodhi-celery - role: openshift/object app: bodhi template: buildconfig.yml objectname: buildconfig.yml - bodhi_version: 4.1.1-1.fc29.infra + bodhi_version: 5.0.0-0.beta.1.761b898.fc29.infra when: env == "staging" - role: openshift/object app: bodhi @@ -108,17 +111,26 @@ - role: openshift/rollout app: bodhi dcname: bodhi-consumer + - role: openshift/rollout + app: bodhi + dcname: bodhi-celery post_tasks: - name: Scale up pods command: oc -n bodhi scale dc/bodhi-consumer --replicas={{ hostvars[groups['bodhi2'][0]]['openshift_pods'] }} when: env == "production" + - name: Scale up pods + command: oc -n bodhi scale dc/bodhi-celery --replicas={{ hostvars[groups['bodhi2'][0]]['openshift_pods'] }} + when: env == "production" - name: Scale up pods command: oc -n bodhi scale dc/bodhi-web --replicas={{ hostvars[groups['bodhi2'][0]]['openshift_pods'] }} when: env == "production" - name: Scale up pods command: oc -n bodhi scale dc/bodhi-consumer --replicas={{ hostvars[groups['bodhi2_stg'][0]]['openshift_pods'] }} when: env == "staging" + - name: Scale up pods + command: oc -n bodhi scale dc/bodhi-celery --replicas={{ hostvars[groups['bodhi2_stg'][0]]['openshift_pods'] }} + when: env == "staging" - name: Scale up pods command: oc -n bodhi scale dc/bodhi-web --replicas={{ hostvars[groups['bodhi2_stg'][0]]['openshift_pods'] }} when: env == "staging" diff --git a/roles/bodhi2/backend/files/bodhi-celery.service b/roles/bodhi2/backend/files/bodhi-celery.service new file mode 100644 index 0000000000..09ba92c5e1 --- /dev/null +++ b/roles/bodhi2/backend/files/bodhi-celery.service @@ -0,0 +1,12 @@ +[Unit] +Description = Bodhi's Celery worker +After = network-online.target +Wants = network-online.target + +[Service] +User = apache +Group = apache +ExecStart = /usr/bin/celery-3 worker -A bodhi.server.tasks.app -l info -Q celery,has_koji_mount + +[Install] +WantedBy = multi-user.target \ No newline at end of file diff --git a/roles/bodhi2/backend/tasks/main.yml b/roles/bodhi2/backend/tasks/main.yml index 24354e74bb..d5d0d82f09 100644 --- a/roles/bodhi2/backend/tasks/main.yml +++ b/roles/bodhi2/backend/tasks/main.yml @@ -18,6 +18,10 @@ - bodhi-composer - python3-pyramid_sawing - sigul + # This next one is not strictly needed since bodhi-composer depends on bodhi-server which + # depends on python3-celery, but since we're depoying a service file calling celery in this + # role I think it makes sense to require it here instead of assuming a dependency chain. + - python3-celery # This is used to generate zchunk data more efficiently - fedora-repo-zdicts # The new-updates-sync script uses this @@ -217,23 +221,34 @@ - bodhi - name: Install production.ini - template: > - src="{{ roles_path }}/bodhi2/base/templates/production.ini.j2" - dest="/etc/bodhi/production.ini" - owner=apache - group=apache - mode=0600 + template: + src: "{{ roles_path }}/bodhi2/base/templates/production.ini.j2" + dest: /etc/bodhi/production.ini + owner: apache + group: apache + mode: 0600 + tags: + - config + - bodhi + +- name: Install celeryconfig.py + template: + src: "{{ roles_path }}/bodhi2/base/templates/celeryconfig.py.j2" + dest: /etc/bodhi/celeryconfig.py + owner: apache + group: apache + mode: 0600 tags: - config - bodhi - name: Install fedora-messaging config - template: > - src="{{ roles_path }}/bodhi2/base/templates/fedora-messaging.toml.j2" - dest="/etc/fedora-messaging/config.toml" - owner=apache - group=apache - mode=0600 + template: + src: "{{ roles_path }}/bodhi2/base/templates/fedora-messaging.toml.j2" + dest: /etc/fedora-messaging/config.toml + owner: apache + group: apache + mode: 0600 tags: - config - bodhi @@ -267,6 +282,19 @@ - bodhi - config +- name: Setup the Celery service + copy: + src: bodhi-celery.service + dest: /etc/systemd/system/bodhi-celery.service + owner: root + group: root + mode: 0644 + notify: + - reload systemd + tags: + - bodhi + - config + - name: ensure apache is disabled on the backend service: name=httpd enabled=no state=stopped tags: @@ -291,7 +319,7 @@ group: apache tags: - bodhi - + - name: Deploy the fedora-messaging cert copy: src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/bodhi{{env_suffix}}.crt" @@ -301,7 +329,7 @@ group: apache tags: - bodhi - + - name: Deploy the fedora-messaging key copy: src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/bodhi{{env_suffix}}.key" @@ -311,9 +339,15 @@ group: apache tags: - bodhi - -- name: ensure fedora-messaging is enabled and started on the backend - service: name=fm-consumer@config.service enabled=yes state=started + +- name: ensure fedora-messaging and celery are enabled and started on the backend + service: + name: "{{ item }}" + enabled: yes + state: started + with_items: + - fm-consumer@config + - bodhi-celery tags: - bodhi diff --git a/roles/bodhi2/base/tasks/main.yml b/roles/bodhi2/base/tasks/main.yml index d7b6be4306..5ed664707b 100644 --- a/roles/bodhi2/base/tasks/main.yml +++ b/roles/bodhi2/base/tasks/main.yml @@ -11,3 +11,77 @@ tags: - config - bodhi + +# Bodhi virtualhost in RabbitMQ + +- name: Configure the bodhi virtual host + run_once: true + delegate_to: "rabbitmq01{{ env_suffix }}.phx2.fedoraproject.org" + rabbitmq_vhost: + name: /bodhi + state: present + tags: + - rabbitmq_cluster + - config + - bodhi + +- name: Configure the HA policy for the bodhi queues + run_once: true + delegate_to: "rabbitmq01{{ env_suffix }}.phx2.fedoraproject.org" + rabbitmq_policy: + name: HA + apply_to: queues + pattern: .* + tags: + ha-mode: all + ha-sync-mode: automatic # Auto sync queues to new cluster members + ha-sync-batch-size: 10000 # Larger is faster, but must finish in 1 net_ticktime + vhost: /bodhi + tags: + - rabbitmq_cluster + - config + - bodhi + +- name: Grant the admin user access to the bodhi vhost + run_once: true + delegate_to: "rabbitmq01{{ env_suffix }}.phx2.fedoraproject.org" + rabbitmq_user: + user: admin + vhost: /bodhi + configure_priv: .* + read_priv: .* + write_priv: .* + tags: + - rabbitmq_cluster + - config + - bodhi + +# Create a user for Celery +# - name: Create a user for Celery usage +# run_once: true +# delegate_to: "rabbitmq01{{ env_suffix }}.phx2.fedoraproject.org" +# rabbitmq_user: +# user: "bodhi-celery{{ env_suffix }}" +# vhost: /bodhi +# configure_priv: .* +# write_priv: .* +# read_priv: .* +# state: present +# tags: +# - rabbitmq_cluster +# - config +# - bodhi + +- name: Grant the bodhi user access to the bodhi vhost + run_once: true + delegate_to: "rabbitmq01{{ env_suffix }}.phx2.fedoraproject.org" + rabbitmq_user: + user: "bodhi{{ env_suffix }}" + vhost: /bodhi + configure_priv: .* + read_priv: .* + write_priv: .* + tags: + - rabbitmq_cluster + - config + - bodhi \ No newline at end of file diff --git a/roles/bodhi2/base/templates/celeryconfig.py.j2 b/roles/bodhi2/base/templates/celeryconfig.py.j2 new file mode 100644 index 0000000000..7b8749d916 --- /dev/null +++ b/roles/bodhi2/base/templates/celeryconfig.py.j2 @@ -0,0 +1,29 @@ +# +# Celery configuration file +# See: docs.celeryproject.org/en/latest/userguide/configuration.html +# + +# Broker URL +# This might be more appropriate in prod: +# broker_url = amqps://user:password@hostname:port//vhost +# broker_use_ssl = +# keyfile=/var/ssl/private/worker-key.pem +# certfile=/var/ssl/amqp-server-cert.pem +# ca_certs=/var/ssl/myca.pem +# cert_reqs=ssl.CERT_REQUIRED +broker_url = "amqps://bodhi{{ env_suffix }}:@rabbitmq{{ env_suffix }}.fedoraproject.org//bodhi" +broker_use_ssl = + ca_certs=/etc/pki/fedora-messaging/cacert.pem + keyfile=/etc/pki/fedora-messaging/bodhi-key.pem + certfile=/etc/pki/fedora-messaging/bodhi-cert.pem + cert_reqs=ssl.CERT_REQUIRED + +# Where the tasks are defined +imports = "bodhi.server.tasks" + +# Task routing +task_routes = { + # Route the compose task to a specific queue that will only be run on hosts + # that have a Koji mount. + 'compose': {'queue': 'has_koji_mount'}, +} \ No newline at end of file diff --git a/roles/bodhi2/base/templates/configmap.yml b/roles/bodhi2/base/templates/configmap.yml index fd67a00357..cb37e3f4c0 100644 --- a/roles/bodhi2/base/templates/configmap.yml +++ b/roles/bodhi2/base/templates/configmap.yml @@ -110,6 +110,8 @@ data: {{ load_file('logging.yaml') | indent }} production.ini: |- {{ load_file('production.ini.j2') | indent }} + celeryconfig.py: |- + {{ load_file('celeryconfig.py.j2') | indent }} --- apiVersion: v1 kind: ConfigMap diff --git a/roles/bodhi2/base/templates/logging.yaml b/roles/bodhi2/base/templates/logging.yaml index 79e3383976..c947262795 100644 --- a/roles/bodhi2/base/templates/logging.yaml +++ b/roles/bodhi2/base/templates/logging.yaml @@ -54,3 +54,19 @@ loggers: level: DEBUG {% endif %} handlers: [console, smtp] + celery: +{% if env == "production" %} + level: INFO +{% else %} + level: DEBUG +{% endif %} + handlers: [console, smtp] + propagate: 0 + celery_worker_job: +{% if env == "production" %} + level: ERROR +{% else %} + level: INFO +{% endif %} + handlers: [console, smtp] + propagate: 1 \ No newline at end of file diff --git a/roles/bodhi2/base/templates/production.ini.j2 b/roles/bodhi2/base/templates/production.ini.j2 index 58b505bb66..1587244135 100644 --- a/roles/bodhi2/base/templates/production.ini.j2 +++ b/roles/bodhi2/base/templates/production.ini.j2 @@ -425,14 +425,6 @@ koji_url = https://koji{{env_suffix}}.fedoraproject.org # fmn_url = https://apps.fedoraproject.org/notifications/ fmn_url = https://apps{{env_suffix}}.fedoraproject.org/notifications/ -# If this is defined, fedmenu's JS will be injected into the master template. Fedora's fedmenu URL -# is https://apps.fedoraproject.org/fedmenu and its data_url is -# https://apps.fedoraproject.org/js/data.js -# fedmenu.url = -# fedmenu.data_url = -fedmenu.url = https://apps{{env_suffix}}.fedoraproject.org/fedmenu -fedmenu.data_url = https://apps{{env_suffix}}.fedoraproject.org/js/data.js - # Koji krb5 # krb_principal = # krb_keytab = @@ -517,10 +509,14 @@ bugtracker = bugzilla # A URL to a Bugzilla instance's xmlrpc.cgi script for Bodhi to use. # bz_server = https://bugzilla.redhat.com/xmlrpc.cgi +# A URL to a Bugzilla instance's REST api for Bodhi to use. +# bz_server_rest = https://bugzilla.redhat.com/rest/ {% if env == 'production' %} bz_server = https://bugzilla.redhat.com/xmlrpc.cgi +bz_server_rest = https://bugzilla.redhat.com/rest/ {% elif env == 'staging' %} bz_server = https://partner-bugzilla.redhat.com/xmlrpc.cgi +bz_server_rest = https://partner-bugzilla.redhat.com/rest/ {% endif %} # Bodhi will avoid touching bugs that are not against the following comma-separated products. @@ -657,7 +653,7 @@ admin_packager_groups = provenpackager releng-team security_respons openid.provider = https://id{{env_suffix}}.fedoraproject.org/openid/ openid.url = https://id{{env_suffix}}.fedoraproject.org/ openid_template = {username}.id{{env_suffix}}.fedoraproject.org -openid.sreg_required = email +openid.sreg_required = email nickname # CORS allowed origins for cornice services # This can be wide-open. read-only, we don't care as much about. @@ -747,6 +743,9 @@ cache.short_term.expire = 60 cache.default_term.expire = 300 cache.long_term.expire = 3600 +# Celery configuration file +celery_config = /etc/bodhi/celeryconfig.py + [server:main] use = egg:waitress#main host = 0.0.0.0 diff --git a/roles/openshift-apps/bodhi/templates/buildconfig.yml b/roles/openshift-apps/bodhi/templates/buildconfig.yml index 85ccb6cc59..d6ff3821b9 100644 --- a/roles/openshift-apps/bodhi/templates/buildconfig.yml +++ b/roles/openshift-apps/bodhi/templates/buildconfig.yml @@ -112,3 +112,39 @@ items: name: bodhi-consumer:latest kind: List metadata: {} +--- +apiVersion: v1 +items: +- apiVersion: v1 + kind: BuildConfig + metadata: + labels: + build: bodhi-celery + name: bodhi-celery + spec: + runPolicy: Serial + source: + dockerfile: |- + FROM bodhi-base + LABEL \ + name="bodhi-celery" \ + vendor="Fedora Infrastructure" \ + license="MIT" + ENTRYPOINT /usr/bin/celery-3 worker -A bodhi.server.tasks.app -l info -Q celery + type: Dockerfile + strategy: + type: Docker + dockerStrategy: + from: + kind: "ImageStreamTag" + name: "bodhi-base:latest" + noCache: false + triggers: + - type: "imageChange" + imageChange: {} + output: + to: + kind: ImageStreamTag + name: bodhi-celery:latest +kind: List +metadata: {} \ No newline at end of file diff --git a/roles/openshift-apps/bodhi/templates/deploymentconfig.yml b/roles/openshift-apps/bodhi/templates/deploymentconfig.yml index 23d49770d6..5f83b8f56e 100644 --- a/roles/openshift-apps/bodhi/templates/deploymentconfig.yml +++ b/roles/openshift-apps/bodhi/templates/deploymentconfig.yml @@ -205,3 +205,98 @@ items: - type: ConfigChange kind: List metadata: {} +--- +apiVersion: v1 +items: +- apiVersion: v1 + kind: DeploymentConfig + metadata: + labels: + app: bodhi + service: celery + name: bodhi-celery + spec: +{% if env == "staging" %} + replicas: {{ hostvars[groups['bodhi2_stg'][0]]['openshift_pods'] }} +{% else %} + replicas: {{ hostvars[groups['bodhi2'][0]]['openshift_pods'] }} +{% endif %} + selector: + deploymentconfig: bodhi-celery + strategy: + activeDeadlineSeconds: 21600 + recreateParams: + timeoutSeconds: 600 + resources: {} + rollingParams: + intervalSeconds: 1 + maxSurge: 25% + maxUnavailable: 25% + timeoutSeconds: 600 + updatePeriodSeconds: 1 + type: Rolling + template: + metadata: + creationTimestamp: null + labels: + app: bodhi-celery + deploymentconfig: bodhi-celery + spec: + containers: + - name: bodhi-celery + image: bodhi-celery:latest + resources: {} + volumeMounts: + - name: config-volume + mountPath: /etc/bodhi + readOnly: true + - name: keytab-volume + mountPath: /etc/keytabs + readOnly: true + - name: fedora-messaging-config-volume + mountPath: /etc/fedora-messaging + readOnly: true + - name: fedora-messaging-ca-volume + mountPath: /etc/pki/fedora-messaging/cacert.pem + subPath: cacert.pem + readOnly: true + - name: fedora-messaging-crt-volume + mountPath: /etc/pki/fedora-messaging/bodhi-cert.pem + subPath: bodhi-cert.pem + readOnly: true + - name: fedora-messaging-key-volume + mountPath: /etc/pki/fedora-messaging/bodhi-key.pem + subPath: bodhi-key.pem + readOnly: true + volumes: + - name: config-volume + configMap: + name: bodhi-configmap + - name: keytab-volume + secret: + secretName: bodhi-keytab + - name: fedora-messaging-config-volume + configMap: + name: fedora-messaging-configmap + - name: fedora-messaging-ca-volume + secret: + secretName: bodhi-fedora-messaging-ca + - name: fedora-messaging-crt-volume + secret: + secretName: bodhi-fedora-messaging-crt + - name: fedora-messaging-key-volume + secret: + secretName: bodhi-fedora-messaging-key + triggers: + - imageChangeParams: + automatic: true + containerNames: + - bodhi-celery + from: + kind: ImageStreamTag + name: bodhi-celery:latest + namespace: bodhi + type: ImageChange + - type: ConfigChange +kind: List +metadata: {} \ No newline at end of file