Updated playbooks for MBS v3 deployment

inventory/group_vars/mbs_backend

@@ -40,3 +40,7 @@ csi_relationship: |
     mbs-frontend nodes.
 
     NOTE - this system has a KRB service principal with elevated koji privileges.
+
+mbs_broker_url: "amqp://mbs{{ env_suffix }}:{{ mbs_backend_amqp_password}}@rabbitmq{{ env_suffix }}.fedoraproject.org/%2Fmbs"
+mbs_num_workers: 3
+mbs_systemd_wait_for_rabbitmq: true

inventory/group_vars/mbs_backend_stg

@@ -47,3 +47,7 @@ csi_relationship: |
     mbs-frontend nodes.
 
     NOTE - this system has a KRB service principal with elevated koji privileges.
+
+mbs_broker_url: "amqp://mbs{{ env_suffix }}:{{ mbs_backend_amqp_password}}@rabbitmq{{ env_suffix }}.fedoraproject.org/%2Fmbs"
+mbs_num_workers: 3
+mbs_systemd_wait_for_rabbitmq: true

roles/mbs/common/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+mbs_broker_url: ""
+mbs_systemd_wait_for_rabbitmq: false
+mbs_celery_max_worker_tasks: 50

roles/mbs/common/tasks/main.yml

@@ -13,6 +13,8 @@
   notify:
   - restart apache
   - restart fedmsg-hub
+  - restart mbs poller
+  - restart mbs workers
   tags:
   - mbs
   - mbs/common
@@ -36,6 +38,8 @@
   notify:
   - restart apache
   - restart fedmsg-hub
+  - restart mbs poller
+  - restart mbs workers
   tags:
   - mbs
   - mbs/common
@@ -102,6 +106,108 @@
   - mbs
   - mbs/common
 
+- name: Configure MBS virtual host in RabbitMQ
+  block:
+  - name: Configure the MBS virtual host
+    run_once: true
+    delegate_to: "rabbitmq01{{ env_suffix }}.iad2.fedoraproject.org"
+    rabbitmq_vhost:
+      name: /mbs
+      state: present
+    tags:
+      - rabbitmq_cluster
+      - config
+      - mbs
+      - mbs/common
+
+  - name: Configure the HA policy for the MBS queues
+    run_once: true
+    delegate_to: "rabbitmq01{{ env_suffix }}.iad2.fedoraproject.org"
+    rabbitmq_policy:
+      name: HA
+      apply_to: queues
+      pattern: .*
+      tags:
+        ha-mode: all
+        ha-sync-mode: automatic  # Auto sync queues to new cluster members
+        ha-sync-batch-size: 10000  # Larger is faster, but must finish in 1 net_ticktime
+      vhost: /mbs
+    tags:
+      - rabbitmq_cluster
+      - config
+      - mbs
+      - mbs/common
+
+  - name: Grant the mbs user access to the MBS vhost
+    run_once: true
+    delegate_to: "rabbitmq01{{ env_suffix }}.iad2.fedoraproject.org"
+    rabbitmq_user:
+      user: "mbs{{ env_suffix }}"
+      password: "{{ mbs_backend_amqp_password }}"
+      vhost: /mbs
+      configure_priv: .*
+      read_priv: .*
+      write_priv: .*
+      update_password: always
+    tags:
+      - rabbitmq_cluster
+      - config
+      - mbs
+      - mbs/common
+  when: not mbs_frontend
+
+- name: Configure the MBS workers and poller
+  block:
+    - name: Add the systemd service files
+      template:
+        src: "{{ item.src }}"
+        dest: "{{ item.dest }}"
+        owner: root
+        group: root
+        mode: "0644"
+      with_items:
+      - src: mbs-worker@.service.j2
+        dest: /etc/systemd/system/mbs-worker@.service
+      - src: mbs-poller.service.j2
+        dest: /etc/systemd/system/mbs-poller.service
+      notify:
+      - restart mbs poller
+      - restart mbs workers
+      tags:
+        - mbs
+        - mbs/common
+
+    - name: Enable the MBS workers
+      systemd:
+        name: "mbs-worker@{{ item }}"
+        daemon_reload: yes
+        enabled: yes
+        state: started
+      with_sequence: start=0 end={{ mbs_num_workers - 1 }}
+      tags:
+        - mbs
+        - mbs/common
+
+    - name: Populate the service facts to detect if there are MBS workers to disable
+      service_facts: {}
+      tags:
+        - mbs
+        - mbs/common
+
+    - name: Disable any extra MBS workers
+      systemd:
+        name: "mbs-worker@{{ worker_num }}"
+        enabled: no
+        state: stopped
+      with_items: "{{ ansible_facts.services | select('match', 'mbs-worker@\\d+.service') | list }}"
+      vars:
+        worker_num: "{{ item | regex_search('\\d+') }}"
+      when: (worker_num | int) >= mbs_num_workers
+      tags:
+        - mbs
+        - mbs/common
+  when: not mbs_frontend
+
 - name: create /etc/module-build-service/default-modules directory
   file:
     path: /etc/module-build-service/default-modules

roles/mbs/common/templates/config.py

@@ -19,6 +19,8 @@ class BaseConfiguration(object):
     HOST = '0.0.0.0'
     PORT = 5000
 
+    CELERY_BROKER_URL = '{{ mbs_broker_url }}'
+
     # Global network-related values, in seconds
     NET_TIMEOUT = 120
     NET_RETRY_INTERVAL = 30
@@ -63,6 +65,9 @@ class BaseConfiguration(object):
     KRB_PRINCIPAL = None
     KRB_CCACHE = None
 
+    # Number of celery workers
+    NUM_WORKERS = {{ mbs_num_workers }}
+
     # AMQ prefixed variables are required only while using 'amq' as messaging backend
     # Addresses to listen to
     AMQ_RECV_ADDRESSES = ['amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.koji',

roles/mbs/common/templates/mbs-poller.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=MBS Poller
+After=network.target{{ ' rabbitmq-server.service' if mbs_systemd_wait_for_rabbitmq else '' }}
+
+[Service]
+Type=simple
+# Always restart the service, even if it exits cleanly
+Restart=always
+RestartSec=1
+User=fedmsg
+ExecStart=/usr/bin/celery beat -A module_build_service.scheduler.celery_app --loglevel=info --pidfile /var/run/fedmsg/mbs-scheduler.pid -s /var/run/fedmsg/mbs-scheduler.db
+
+[Install]
+WantedBy=multi-user.target

roles/mbs/common/templates/mbs-worker@.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=MBS worker %I
+After=network.target{{ ' rabbitmq-server.service' if mbs_systemd_wait_for_rabbitmq else '' }}
+
+[Service]
+Type=simple
+# Always restart the service, even if it exits cleanly
+Restart=always
+RestartSec=1
+User=fedmsg
+ExecStart=/usr/bin/celery worker -n mbs-worker-%I -Q mbs-default,mbs-%I -A module_build_service.scheduler.celery_app --loglevel=info --max-tasks-per-child={{ mbs_celery_max_worker_tasks }}
+
+[Install]
+WantedBy=multi-user.target