diff --git a/inventory/group_vars/mbs_backend b/inventory/group_vars/mbs_backend index 8147b6d2aa..6ce70aaea8 100644 --- a/inventory/group_vars/mbs_backend +++ b/inventory/group_vars/mbs_backend @@ -40,3 +40,7 @@ csi_relationship: | mbs-frontend nodes. NOTE - this system has a KRB service principal with elevated koji privileges. + +mbs_broker_url: "amqp://mbs{{ env_suffix }}:{{ mbs_backend_amqp_password}}@rabbitmq{{ env_suffix }}.fedoraproject.org/%2Fmbs" +mbs_num_workers: 3 +mbs_systemd_wait_for_rabbitmq: true diff --git a/inventory/group_vars/mbs_backend_stg b/inventory/group_vars/mbs_backend_stg index 266a5bd559..c15a7c4084 100644 --- a/inventory/group_vars/mbs_backend_stg +++ b/inventory/group_vars/mbs_backend_stg @@ -47,3 +47,7 @@ csi_relationship: | mbs-frontend nodes. NOTE - this system has a KRB service principal with elevated koji privileges. + +mbs_broker_url: "amqp://mbs{{ env_suffix }}:{{ mbs_backend_amqp_password}}@rabbitmq{{ env_suffix }}.fedoraproject.org/%2Fmbs" +mbs_num_workers: 3 +mbs_systemd_wait_for_rabbitmq: true diff --git a/roles/mbs/common/defaults/main.yml b/roles/mbs/common/defaults/main.yml new file mode 100644 index 0000000000..9c13828e73 --- /dev/null +++ b/roles/mbs/common/defaults/main.yml @@ -0,0 +1,4 @@ +--- +mbs_broker_url: "" +mbs_systemd_wait_for_rabbitmq: false +mbs_celery_max_worker_tasks: 50 diff --git a/roles/mbs/common/tasks/main.yml b/roles/mbs/common/tasks/main.yml index bfbb82cc13..27b35736e2 100644 --- a/roles/mbs/common/tasks/main.yml +++ b/roles/mbs/common/tasks/main.yml @@ -13,6 +13,8 @@ notify: - restart apache - restart fedmsg-hub + - restart mbs poller + - restart mbs workers tags: - mbs - mbs/common @@ -36,6 +38,8 @@ notify: - restart apache - restart fedmsg-hub + - restart mbs poller + - restart mbs workers tags: - mbs - mbs/common @@ -102,6 +106,108 @@ - mbs - mbs/common +- name: Configure MBS virtual host in RabbitMQ + block: + - name: Configure the MBS virtual host + run_once: true + delegate_to: "rabbitmq01{{ env_suffix }}.iad2.fedoraproject.org" + rabbitmq_vhost: + name: /mbs + state: present + tags: + - rabbitmq_cluster + - config + - mbs + - mbs/common + + - name: Configure the HA policy for the MBS queues + run_once: true + delegate_to: "rabbitmq01{{ env_suffix }}.iad2.fedoraproject.org" + rabbitmq_policy: + name: HA + apply_to: queues + pattern: .* + tags: + ha-mode: all + ha-sync-mode: automatic # Auto sync queues to new cluster members + ha-sync-batch-size: 10000 # Larger is faster, but must finish in 1 net_ticktime + vhost: /mbs + tags: + - rabbitmq_cluster + - config + - mbs + - mbs/common + + - name: Grant the mbs user access to the MBS vhost + run_once: true + delegate_to: "rabbitmq01{{ env_suffix }}.iad2.fedoraproject.org" + rabbitmq_user: + user: "mbs{{ env_suffix }}" + password: "{{ mbs_backend_amqp_password }}" + vhost: /mbs + configure_priv: .* + read_priv: .* + write_priv: .* + update_password: always + tags: + - rabbitmq_cluster + - config + - mbs + - mbs/common + when: not mbs_frontend + +- name: Configure the MBS workers and poller + block: + - name: Add the systemd service files + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: root + group: root + mode: "0644" + with_items: + - src: mbs-worker@.service.j2 + dest: /etc/systemd/system/mbs-worker@.service + - src: mbs-poller.service.j2 + dest: /etc/systemd/system/mbs-poller.service + notify: + - restart mbs poller + - restart mbs workers + tags: + - mbs + - mbs/common + + - name: Enable the MBS workers + systemd: + name: "mbs-worker@{{ item }}" + daemon_reload: yes + enabled: yes + state: started + with_sequence: start=0 end={{ mbs_num_workers - 1 }} + tags: + - mbs + - mbs/common + + - name: Populate the service facts to detect if there are MBS workers to disable + service_facts: {} + tags: + - mbs + - mbs/common + + - name: Disable any extra MBS workers + systemd: + name: "mbs-worker@{{ worker_num }}" + enabled: no + state: stopped + with_items: "{{ ansible_facts.services | select('match', 'mbs-worker@\\d+.service') | list }}" + vars: + worker_num: "{{ item | regex_search('\\d+') }}" + when: (worker_num | int) >= mbs_num_workers + tags: + - mbs + - mbs/common + when: not mbs_frontend + - name: create /etc/module-build-service/default-modules directory file: path: /etc/module-build-service/default-modules diff --git a/roles/mbs/common/templates/config.py b/roles/mbs/common/templates/config.py index 821c703336..d8d9c95518 100644 --- a/roles/mbs/common/templates/config.py +++ b/roles/mbs/common/templates/config.py @@ -19,6 +19,8 @@ class BaseConfiguration(object): HOST = '0.0.0.0' PORT = 5000 + CELERY_BROKER_URL = '{{ mbs_broker_url }}' + # Global network-related values, in seconds NET_TIMEOUT = 120 NET_RETRY_INTERVAL = 30 @@ -63,6 +65,9 @@ class BaseConfiguration(object): KRB_PRINCIPAL = None KRB_CCACHE = None + # Number of celery workers + NUM_WORKERS = {{ mbs_num_workers }} + # AMQ prefixed variables are required only while using 'amq' as messaging backend # Addresses to listen to AMQ_RECV_ADDRESSES = ['amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.koji', diff --git a/roles/mbs/common/templates/mbs-poller.service.j2 b/roles/mbs/common/templates/mbs-poller.service.j2 new file mode 100644 index 0000000000..776eee86fc --- /dev/null +++ b/roles/mbs/common/templates/mbs-poller.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=MBS Poller +After=network.target{{ ' rabbitmq-server.service' if mbs_systemd_wait_for_rabbitmq else '' }} + +[Service] +Type=simple +# Always restart the service, even if it exits cleanly +Restart=always +RestartSec=1 +User=fedmsg +ExecStart=/usr/bin/celery beat -A module_build_service.scheduler.celery_app --loglevel=info --pidfile /var/run/fedmsg/mbs-scheduler.pid -s /var/run/fedmsg/mbs-scheduler.db + +[Install] +WantedBy=multi-user.target diff --git a/roles/mbs/common/templates/mbs-worker@.service.j2 b/roles/mbs/common/templates/mbs-worker@.service.j2 new file mode 100644 index 0000000000..e9287d22ea --- /dev/null +++ b/roles/mbs/common/templates/mbs-worker@.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=MBS worker %I +After=network.target{{ ' rabbitmq-server.service' if mbs_systemd_wait_for_rabbitmq else '' }} + +[Service] +Type=simple +# Always restart the service, even if it exits cleanly +Restart=always +RestartSec=1 +User=fedmsg +ExecStart=/usr/bin/celery worker -n mbs-worker-%I -Q mbs-default,mbs-%I -A module_build_service.scheduler.celery_app --loglevel=info --max-tasks-per-child={{ mbs_celery_max_worker_tasks }} + +[Install] +WantedBy=multi-user.target