From e671d4a98441dc1e9ca7e9d39252c02c21c36ba0 Mon Sep 17 00:00:00 2001
From: Pierre-Yves Chibon <pingou@pingoured.fr>
Date: Mon, 23 Apr 2018 16:38:04 +0200
Subject: [PATCH] Split the tasks into multiple queues on stg.pagure.io

Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>
---
 .../frontend/files/pagure_fast_worker.service | 15 +++++++++++
 .../files/pagure_medium_worker.service        | 15 +++++++++++
 .../frontend/files/pagure_slow_worker.service | 15 +++++++++++
 roles/pagure/frontend/tasks/main.yml          | 27 +++++++++++++++++++
 roles/pagure/frontend/templates/pagure.cfg    |  6 +++++
 5 files changed, 78 insertions(+)
 create mode 100644 roles/pagure/frontend/files/pagure_fast_worker.service
 create mode 100644 roles/pagure/frontend/files/pagure_medium_worker.service
 create mode 100644 roles/pagure/frontend/files/pagure_slow_worker.service

diff --git a/roles/pagure/frontend/files/pagure_fast_worker.service b/roles/pagure/frontend/files/pagure_fast_worker.service
new file mode 100644
index 0000000000..b8fde35439
--- /dev/null
+++ b/roles/pagure/frontend/files/pagure_fast_worker.service
@@ -0,0 +1,15 @@
+[Unit]
+Description=Pagure worker for backend git interaction
+After=redis.target
+Documentation=https://pagure.io/pagure
+
+[Service]
+ExecStart=/usr/bin/celery worker -A pagure.lib.tasks --loglevel=info --autoreload -Q fast_workers -c 5
+Environment="PAGURE_CONFIG=/etc/pagure/pagure.cfg"
+Type=simple
+User=git
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+
diff --git a/roles/pagure/frontend/files/pagure_medium_worker.service b/roles/pagure/frontend/files/pagure_medium_worker.service
new file mode 100644
index 0000000000..2d3d4049e0
--- /dev/null
+++ b/roles/pagure/frontend/files/pagure_medium_worker.service
@@ -0,0 +1,15 @@
+[Unit]
+Description=Pagure worker for backend git interaction
+After=redis.target
+Documentation=https://pagure.io/pagure
+
+[Service]
+ExecStart=/usr/bin/celery worker -A pagure.lib.tasks --loglevel=info --autoreload -Q medium_workers -c 5
+Environment="PAGURE_CONFIG=/etc/pagure/pagure.cfg"
+Type=simple
+User=git
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+
diff --git a/roles/pagure/frontend/files/pagure_slow_worker.service b/roles/pagure/frontend/files/pagure_slow_worker.service
new file mode 100644
index 0000000000..6d7a0ce515
--- /dev/null
+++ b/roles/pagure/frontend/files/pagure_slow_worker.service
@@ -0,0 +1,15 @@
+[Unit]
+Description=Pagure worker for backend git interaction
+After=redis.target
+Documentation=https://pagure.io/pagure
+
+[Service]
+ExecStart=/usr/bin/celery worker -A pagure.lib.tasks --loglevel=info --autoreload -Q slow_workers -c 5
+Environment="PAGURE_CONFIG=/etc/pagure/pagure.cfg"
+Type=simple
+User=git
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+
diff --git a/roles/pagure/frontend/tasks/main.yml b/roles/pagure/frontend/tasks/main.yml
index 9417314e39..65f79c676a 100644
--- a/roles/pagure/frontend/tasks/main.yml
+++ b/roles/pagure/frontend/tasks/main.yml
@@ -190,6 +190,20 @@
   - stunnel
   - config
 
+- name: Add the different service files for the different workers
+  copy: src={{ item }}.service
+        dest=/etc/systemd/system/{{ item }}.service
+        owner=root group=root mode=0755
+  when: env == 'pagure-staging'
+  with_items:
+  - pagure_fast_worker
+  - pagure_medium_worker
+  - pagure_slow_worker
+  notify:
+  - reload systemd
+  tags:
+  - pagure
+
 
 # Set-up Pagure
 
@@ -379,3 +393,16 @@
   - pagure
   - service
   - postfix
+
+- name: Start and enable the worker services
+  service: name={{ item }} enabled=yes state=started
+  with_items:
+  - pagure_fast_worker
+  - pagure_medium_worker
+  - pagure_slow_worker
+  when: env == 'pagure-staging'
+  ignore_errors: true
+  tags:
+   - pagure
+   - service
+   - postfix
diff --git a/roles/pagure/frontend/templates/pagure.cfg b/roles/pagure/frontend/templates/pagure.cfg
index 1a40f36504..a0350a68ff 100644
--- a/roles/pagure/frontend/templates/pagure.cfg
+++ b/roles/pagure/frontend/templates/pagure.cfg
@@ -286,3 +286,9 @@ CROSS_PROJECT_ACLS = [
 BLACKLISTED_GROUPS = ['forks', 'group', 'rpms', 'modules', 'container', 'tests']
 
 GITOLITE_CELERY_QUEUE = 'gitolite_queue'
+{% if env == 'pagure-staging' %}
+FAST_CELERY_QUEUE = 'fast_workers'
+MEDIUM_CELERY_QUEUE = 'medium_workers'
+SLOW_CELERY_QUEUE = 'slow_workers'
+{% else %}
+{% endif %}