HyperKitty: the new version uses Django-Q

playbooks/manual/staging-sync/mailman.yml

@@ -17,7 +17,7 @@
   - service: name=mailman3 state=stopped
 
 
-- name: drop and re-create the staging dsb entirely
+- name: drop and re-create the staging dbs entirely
   hosts: db01.stg.phx2.fedoraproject.org
   user: root
   become: yes

roles/mailman/files/post-update.sh

@@ -10,18 +10,21 @@ BASEDIR=`yamlget basedir $CONFFILE`
 CONFDIR=`yamlget confdir $CONFFILE`
 INDEXDIR=$BASEDIR/fulltext_index
 
-# Give database rights to the non-admin user (must be done before loading initial data)
 sleep $[ ( $RANDOM % 10 )  + 1 ]s # avoid simultaneous lockups on parallel servers. Yes, this is dirty.
-$BASEDIR/bin/pg-give-rights.py > /dev/null
 
 echo "Stop services"
-systemctl stop crond mailman3 httpd
+systemctl stop webui-qcluster crond mailman3 httpd
 
 echo "static files"
 django-admin collectstatic --clear --noinput --verbosity 0 --pythonpath $CONFDIR --settings settings
 django-admin compress --pythonpath $CONFDIR --settings settings
 echo "db migration"
 django-admin migrate --pythonpath $CONFDIR --settings settings_admin --noinput
+
+echo "give database rights to the non-admin user"
+# this must be done before loading initial data
+$BASEDIR/bin/pg-give-rights.py > /dev/null
+
 echo "load initial data"
 django-admin loaddata $CONFDIR/initial-data.json --pythonpath $CONFDIR --settings settings
 mkdir -p $INDEXDIR
@@ -36,4 +39,4 @@ echo "unit tests"
 django-admin test --pythonpath $CONFDIR --settings settings_test hyperkitty postorius
 
 # Restart services
-systemctl start httpd mailman3 crond
+systemctl start httpd mailman3 crond webui-qcluster

roles/mailman/files/settings_test.py

@@ -44,7 +44,38 @@ HAYSTACK_CONNECTIONS = {
 }
 HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'
 
-LOGGING = {}
+#
+# Asynchronous tasks
+#
+Q_CLUSTER = {
+    'orm': 'default',
+    'sync': True,
+}
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+        },
+    },
+    'loggers': {
+        'hyperkitty': {
+            'handlers': ['console'],
+            'level': 'INFO',
+            'propagate': True,
+        },
+        'django_mailman3.lib.mailman': {
+            'handlers': ['console'],
+            'level': 'ERROR',
+        },
+        'django-q': {
+            'handlers': ['console'],
+            'level': 'WARNING',
+        },
+    },
+}
 
 # Disable caching
 CACHES = {

roles/mailman/handlers/main.yml

@@ -7,3 +7,6 @@
 
 - name: restart memcached
   service: name=memcached state=restarted
+
+- name: systemctl daemon-reload
+  command: /usr/bin/systemctl daemon-reload

roles/mailman/tasks/main.yml

@@ -431,6 +431,15 @@
             dest=/etc/yum/post-actions/hyperkitty.action
   tags: mailman
 
+# Systemd
+- name: install the systemd service file
+  template: src=webui-qcluster.service.j2 dest=/etc/systemd/system/webui-qcluster.service
+  notify:
+  - systemctl daemon-reload
+  tags:
+  - config
+  - mailman
+
 #
 # Only run this on mailman01 for now.
 # TODO: run it on lists-dev too
@@ -469,5 +478,6 @@
   - httpd
   - mailman3
   - postfix
+  - webui-qcluster
   tags: mailman
   when: inventory_hostname.startswith('mailman01.phx2') or inventory_hostname.startswith('lists-dev')

roles/mailman/templates/settings.py.j2

@@ -66,6 +66,7 @@ INSTALLED_APPS = (
     'allauth',
     'allauth.account',
     'allauth.socialaccount',
+    'django_q',
     'django_mailman3.lib.auth.fedora',
     {% for service_name, service_data in mailman_login.items() %}
     'allauth.socialaccount.providers.{{ service_data.provider }}',
@@ -314,6 +315,16 @@ HAYSTACK_CONNECTIONS = {
 }
 
 
+#
+# Asynchronous tasks
+#
+Q_CLUSTER = {
+    'timeout': 300,
+    'save_limit': 500,
+    'orm': 'default',
+}
+
+
 # A sample logging configuration. The only tangible logging
 # performed by this configuration is to send an email to
 # the site admins on every HTTP 500 error when DEBUG=False.

roles/mailman/templates/webui-qcluster.service.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=Mailman web UI async tasks runner
+After=network.target remote-fs.target
+
+[Service]
+ExecStart=/usr/bin/django-admin qcluster --pythonpath {{ mailman_webui_confdir }} --settings settings
+User=apache
+Restart=always
+
+[Install]
+WantedBy=multi-user.target