diff --git a/playbooks/manual/staging-sync/mailman.yml b/playbooks/manual/staging-sync/mailman.yml index 5c6470569c..d762cc43d4 100644 --- a/playbooks/manual/staging-sync/mailman.yml +++ b/playbooks/manual/staging-sync/mailman.yml @@ -17,7 +17,7 @@ - service: name=mailman3 state=stopped -- name: drop and re-create the staging dsb entirely +- name: drop and re-create the staging dbs entirely hosts: db01.stg.phx2.fedoraproject.org user: root become: yes diff --git a/roles/mailman/files/post-update.sh b/roles/mailman/files/post-update.sh index fe9383f2e7..ffb59678b1 100755 --- a/roles/mailman/files/post-update.sh +++ b/roles/mailman/files/post-update.sh @@ -10,18 +10,21 @@ BASEDIR=`yamlget basedir $CONFFILE` CONFDIR=`yamlget confdir $CONFFILE` INDEXDIR=$BASEDIR/fulltext_index -# Give database rights to the non-admin user (must be done before loading initial data) sleep $[ ( $RANDOM % 10 ) + 1 ]s # avoid simultaneous lockups on parallel servers. Yes, this is dirty. -$BASEDIR/bin/pg-give-rights.py > /dev/null echo "Stop services" -systemctl stop crond mailman3 httpd +systemctl stop webui-qcluster crond mailman3 httpd echo "static files" django-admin collectstatic --clear --noinput --verbosity 0 --pythonpath $CONFDIR --settings settings django-admin compress --pythonpath $CONFDIR --settings settings echo "db migration" django-admin migrate --pythonpath $CONFDIR --settings settings_admin --noinput + +echo "give database rights to the non-admin user" +# this must be done before loading initial data +$BASEDIR/bin/pg-give-rights.py > /dev/null + echo "load initial data" django-admin loaddata $CONFDIR/initial-data.json --pythonpath $CONFDIR --settings settings mkdir -p $INDEXDIR @@ -36,4 +39,4 @@ echo "unit tests" django-admin test --pythonpath $CONFDIR --settings settings_test hyperkitty postorius # Restart services -systemctl start httpd mailman3 crond +systemctl start httpd mailman3 crond webui-qcluster diff --git a/roles/mailman/files/settings_test.py b/roles/mailman/files/settings_test.py index f1cb3093b5..f2e2af9417 100644 --- a/roles/mailman/files/settings_test.py +++ b/roles/mailman/files/settings_test.py @@ -44,7 +44,38 @@ HAYSTACK_CONNECTIONS = { } HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor' -LOGGING = {} +# +# Asynchronous tasks +# +Q_CLUSTER = { + 'orm': 'default', + 'sync': True, +} + +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + }, + }, + 'loggers': { + 'hyperkitty': { + 'handlers': ['console'], + 'level': 'INFO', + 'propagate': True, + }, + 'django_mailman3.lib.mailman': { + 'handlers': ['console'], + 'level': 'ERROR', + }, + 'django-q': { + 'handlers': ['console'], + 'level': 'WARNING', + }, + }, +} # Disable caching CACHES = { diff --git a/roles/mailman/handlers/main.yml b/roles/mailman/handlers/main.yml index 8efdefd78f..685b5fc917 100644 --- a/roles/mailman/handlers/main.yml +++ b/roles/mailman/handlers/main.yml @@ -7,3 +7,6 @@ - name: restart memcached service: name=memcached state=restarted + +- name: systemctl daemon-reload + command: /usr/bin/systemctl daemon-reload diff --git a/roles/mailman/tasks/main.yml b/roles/mailman/tasks/main.yml index 423ad39175..8ba14a361f 100644 --- a/roles/mailman/tasks/main.yml +++ b/roles/mailman/tasks/main.yml @@ -431,6 +431,15 @@ dest=/etc/yum/post-actions/hyperkitty.action tags: mailman +# Systemd +- name: install the systemd service file + template: src=webui-qcluster.service.j2 dest=/etc/systemd/system/webui-qcluster.service + notify: + - systemctl daemon-reload + tags: + - config + - mailman + # # Only run this on mailman01 for now. # TODO: run it on lists-dev too @@ -469,5 +478,6 @@ - httpd - mailman3 - postfix + - webui-qcluster tags: mailman when: inventory_hostname.startswith('mailman01.phx2') or inventory_hostname.startswith('lists-dev') diff --git a/roles/mailman/templates/settings.py.j2 b/roles/mailman/templates/settings.py.j2 index 132c769ee7..fe226daba5 100644 --- a/roles/mailman/templates/settings.py.j2 +++ b/roles/mailman/templates/settings.py.j2 @@ -66,6 +66,7 @@ INSTALLED_APPS = ( 'allauth', 'allauth.account', 'allauth.socialaccount', + 'django_q', 'django_mailman3.lib.auth.fedora', {% for service_name, service_data in mailman_login.items() %} 'allauth.socialaccount.providers.{{ service_data.provider }}', @@ -314,6 +315,16 @@ HAYSTACK_CONNECTIONS = { } +# +# Asynchronous tasks +# +Q_CLUSTER = { + 'timeout': 300, + 'save_limit': 500, + 'orm': 'default', +} + + # A sample logging configuration. The only tangible logging # performed by this configuration is to send an email to # the site admins on every HTTP 500 error when DEBUG=False. diff --git a/roles/mailman/templates/webui-qcluster.service.j2 b/roles/mailman/templates/webui-qcluster.service.j2 new file mode 100644 index 0000000000..1eb2eb854c --- /dev/null +++ b/roles/mailman/templates/webui-qcluster.service.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Mailman web UI async tasks runner +After=network.target remote-fs.target + +[Service] +ExecStart=/usr/bin/django-admin qcluster --pythonpath {{ mailman_webui_confdir }} --settings settings +User=apache +Restart=always + +[Install] +WantedBy=multi-user.target