diff --git a/roles/mailman/files/post-update.sh b/roles/mailman/files/post-update.sh
index 46fa2d80ad..5b94b32ccf 100755
--- a/roles/mailman/files/post-update.sh
+++ b/roles/mailman/files/post-update.sh
@@ -33,7 +33,7 @@ restorecon -r $BASEDIR/{bin,config,fulltext_index,static,templates}
 
 # Run unit tests
 echo "unit tests"
-django-admin test --pythonpath $CONFDIR --settings settings_test hyperkitty postorius django_mailman3
+django-admin test --pythonpath $CONFDIR --settings settings_test hyperkitty postorius
 
 # Restart services
 systemctl start httpd mailman3 crond
diff --git a/roles/mailman/files/settings_test.py b/roles/mailman/files/settings_test.py
index 1b5c79b6e0..b416a2ec4a 100644
--- a/roles/mailman/files/settings_test.py
+++ b/roles/mailman/files/settings_test.py
@@ -45,3 +45,10 @@ HAYSTACK_CONNECTIONS = {
 HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'
 
 LOGGING = {}
+
+# Disable caching
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
+    }
+}