Update to the latest changes in HyperKitty

roles/mailman/files/import-mm2.py

@@ -1,21 +1,28 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
+
+from __future__ import unicode_literals, absolute_import, print_function
 
 import os
 import sys
 import subprocess
 import pickle
 from optparse import OptionParser
+from locale import getpreferredencoding
 import yaml
 
+MAILMAN_BIN = subprocess.check_output(["which", "mailman"]).decode("ascii").strip()
+
+from mailman.commands.cli_import import Bouncer
+sys.modules["Mailman.Bouncer"] = Bouncer
 
 def call(command):
-    print "PYTHONPATH=%s" % os.environ["PYTHONPATH"], " ".join(command)
+    print(" ".join(command))
     subprocess.check_call(command, env=os.environ)
 
 def cmdget(command):
-    print "PYTHONPATH=%s" % os.environ["PYTHONPATH"], " ".join(command)
+    print(" ".join(command))
     out = subprocess.check_output(command, env=os.environ)
-    return out.strip()
+    return out.decode(getpreferredencoding()).strip()
 
 
 class Importer(object):
@@ -26,10 +33,11 @@ class Importer(object):
         self.index_path = self._get_index_path()
         self.existing_lists = [ l.strip() for l in
                 cmdget(["sudo", "-u", "mailman",
-                "mailman3", "lists", "-q"]).split("\n") ]
+                MAILMAN_BIN, "lists", "-q"]).split("\n") ]
         self.excluded = opts.exclude.strip().split(",")
 
     def _get_index_path(self):
+        return None
         sys.path.append(self.config["confdir"])
         settings = __import__("settings")
         sys.path.pop()
@@ -43,40 +51,41 @@ class Importer(object):
         for index, listname in enumerate(all_listnames):
             listaddr = "%s@%s" % (listname, self.config["domain"])
             if listname in self.excluded or listaddr in self.excluded:
-                print "Skipping excluded list %s" % listaddr
+                print("Skipping excluded list %s" % listaddr)
                 continue
-            print listaddr, "(%d/%d)" % (index+1, len(all_listnames))
+            print(listaddr, "(%d/%d)" % (index+1, len(all_listnames)))
             confpickle = os.path.join(mm2libdir, 'lists', listname,
                                       'config.pck')
             if not os.path.exists(confpickle):
-                print "Missing configuration pickle:", confpickle
+                print("Missing configuration pickle:", confpickle)
                 continue
             list_is_new = bool(listaddr not in self.existing_lists)
             if list_is_new:
-                call(["sudo", "-u", "mailman", "mailman3", "create", "-d",
+                call(["sudo", "-u", "mailman", MAILMAN_BIN, "create", "-d",
                       listaddr])
-                call(["sudo", "-u", "mailman", "mailman3", "import21",
+                call(["sudo", "-u", "mailman", MAILMAN_BIN, "import21",
                       listaddr, confpickle])
             if not self.opts.no_archives:
                 archivefile = os.path.join(
                         mm2libdir, "archives", "private",
                         "%s.mbox" % listname, "%s.mbox" % listname)
-                archive_policy = bool(pickle.load(
+                archive_policy = bool(pickle.load(open(confpickle, "rb"),
-                                      open(confpickle)).get('archive'))
+                    encoding="utf-8", errors="ignore").get('archive'))
                 if not archive_policy:
-                    print "List %s wants no archiving" % listname
+                    print("List %s wants no archiving" % listname)
                     continue
                 if os.path.exists(archivefile) and \
                         (list_is_new or not self.opts.new_only):
-                    call(["sudo", "kittystore-import", "-p",
+                    call(["sudo", "django-admin", "hyperkitty_import",
-                         self.config["confdir"], "-s", "settings_admin",
+                          "--pythonpath", self.config["confdir"],
-                         "-l", listaddr, "--continue", "--no-sync-mailman",
+                          "--settings", "settings", "-l", listaddr,
-                         archivefile])
+                          "--no-sync-mailman", archivefile])
                 if self.index_path:
                     call(["sudo", "chown", "mailman:apache", "-R", self.index_path])
                     call(["sudo", "chmod", "g+w", self.index_path])
-        call(["sudo", "kittystore-sync-mailman", "-p",
+        call(["sudo", "django-admin", "mailman_sync",
-             self.config["confdir"], "-s", "settings_admin"])
+              "--pythonpath", self.config["confdir"],
+              "--settings", "settings"])
 
 
 
@@ -101,10 +110,6 @@ def main():
     with open(opts.config) as conffile:
         config = yaml.safe_load(conffile)
 
-    sys.path.append(config["mm21codedir"])
-    # set the env var to propagate to subprocesses
-    os.environ["PYTHONPATH"] = config["mm21codedir"]
-
     importer = Importer(opts, config)
     importer.import_dir(mm2libdir)
 

roles/mailman/files/mailman-hyperkitty.cfg

@@ -8,5 +8,5 @@
 #
 
 [general]
-base_url: http://{{ mailman_url }}
+base_url: http://localhost/archives
-django_settings: {{ mailman_webui_confdir }}/settings.py
+api_key: SecretArchiverAPIKey

roles/mailman/files/pg-give-rights.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 # vim: et ts=4 sw=4 fileencoding=utf-8
 
 """
@@ -57,22 +57,22 @@ def main():
     site.addsitedir(conf["confdir"])
     import settings_admin
 
-    # KittyStore
+    ## KittyStore
-    dbspec = re.match("""
+    #dbspec = re.match("""
-        postgresql://
+    #    postgresql://
-        (?P<user>[a-z]+)
+    #    (?P<user>[a-z]+)
-        :
+    #    :
-        (?P<password>[^@]+)
+    #    (?P<password>[^@]+)
-        @
+    #    @
-        (?P<host>[^/]+)
+    #    (?P<host>[^/]+)
-        /
+    #    /
-        (?P<database>[^/?]+)
+    #    (?P<database>[^/?]+)
-        """, settings_admin.KITTYSTORE_URL, re.X)
+    #    """, settings_admin.KITTYSTORE_URL, re.X)
-    give_rights(dbspec.group("host"),
+    #give_rights(dbspec.group("host"),
-                dbspec.group("user"),
+    #            dbspec.group("user"),
-                dbspec.group("password"),
+    #            dbspec.group("password"),
-                dbspec.group("database")
+    #            dbspec.group("database")
-                )
+    #            )
 
     # HyperKitty
     give_rights(

roles/mailman/files/post-update.sh

@@ -8,13 +8,12 @@ export PATH=$PATH:$(dirname $(realpath $0)) # make yamlget available
 
 BASEDIR=`yamlget basedir $CONFFILE`
 CONFDIR=`yamlget confdir $CONFFILE`
-INDEXDIR=$BASEDIR/kittystore_search_index
+INDEXDIR=$BASEDIR/fulltext_index
 
 django-admin collectstatic --clear --noinput --verbosity 0 --pythonpath $CONFDIR --settings settings
 django-admin compress --pythonpath $CONFDIR --settings settings
 django-admin syncdb --pythonpath $CONFDIR --settings settings_admin --noinput --migrate
 django-admin loaddata /etc/postorius/sites/default/initial-user.json --pythonpath $CONFDIR --settings settings_admin
-kittystore-updatedb --pythonpath $CONFDIR --settings settings_admin
 chown mailman:mailman -R $INDEXDIR
 chmod g+w -R $INDEXDIR
 
@@ -27,7 +26,6 @@ restorecon -r "$BASEDIR"
 
 # Run unit tests
 django-admin test --pythonpath $CONFDIR --settings settings_test hyperkitty
-nose2 kittystore
 
 # Reload Apache to flush the python cache
 systemctl reload httpd

roles/mailman/files/settings_test.py

@@ -1,19 +0,0 @@
-#-*- coding: utf-8 -*-
-
-"""
-Copy of the Django settings file, but with databases set for unit tests.
-"""
-
-from settings import *
-try:
-    from settings_local import *
-except ImportError:
-    pass
-
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': ':memory:',
-    }
-}
-KITTYSTORE_URL = 'sqlite:'

roles/mailman/files/urls.py

@@ -10,11 +10,6 @@ from django.conf import settings
 from django.contrib import admin
 admin.autodiscover()
 
-# Import hyperkitty urls and set urlpatterns if you want to hook
-# hyperkitty into an existing django site.
-# Otherwise set ROOT_URLCONF in settings.py to
-# `hyperkitty.urls`.
-
 from django.core.urlresolvers import reverse_lazy
 from django.views.generic import RedirectView
 
@@ -23,6 +18,6 @@ urlpatterns = patterns('',
     #url(r'^$', 'postorius.views.list_index'),
     url(r'^admin/', include('postorius.urls')),
     url(r'^archives/', include('hyperkitty.urls')),
-    url(r'', include('social_auth.urls')),
+    url(r'', include('social_auth.urls'), {"SSL": True}),
-    url(r'', include('django_browserid.urls')),
+    url(r'', include('django_browserid.urls'), {"SSL": True}),
 )

roles/mailman/tasks/main.yml

@@ -13,7 +13,7 @@
   command: semanage fcontext -a -t etc_t "{{ mailman_webui_confdir }}(/.*)?"
 
 - name: set the SELinux policy for the fulltext index
-  command: semanage fcontext -a -t httpd_sys_content_t "{{ mailman_webui_basedir }}/kittystore_search_index(/.*)?"
+  command: semanage fcontext -a -t httpd_sys_rw_content_t "{{ mailman_webui_basedir }}/fulltext_index(/.*)?"
 
 - name: set the SELinux policy for the static files directory
   command: semanage fcontext -a -t httpd_sys_content_t "{{ mailman_webui_basedir }}/static(/.*)?"
@@ -21,6 +21,9 @@
 - name: set the SELinux policy for the log directory
   command: semanage fcontext -a -t httpd_log_t "/var/log/hyperkitty(/.*)?"
 
+- name: set the SELinux policy for the generated postfix databases
+  command: semanage fcontext -a -t etc_aliases_t "{{ mailman_webui_basedir }}/var/data/postfix_.*"
+
 - name: allow Apache to remotely connect to PostgreSQL
   seboolean: name=httpd_can_network_connect_db state=yes persistent=yes
 
@@ -50,15 +53,18 @@
   - mailman3
   - mailman3-selinux
   - python-psycopg2
-  - python-storm-postgresql
-  - kittystore
   - hyperkitty
   - hyperkitty-selinux
   - postorius
   - memcached
   - python-pylibmc
+  - python-django-haystack-xapian
   - yum-plugin-post-transaction-actions
   - mailman3-fedmsg-plugin
+  # to run the test suite:
+  - python-beautifulsoup4
+  - python-mock
+  - python-whoosh
   tags:
   - packages
 
@@ -73,6 +79,13 @@
   notify:
     - restart mailman3
 
+- name: add postfix to the mailman group
+  user: name=postfix groups=mailman append=yes
+  tags:
+  - config
+  notify:
+    - restart postfix
+
 # for access to the full-text index
 - name: add apache to the mailman group
   user: name=apache groups=mailman append=yes
@@ -130,13 +143,6 @@
   tags:
   - config
 
-- name: install the hyperkitty settings test file
-  copy: src=settings_test.py
-        dest="{{ mailman_webui_confdir }}/settings_test.py"
-        owner=root group=root mode=0640
-  tags:
-  - config
-
 - name: install the hyperkitty urls file
   copy: src=urls.py
         dest="{{ mailman_webui_confdir }}/urls.py"
@@ -175,8 +181,8 @@
   notify:
     - reload apache
 
-- name: create the kittystore plain text index dir
+- name: create the fulltext index dir
-  file: path="{{ mailman_webui_basedir }}/kittystore_search_index"
+  file: path="{{ mailman_webui_basedir }}/fulltext_index"
         state=directory owner=mailman group=mailman mode=0755
 
 - name: create the hyperkitty static files dir
@@ -187,7 +193,7 @@
 # Plug HyperKitty into Mailman
 #
 - name: copy the mailman-hyperkitty conffile
-  template: src=mailman-hyperkitty.cfg.j2
+  copy: src=mailman-hyperkitty.cfg
         dest="{{ mailman_webui_confdir }}/mailman-hyperkitty.cfg"
   tags:
   - config
@@ -228,18 +234,12 @@
   - pg-give-rights.py
   - post-update.sh
   - import-mm2.py
-  - clean-sessions.sh
 
 - name: copy the initial user fixture
   copy: src=postorius.initial-user.json
         dest=/etc/postorius/sites/default/initial-user.json
         owner=root group=apache mode=0640
 
-- name: install the cronjobs
-  file: dest=/etc/cron.daily/hyperkitty-clean-sessions.sh
-        src="{{ mailman_webui_basedir }}/bin/clean-sessions.sh"
-        state=link
-
 
 # Sync databases
 - name: install the post-transaction trigger

roles/mailman/templates/crontab.j2

@@ -0,0 +1,10 @@
+# This goes in /etc/cron.d/.
+# Replace "apache" by your webserver user ("www-data" on Debian systems) and
+# set the path to the Django project directory
+
+@hourly  apache  django-admin runjobs hourly  --pythonpath {{ mailman_webui_confdir }} --settings settings
+@daily   apache  django-admin runjobs daily   --pythonpath {{ mailman_webui_confdir }} --settings settings
+@weekly  apache  django-admin runjobs weekly  --pythonpath {{ mailman_webui_confdir }} --settings settings
+@monthly apache  django-admin runjobs monthly --pythonpath {{ mailman_webui_confdir }} --settings settings
+@yearly  apache  django-admin runjobs yearly  --pythonpath {{ mailman_webui_confdir }} --settings settings
+2,17,32,47 * * * * apache  django-admin runjobs quarter_hourly --pythonpath {{ mailman_webui_confdir }} --settings settings

roles/mailman/templates/mailman.cfg.j2

@@ -9,7 +9,13 @@
 site_owner: root@localhost
 
 # Set the paths to be Fedora-compliant
-layout: fhs
+#layout: fhs
+# Until the mailman3 rpm is back:
+layout: dev
+
+[paths.dev]
+var_dir = /srv/webui/var
+
 [paths.fhs]
 bin_dir: /usr/libexec/mailman3
 var_dir: /var/lib/mailman3
@@ -19,18 +25,18 @@ lock_dir: /run/lock/mailman3
 ext_dir: /etc/mailman3.d
 pid_file: /run/mailman3/master.pid
 
-[database]
+#[database]
-class: mailman.database.postgresql.PostgreSQLDatabase
+#class: mailman.database.postgresql.PostgreSQLDatabase
-url: postgresql://mailmanadmin:{{ mailman_mailman_db_pass }}@{{ mailman_db_server }}/mailman
+#url: postgresql://mailmanadmin:{{ mailman_mailman_db_pass }}@{{ mailman_db_server }}/mailman
 
 [archiver.hyperkitty]
-class: hyperkitty.archiver.Archiver
+class: mailman_hyperkitty.Archiver
 enable: yes
 configuration: /etc/mailman3.d/hyperkitty.cfg
 
-[archiver.fedmsg]
+#[archiver.fedmsg]
-class: mailman3_fedmsg_plugin.Archiver
+#class: mailman3_fedmsg_plugin.Archiver
-enable: yes
+#enable: yes
 
 [archiver.prototype]
 enable: yes

roles/mailman/templates/settings.py.j2

@@ -20,6 +20,8 @@ MANAGERS = ADMINS
 MAILMAN_REST_SERVER = MAILMAN_API_URL = 'http://localhost:8001'
 MAILMAN_API_USER = MAILMAN_USER = 'restadmin'
 MAILMAN_API_PASS = MAILMAN_PASS = 'restpass'
+MAILMAN_ARCHIVER_KEY = 'SecretArchiverAPIKey'
+MAILMAN_ARCHIVER_FROM = ("127.0.0.1", "::1")
 
 # CSS theme for postorius
 MAILMAN_THEME = "default"
@@ -37,7 +39,12 @@ DATABASES = {
 
 # Hosts/domain names that are valid for this site; required if DEBUG is False
 # See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
-ALLOWED_HOSTS = [".fedoraproject.org", "discuss.arquillian.org", "127.0.0.1"] # 127.0.0.1: HAProxy ping
+ALLOWED_HOSTS = [
+    ".fedoraproject.org",
+    "discuss.arquillian.org",
+    "localhost", # Archiving API from Mailman
+    "127.0.0.1", # HAProxy ping
+]
 # And for BrowserID too, see
 # http://django-browserid.rtfd.org/page/user/settings.html#django.conf.settings.BROWSERID_AUDIENCES
 BROWSERID_AUDIENCES = [
@@ -151,10 +158,8 @@ MIDDLEWARE_CLASSES = (
     'django.contrib.messages.middleware.MessageMiddleware',
     # Uncomment the next line for simple clickjacking protection:
     # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
-    'hyperkitty.middleware.KittyStoreDjangoMiddleware',
     'hyperkitty.middleware.SSLRedirect',
     'hyperkitty.middleware.TimezoneMiddleware',
-    'hyperkitty.middleware.MailmanUserMetadata',
 )
 
 ROOT_URLCONF = 'urls'
@@ -196,20 +201,18 @@ INSTALLED_APPS = (
     'django_gravatar',
     'south',
     'crispy_forms',
-    'compressor',
     'paintstore',
+    'compressor',
     'django_browserid',
+    'haystack',
+    'django_extensions',
     'postorius',
 )
 
 
-import django
+# Django 1.6 defaults to a JSON serializer, but it won't work with django-openid, see
-if django.VERSION[:2] < (1, 6):
+# https://bugs.launchpad.net/django-openid-auth/+bug/1252826
-    TEST_RUNNER = 'discover_runner.DiscoverRunner'
+SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
-else:
-    # Django 1.6 defaults to a JSON serializer, but it won't work with django-openid, see
-    # https://bugs.launchpad.net/django-openid-auth/+bug/1252826
-    SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
 
 
 REST_FRAMEWORK = {
@@ -273,6 +276,17 @@ COMPRESS_OFFLINE = True
 #INTERNAL_IPS = ('127.0.0.1',)
 
 
+#
+# Full-text search engine
+#
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
+        'PATH': "{{ mailman_webui_basedir }}/fulltext_index",
+    },
+}
+
+
 # A sample logging configuration. The only tangible logging
 # performed by this configuration is to send an email to
 # the site admins on every HTTP 500 error when DEBUG=False.
@@ -353,23 +367,12 @@ APP_NAME = 'Fedora list archives'
 # By default, only a login through Persona or your email provider is allowed.
 USE_INTERNAL_AUTH = False
 
-# URL to the KittyStore database
-KITTYSTORE_URL = 'postgresql://kittystoreapp:{{ mailman_kittystore_db_pass }}@{{ mailman_db_server }}/kittystore'
-# Path to the KittyStore search index (writable directory)
-KITTYSTORE_SEARCH_INDEX = '{{ mailman_webui_basedir }}/kittystore_search_index'
-
 # Use SSL when logged in
-#USE_SSL = True
+USE_SSL = True
-# In the Fedora infra, SSL is handled by the proxy and we can't detect it
-USE_SSL = False
 
 # Only display mailing-lists from the same virtual host as the webserver
 FILTER_VHOST = False
 
-# WARNING: the KITTYSTORE_DEBUG variable below will output every SQL query.
-# That's a huge amount of text, don't enable it if you don't need to.
-KITTYSTORE_DEBUG = False
-
 # This is for development purposes
 USE_MOCKUPS = False
 

roles/mailman/templates/settings_admin.py.j2

@@ -21,4 +21,3 @@ DATABASES = {
         'PORT': '',
     }
 }
-KITTYSTORE_URL = 'postgresql://kittystoreadmin:{{ mailman_kittystore_admin_db_pass }}@{{ mailman_db_server }}/kittystore'