Tested the mailman 2->3 migration

roles/mailman/files/import-mm2.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import subprocess
+import pickle
+from optparse import OptionParser
+
+
+def call(command):
+    print "PYTHONPATH=%s" % os.environ["PYTHONPATH"], " ".join(command)
+    subprocess.check_call(command, env=os.environ)
+
+def cmdget(command):
+    print "PYTHONPATH=%s" % os.environ["PYTHONPATH"], " ".join(command)
+    out = subprocess.check_output(command, env=os.environ)
+    return out.strip()
+
+
+class Importer(object):
+
+    def __init__(self, opts, config):
+        self.opts = opts
+        self.config = config
+        self.existing_lists = cmdget(
+                ["sudo", "-u", "mailman", "mailman3", "lists", "-q"])
+        self.index_path = self._get_index_path()
+
+    def _get_index_path(self):
+        sys.path.append(self.config["confdir"])
+        settings = __import__("settings")
+        sys.path.pop()
+        return settings.KITTYSTORE_SEARCH_INDEX
+
+    def import_dir(self, mm2libdir):
+        all_listnames = [ d for d in os.listdir(
+                                os.path.join(mm2libdir, 'lists'))
+                          if not d.startswith(".") ]
+        all_listnames.sort()
+        for index, listname in enumerate(all_listnames):
+            listaddr = "%s@%s" % (listname, self.config["domain"])
+            print listaddr, "(%d/%d)" % (index+1, len(all_listnames))
+            confpickle = os.path.join(mm2libdir, 'lists', listname,
+                                      'config.pck')
+            if not os.path.exists(confpickle):
+                print "Missing configuration pickle:", confpickle
+                continue
+            list_is_new = bool(listaddr not in self.existing_lists)
+            if list_is_new:
+                call(["sudo", "-u", "mailman", "mailman3", "create", "-d",
+                      listaddr])
+                call(["sudo", "-u", "mailman", "mailman3", "import21",
+                      listaddr, confpickle])
+            if not self.opts.no_archives:
+                archivefile = os.path.join(
+                        mm2libdir, "archives", "private",
+                        "%s.mbox" % listname, "%s.mbox" % listname)
+                archive_policy = bool(pickle.load(
+                                      open(confpickle)).get('archive'))
+                if not archive_policy:
+                    print "List %s wants no archiving" % listname
+                    continue
+                if os.path.exists(archivefile) and \
+                        (list_is_new or not self.opts.new_only):
+                    call(["sudo", "kittystore-import", "-p",
+                         self.config["confdir"], "-s", "settings_admin",
+                         "-l", listaddr, "--continue", archivefile])
+                if self.index_path:
+                    call(["sudo", "chown", "mailman:", "-R", self.index_path])
+
+
+
+def main():
+    parser = OptionParser()
+    parser.add_option("-n", "--new-only", action="store_true",
+                      help="Only import the archives when the list is new")
+    parser.add_option("-A", "--no-archives", action="store_true",
+                  help="Don't import the archives, only import the list config")
+    parser.add_option("-c", "--config", default="/etc/mailman-migration.conf",
+                  help="Configuration file (default: %defaults)")
+    opts, args = parser.parse_args()
+    if len(args) != 1:
+        parser.error("Only one arg: the Mailman 2.1 lib dir to import")
+
+    mm2libdir = args[0]
+    if not os.path.exists(mm2libdir):
+        parser.error("No such directory: %s" % mm2libdir)
+
+    with open(opts.config) as conffile:
+        config = yaml.safe_load(conffile)
+
+    sys.path.append(config["mm21codedir"])
+    # set the env var to propagate to subprocesses
+    os.environ["PYTHONPATH"] = config["mm21codedir"]
+
+    importer = Importer(opts, config)
+    importer.import_dir(mm2libdir)
+
+
+if __name__ == "__main__":
+    main()

roles/mailman/files/pg-give-rights.py

@@ -5,11 +5,12 @@
 Give non-admin rights to the database app user.
 """
 
-CONF_DIR = "{{ mailman_webui_confdir }}"
+CONFFILE = "/etc/mailman-migration.conf"
 
 
 import site
 import re
+import yaml
 import psycopg2
 
 
@@ -36,7 +37,9 @@ def give_rights(dbhost, dbuser, dbpasswd, dbname):
 
 
 def main():
-    site.addsitedir(CONF_DIR)
+    with open(CONFFILE) as conffile:
+        conf = yaml.safe_load(conffile)
+    site.addsitedir(conf["confdir"])
     import settings_admin
 
     # KittyStore

roles/mailman/files/post-update.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 
-BASEDIR={{ mailman_webui_basedir }}
-CONFDIR={{ mailman_webui_confdir }}
+CONFFILE=/etc/mailman-migration.conf
+BASEDIR=`yamlget basedir $CONFFILE`
+CONFDIR=`yamlget confdir $CONFFILE`
 
 set -e 
 

roles/mailman/files/yamlget

@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 tabstop=4 shiftwidth=4 expandtab smartindent:
+
+u"""
+yamlget
+-------
+
+Output any key in a YAML-formatted file. The aim is to make such a
+configuration file accessible to shell scripts.
+
+.. :Authors:
+      Aurélien Bompard <aurelien@bompard.org> <http://aurelien.bompard.org>
+
+.. :License:
+      GNU GPL v3 or later
+"""
+
+from __future__ import print_function
+
+import os
+import sys
+from optparse import OptionParser
+
+import yaml
+
+
+def get_key(fullkey, data):
+    """
+    Get the requested key from the parsed data.
+    :param fullkey: the key to get, nested values can be accessed by using a
+                    colon (":") as the separator
+    :param data: the parsed data, from yaml.load()
+
+    Examples:
+
+    >>> data = {
+    ...  'bool': [True, False, True, False],
+    ...  'dict': {'hp': 13, 'sp': 5},
+    ...  'float': 3.14159,
+    ...  'int': 42,
+    ...  'list': ['LITE', 'RES_ACID', 'SUS_DEXT'],
+    ...  'none': [None, None],
+    ...  'text': "The Set of Gauntlets 'Pauraegen'",
+    ... }
+    >>> get_key('bool', data)
+    [True, False, True, False]
+    >>> get_key('bool:2', data)
+    False
+    >>> get_key('dict', data)
+    {'hp': 13, 'sp': 5}
+    >>> get_key('dict:hp', data)
+    13
+    >>> get_key('float', data)
+    3.14159
+    >>> get_key('int', data)
+    42
+    >>> get_key('list', data)
+    ['LITE', 'RES_ACID', 'SUS_DEXT']
+    >>> get_key('list:2', data)
+    'RES_ACID'
+    >>> get_key('list:2:5', data)
+    'RES_ACID'
+    >>> get_key('none', data)
+    [None, None]
+    >>> get_key('none:1', data)
+    >>> get_key('text', data)
+    "The Set of Gauntlets 'Pauraegen'"
+    >>> get_key('2', ['item1', 'item2', 'item3'])
+    'item2'
+    """
+
+    value = data
+    while value is not None:
+        key, _sep, fullkey = fullkey.partition(":")
+        if isinstance(value, list):
+            try:
+                key = int(key)
+            except TypeError:
+                print("Wrong key format: %s, it should be an integer" % key,
+                      file=sys.stderr)
+                sys.exit(1)
+            value = value[key - 1] # start at 1, not 0
+        elif isinstance(value, dict):
+            value = value.get(key)
+        else:
+            break # we've got the value now
+        if not fullkey:
+            break # can't go any further
+    return value
+
+def main():
+    parser = OptionParser(usage="%prog <key> <yaml-file>")
+    args = parser.parse_args()[1]
+    if len(args) != 2:
+        parser.error("wrong number of arguments")
+    fullkey, filepath = args
+    if not os.path.exists(filepath):
+        parser.error("no such file: %s" % filepath)
+
+    with open(filepath) as yamlfile:
+        data = yaml.safe_load_all(yamlfile).next()
+
+    #from pprint import pprint; pprint(data)
+    value = get_key(fullkey, data)
+    if value is not None:
+        print(value)
+
+
+if __name__ == "__main__":
+    main()

roles/mailman/tasks/main.yml

@@ -146,16 +146,29 @@
     - restart mailman3
 
 
-# Post-update script
-- name: install the post-update script
-  template: src=post-update.sh.j2
-            dest=${mailman_webui_basedir}/bin/post-update.sh
-            owner=root group=root mode=0755
+# Scripts
+- name: install the migration conffile
+  template: src=mailman-migration.conf.j2
+            dest=/etc/mailman-migration.conf
+            owner=root group=root mode=0644
 
-- name: install the post-update database script
-  template: src=pg-give-rights.py.j2
-            dest=${mailman_webui_basedir}/bin/pg-give-rights.py
-            owner=root group=root mode=0755
+- name: create the scripts dir
+  file: path=${mailman_webui_basedir}/bin
+        state=directory owner=root group=root mode=0755
+
+- name: install the migration environment
+  template: src=mailman-migration-path.sh.j2
+            dest=/etc/profile.d/mailman-migration-path.sh
+            owner=root group=root mode=0644
+
+- name: install the scripts
+  copy: src=${item} dest=${mailman_webui_basedir}/bin/${item}
+        owner=root group=root mode=0755
+  with_items:
+  - yamlget
+  - pg-give-rights.py
+  - post-update.sh
+  - import-mm2.py
 
 - name: copy the initial user fixture
   copy: src=postorius.initial-user.json

roles/mailman/templates/mailman-migration-path.sh.j2

@@ -0,0 +1,3 @@
+PATH=$PATH:{{ mailman_webui_basedir }}
+export PATH
+

roles/mailman/templates/mailman-migration.conf.j2

@@ -0,0 +1,4 @@
+basedir: {{ mailman_webui_basedir }}
+confdir: {{ mailman_webui_confdir }}
+mm21codedir: /usr/lib/mailman
+domain: lists.fedoraproject.org

roles/mailman/templates/mailman.cfg.j2

@@ -30,3 +30,106 @@ configuration: /etc/mailman3.d/hyperkitty.cfg
 
 [archiver.prototype]
 enable: yes
+
+
+# http://www.lingoes.net/en/translator/langcode.htm
+
+[language.pt]
+description: Protuguese
+charset: iso-8859-15
+enabled: yes
+
+[language.cs]
+description: Czech
+charset: utf-8
+enabled: yes
+
+[language.ca]
+description: Catalan
+charset: utf-8
+enabled: yes
+
+[language.ja]
+description: Japanese
+charset: utf-8
+enabled: yes
+
+[language.ar]
+description: Arabic
+charset: utf-8
+enabled: yes
+
+[language.nl]
+description: Dutch
+charset: utf-8
+enabled: yes
+
+[language.pl]
+description: Polish
+charset: utf-8
+enabled: yes
+
+[language.es]
+description: Spanish
+charset: utf-8
+enabled: yes
+
+[language.pt_BR]
+description: Protuguese (Brazil)
+charset: iso-8859-15
+enabled: yes
+
+[language.zh_CN]
+description: Chinese (S)
+charset: utf-8
+enabled: yes
+
+[language.zh_TW]
+description: Chinese (T)
+charset: utf-8
+enabled: yes
+
+[language.ru]
+description: Russian
+charset: utf-8
+enabled: yes
+
+[language.vi]
+description: Vietnamese
+charset: utf-8
+enabled: yes
+
+[language.it]
+description: Italian
+charset: utf-8
+enabled: yes
+
+[language.fr]
+description: French
+charset: utf-8
+enabled: yes
+
+[language.ro]
+description: Romanian
+charset: utf-8
+enabled: yes
+
+[language.de]
+description: German
+charset: utf-8
+enabled: yes
+
+[language.hu]
+description: Hungarian
+charset: utf-8
+enabled: yes
+
+[language.ko]
+description: Korean
+charset: utf-8
+enabled: yes
+
+[language.uk]
+description: Ukrainian
+charset: utf-8
+enabled: yes

roles/mailman/templates/settings.py.j2

@@ -166,7 +166,7 @@ INSTALLED_APPS = (
     'django.contrib.auth',
     'django.contrib.contenttypes',
     'django.contrib.sessions',
-    'django.contrib.sites',
+    #'django.contrib.sites',
     'django.contrib.messages',
     'django.contrib.staticfiles',
     # Uncomment the next line to enable the admin: