diff --git a/roles/mailman/files/import-mm2.py b/roles/mailman/files/import-mm2.py new file mode 100755 index 0000000000..13467433f7 --- /dev/null +++ b/roles/mailman/files/import-mm2.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python + +import os +import sys +import subprocess +import pickle +from optparse import OptionParser + + +def call(command): + print "PYTHONPATH=%s" % os.environ["PYTHONPATH"], " ".join(command) + subprocess.check_call(command, env=os.environ) + +def cmdget(command): + print "PYTHONPATH=%s" % os.environ["PYTHONPATH"], " ".join(command) + out = subprocess.check_output(command, env=os.environ) + return out.strip() + + +class Importer(object): + + def __init__(self, opts, config): + self.opts = opts + self.config = config + self.existing_lists = cmdget( + ["sudo", "-u", "mailman", "mailman3", "lists", "-q"]) + self.index_path = self._get_index_path() + + def _get_index_path(self): + sys.path.append(self.config["confdir"]) + settings = __import__("settings") + sys.path.pop() + return settings.KITTYSTORE_SEARCH_INDEX + + def import_dir(self, mm2libdir): + all_listnames = [ d for d in os.listdir( + os.path.join(mm2libdir, 'lists')) + if not d.startswith(".") ] + all_listnames.sort() + for index, listname in enumerate(all_listnames): + listaddr = "%s@%s" % (listname, self.config["domain"]) + print listaddr, "(%d/%d)" % (index+1, len(all_listnames)) + confpickle = os.path.join(mm2libdir, 'lists', listname, + 'config.pck') + if not os.path.exists(confpickle): + print "Missing configuration pickle:", confpickle + continue + list_is_new = bool(listaddr not in self.existing_lists) + if list_is_new: + call(["sudo", "-u", "mailman", "mailman3", "create", "-d", + listaddr]) + call(["sudo", "-u", "mailman", "mailman3", "import21", + listaddr, confpickle]) + if not self.opts.no_archives: + archivefile = os.path.join( + mm2libdir, "archives", "private", + "%s.mbox" % listname, "%s.mbox" % listname) + archive_policy = bool(pickle.load( + open(confpickle)).get('archive')) + if not archive_policy: + print "List %s wants no archiving" % listname + continue + if os.path.exists(archivefile) and \ + (list_is_new or not self.opts.new_only): + call(["sudo", "kittystore-import", "-p", + self.config["confdir"], "-s", "settings_admin", + "-l", listaddr, "--continue", archivefile]) + if self.index_path: + call(["sudo", "chown", "mailman:", "-R", self.index_path]) + + + +def main(): + parser = OptionParser() + parser.add_option("-n", "--new-only", action="store_true", + help="Only import the archives when the list is new") + parser.add_option("-A", "--no-archives", action="store_true", + help="Don't import the archives, only import the list config") + parser.add_option("-c", "--config", default="/etc/mailman-migration.conf", + help="Configuration file (default: %defaults)") + opts, args = parser.parse_args() + if len(args) != 1: + parser.error("Only one arg: the Mailman 2.1 lib dir to import") + + mm2libdir = args[0] + if not os.path.exists(mm2libdir): + parser.error("No such directory: %s" % mm2libdir) + + with open(opts.config) as conffile: + config = yaml.safe_load(conffile) + + sys.path.append(config["mm21codedir"]) + # set the env var to propagate to subprocesses + os.environ["PYTHONPATH"] = config["mm21codedir"] + + importer = Importer(opts, config) + importer.import_dir(mm2libdir) + + +if __name__ == "__main__": + main() diff --git a/roles/mailman/templates/pg-give-rights.py.j2 b/roles/mailman/files/pg-give-rights.py old mode 100644 new mode 100755 similarity index 91% rename from roles/mailman/templates/pg-give-rights.py.j2 rename to roles/mailman/files/pg-give-rights.py index abeb193112..893d90a194 --- a/roles/mailman/templates/pg-give-rights.py.j2 +++ b/roles/mailman/files/pg-give-rights.py @@ -5,11 +5,12 @@ Give non-admin rights to the database app user. """ -CONF_DIR = "{{ mailman_webui_confdir }}" +CONFFILE = "/etc/mailman-migration.conf" import site import re +import yaml import psycopg2 @@ -36,7 +37,9 @@ def give_rights(dbhost, dbuser, dbpasswd, dbname): def main(): - site.addsitedir(CONF_DIR) + with open(CONFFILE) as conffile: + conf = yaml.safe_load(conffile) + site.addsitedir(conf["confdir"]) import settings_admin # KittyStore diff --git a/roles/mailman/templates/post-update.sh.j2 b/roles/mailman/files/post-update.sh old mode 100644 new mode 100755 similarity index 84% rename from roles/mailman/templates/post-update.sh.j2 rename to roles/mailman/files/post-update.sh index c1acadeb06..e66500b177 --- a/roles/mailman/templates/post-update.sh.j2 +++ b/roles/mailman/files/post-update.sh @@ -1,7 +1,8 @@ #!/bin/bash -BASEDIR={{ mailman_webui_basedir }} -CONFDIR={{ mailman_webui_confdir }} +CONFFILE=/etc/mailman-migration.conf +BASEDIR=`yamlget basedir $CONFFILE` +CONFDIR=`yamlget confdir $CONFFILE` set -e diff --git a/roles/mailman/files/yamlget b/roles/mailman/files/yamlget new file mode 100755 index 0000000000..68a5876bf9 --- /dev/null +++ b/roles/mailman/files/yamlget @@ -0,0 +1,110 @@ +#!/usr/bin/env python +# vim: set fileencoding=utf-8 tabstop=4 shiftwidth=4 expandtab smartindent: + +u""" +yamlget +------- + +Output any key in a YAML-formatted file. The aim is to make such a +configuration file accessible to shell scripts. + +.. :Authors: + Aurélien Bompard + +.. :License: + GNU GPL v3 or later +""" + +from __future__ import print_function + +import os +import sys +from optparse import OptionParser + +import yaml + + +def get_key(fullkey, data): + """ + Get the requested key from the parsed data. + :param fullkey: the key to get, nested values can be accessed by using a + colon (":") as the separator + :param data: the parsed data, from yaml.load() + + Examples: + + >>> data = { + ... 'bool': [True, False, True, False], + ... 'dict': {'hp': 13, 'sp': 5}, + ... 'float': 3.14159, + ... 'int': 42, + ... 'list': ['LITE', 'RES_ACID', 'SUS_DEXT'], + ... 'none': [None, None], + ... 'text': "The Set of Gauntlets 'Pauraegen'", + ... } + >>> get_key('bool', data) + [True, False, True, False] + >>> get_key('bool:2', data) + False + >>> get_key('dict', data) + {'hp': 13, 'sp': 5} + >>> get_key('dict:hp', data) + 13 + >>> get_key('float', data) + 3.14159 + >>> get_key('int', data) + 42 + >>> get_key('list', data) + ['LITE', 'RES_ACID', 'SUS_DEXT'] + >>> get_key('list:2', data) + 'RES_ACID' + >>> get_key('list:2:5', data) + 'RES_ACID' + >>> get_key('none', data) + [None, None] + >>> get_key('none:1', data) + >>> get_key('text', data) + "The Set of Gauntlets 'Pauraegen'" + >>> get_key('2', ['item1', 'item2', 'item3']) + 'item2' + """ + + value = data + while value is not None: + key, _sep, fullkey = fullkey.partition(":") + if isinstance(value, list): + try: + key = int(key) + except TypeError: + print("Wrong key format: %s, it should be an integer" % key, + file=sys.stderr) + sys.exit(1) + value = value[key - 1] # start at 1, not 0 + elif isinstance(value, dict): + value = value.get(key) + else: + break # we've got the value now + if not fullkey: + break # can't go any further + return value + +def main(): + parser = OptionParser(usage="%prog ") + args = parser.parse_args()[1] + if len(args) != 2: + parser.error("wrong number of arguments") + fullkey, filepath = args + if not os.path.exists(filepath): + parser.error("no such file: %s" % filepath) + + with open(filepath) as yamlfile: + data = yaml.safe_load_all(yamlfile).next() + + #from pprint import pprint; pprint(data) + value = get_key(fullkey, data) + if value is not None: + print(value) + + +if __name__ == "__main__": + main() diff --git a/roles/mailman/tasks/main.yml b/roles/mailman/tasks/main.yml index d0744f45fc..e898259b11 100644 --- a/roles/mailman/tasks/main.yml +++ b/roles/mailman/tasks/main.yml @@ -146,16 +146,29 @@ - restart mailman3 -# Post-update script -- name: install the post-update script - template: src=post-update.sh.j2 - dest=${mailman_webui_basedir}/bin/post-update.sh - owner=root group=root mode=0755 +# Scripts +- name: install the migration conffile + template: src=mailman-migration.conf.j2 + dest=/etc/mailman-migration.conf + owner=root group=root mode=0644 -- name: install the post-update database script - template: src=pg-give-rights.py.j2 - dest=${mailman_webui_basedir}/bin/pg-give-rights.py - owner=root group=root mode=0755 +- name: create the scripts dir + file: path=${mailman_webui_basedir}/bin + state=directory owner=root group=root mode=0755 + +- name: install the migration environment + template: src=mailman-migration-path.sh.j2 + dest=/etc/profile.d/mailman-migration-path.sh + owner=root group=root mode=0644 + +- name: install the scripts + copy: src=${item} dest=${mailman_webui_basedir}/bin/${item} + owner=root group=root mode=0755 + with_items: + - yamlget + - pg-give-rights.py + - post-update.sh + - import-mm2.py - name: copy the initial user fixture copy: src=postorius.initial-user.json diff --git a/roles/mailman/templates/mailman-migration-path.sh.j2 b/roles/mailman/templates/mailman-migration-path.sh.j2 new file mode 100644 index 0000000000..31711313d0 --- /dev/null +++ b/roles/mailman/templates/mailman-migration-path.sh.j2 @@ -0,0 +1,3 @@ +PATH=$PATH:{{ mailman_webui_basedir }} +export PATH + diff --git a/roles/mailman/templates/mailman-migration.conf.j2 b/roles/mailman/templates/mailman-migration.conf.j2 new file mode 100644 index 0000000000..5220339e3d --- /dev/null +++ b/roles/mailman/templates/mailman-migration.conf.j2 @@ -0,0 +1,4 @@ +basedir: {{ mailman_webui_basedir }} +confdir: {{ mailman_webui_confdir }} +mm21codedir: /usr/lib/mailman +domain: lists.fedoraproject.org diff --git a/roles/mailman/templates/mailman.cfg.j2 b/roles/mailman/templates/mailman.cfg.j2 index 3487ae2343..054b52e35a 100644 --- a/roles/mailman/templates/mailman.cfg.j2 +++ b/roles/mailman/templates/mailman.cfg.j2 @@ -30,3 +30,106 @@ configuration: /etc/mailman3.d/hyperkitty.cfg [archiver.prototype] enable: yes + + +# http://www.lingoes.net/en/translator/langcode.htm + +[language.pt] +description: Protuguese +charset: iso-8859-15 +enabled: yes + +[language.cs] +description: Czech +charset: utf-8 +enabled: yes + +[language.ca] +description: Catalan +charset: utf-8 +enabled: yes + +[language.ja] +description: Japanese +charset: utf-8 +enabled: yes + +[language.ar] +description: Arabic +charset: utf-8 +enabled: yes + +[language.nl] +description: Dutch +charset: utf-8 +enabled: yes + +[language.pl] +description: Polish +charset: utf-8 +enabled: yes + +[language.es] +description: Spanish +charset: utf-8 +enabled: yes + +[language.pt_BR] +description: Protuguese (Brazil) +charset: iso-8859-15 +enabled: yes + +[language.zh_CN] +description: Chinese (S) +charset: utf-8 +enabled: yes + +[language.zh_TW] +description: Chinese (T) +charset: utf-8 +enabled: yes + +[language.ru] +description: Russian +charset: utf-8 +enabled: yes + +[language.vi] +description: Vietnamese +charset: utf-8 +enabled: yes + +[language.it] +description: Italian +charset: utf-8 +enabled: yes + +[language.fr] +description: French +charset: utf-8 +enabled: yes + +[language.ro] +description: Romanian +charset: utf-8 +enabled: yes + +[language.de] +description: German +charset: utf-8 +enabled: yes + +[language.hu] +description: Hungarian +charset: utf-8 +enabled: yes + +[language.ko] +description: Korean +charset: utf-8 +enabled: yes + +[language.uk] +description: Ukrainian +charset: utf-8 +enabled: yes diff --git a/roles/mailman/templates/settings.py.j2 b/roles/mailman/templates/settings.py.j2 index b9505bb6c9..7ac1f9620d 100644 --- a/roles/mailman/templates/settings.py.j2 +++ b/roles/mailman/templates/settings.py.j2 @@ -166,7 +166,7 @@ INSTALLED_APPS = ( 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', - 'django.contrib.sites', + #'django.contrib.sites', 'django.contrib.messages', 'django.contrib.staticfiles', # Uncomment the next line to enable the admin: