diff --git a/roles/mailman/files/import-mm2.py b/roles/mailman/files/import-mm2.py index 15a01fa368..b0b844716a 100755 --- a/roles/mailman/files/import-mm2.py +++ b/roles/mailman/files/import-mm2.py @@ -1,21 +1,28 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 + +from __future__ import unicode_literals, absolute_import, print_function import os import sys import subprocess import pickle from optparse import OptionParser +from locale import getpreferredencoding import yaml +MAILMAN_BIN = subprocess.check_output(["which", "mailman"]).decode("ascii").strip() + +from mailman.commands.cli_import import Bouncer +sys.modules["Mailman.Bouncer"] = Bouncer def call(command): - print "PYTHONPATH=%s" % os.environ["PYTHONPATH"], " ".join(command) + print(" ".join(command)) subprocess.check_call(command, env=os.environ) def cmdget(command): - print "PYTHONPATH=%s" % os.environ["PYTHONPATH"], " ".join(command) + print(" ".join(command)) out = subprocess.check_output(command, env=os.environ) - return out.strip() + return out.decode(getpreferredencoding()).strip() class Importer(object): @@ -26,10 +33,11 @@ class Importer(object): self.index_path = self._get_index_path() self.existing_lists = [ l.strip() for l in cmdget(["sudo", "-u", "mailman", - "mailman3", "lists", "-q"]).split("\n") ] + MAILMAN_BIN, "lists", "-q"]).split("\n") ] self.excluded = opts.exclude.strip().split(",") def _get_index_path(self): + return None sys.path.append(self.config["confdir"]) settings = __import__("settings") sys.path.pop() @@ -43,40 +51,41 @@ class Importer(object): for index, listname in enumerate(all_listnames): listaddr = "%s@%s" % (listname, self.config["domain"]) if listname in self.excluded or listaddr in self.excluded: - print "Skipping excluded list %s" % listaddr + print("Skipping excluded list %s" % listaddr) continue - print listaddr, "(%d/%d)" % (index+1, len(all_listnames)) + print(listaddr, "(%d/%d)" % (index+1, len(all_listnames))) confpickle = os.path.join(mm2libdir, 'lists', listname, 'config.pck') if not os.path.exists(confpickle): - print "Missing configuration pickle:", confpickle + print("Missing configuration pickle:", confpickle) continue list_is_new = bool(listaddr not in self.existing_lists) if list_is_new: - call(["sudo", "-u", "mailman", "mailman3", "create", "-d", + call(["sudo", "-u", "mailman", MAILMAN_BIN, "create", "-d", listaddr]) - call(["sudo", "-u", "mailman", "mailman3", "import21", + call(["sudo", "-u", "mailman", MAILMAN_BIN, "import21", listaddr, confpickle]) if not self.opts.no_archives: archivefile = os.path.join( mm2libdir, "archives", "private", "%s.mbox" % listname, "%s.mbox" % listname) - archive_policy = bool(pickle.load( - open(confpickle)).get('archive')) + archive_policy = bool(pickle.load(open(confpickle, "rb"), + encoding="utf-8", errors="ignore").get('archive')) if not archive_policy: - print "List %s wants no archiving" % listname + print("List %s wants no archiving" % listname) continue if os.path.exists(archivefile) and \ (list_is_new or not self.opts.new_only): - call(["sudo", "kittystore-import", "-p", - self.config["confdir"], "-s", "settings_admin", - "-l", listaddr, "--continue", "--no-sync-mailman", - archivefile]) + call(["sudo", "django-admin", "hyperkitty_import", + "--pythonpath", self.config["confdir"], + "--settings", "settings", "-l", listaddr, + "--no-sync-mailman", archivefile]) if self.index_path: call(["sudo", "chown", "mailman:apache", "-R", self.index_path]) call(["sudo", "chmod", "g+w", self.index_path]) - call(["sudo", "kittystore-sync-mailman", "-p", - self.config["confdir"], "-s", "settings_admin"]) + call(["sudo", "django-admin", "mailman_sync", + "--pythonpath", self.config["confdir"], + "--settings", "settings"]) @@ -101,10 +110,6 @@ def main(): with open(opts.config) as conffile: config = yaml.safe_load(conffile) - sys.path.append(config["mm21codedir"]) - # set the env var to propagate to subprocesses - os.environ["PYTHONPATH"] = config["mm21codedir"] - importer = Importer(opts, config) importer.import_dir(mm2libdir) diff --git a/roles/mailman/templates/mailman-hyperkitty.cfg.j2 b/roles/mailman/files/mailman-hyperkitty.cfg similarity index 76% rename from roles/mailman/templates/mailman-hyperkitty.cfg.j2 rename to roles/mailman/files/mailman-hyperkitty.cfg index 7ea9aa0dbb..00b6a4e503 100644 --- a/roles/mailman/templates/mailman-hyperkitty.cfg.j2 +++ b/roles/mailman/files/mailman-hyperkitty.cfg @@ -8,5 +8,5 @@ # [general] -base_url: http://{{ mailman_url }} -django_settings: {{ mailman_webui_confdir }}/settings.py +base_url: http://localhost/archives +api_key: SecretArchiverAPIKey diff --git a/roles/mailman/files/pg-give-rights.py b/roles/mailman/files/pg-give-rights.py index 0da1455404..48fd2202c8 100755 --- a/roles/mailman/files/pg-give-rights.py +++ b/roles/mailman/files/pg-give-rights.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # vim: et ts=4 sw=4 fileencoding=utf-8 """ @@ -57,22 +57,22 @@ def main(): site.addsitedir(conf["confdir"]) import settings_admin - # KittyStore - dbspec = re.match(""" - postgresql:// - (?P[a-z]+) - : - (?P[^@]+) - @ - (?P[^/]+) - / - (?P[^/?]+) - """, settings_admin.KITTYSTORE_URL, re.X) - give_rights(dbspec.group("host"), - dbspec.group("user"), - dbspec.group("password"), - dbspec.group("database") - ) + ## KittyStore + #dbspec = re.match(""" + # postgresql:// + # (?P[a-z]+) + # : + # (?P[^@]+) + # @ + # (?P[^/]+) + # / + # (?P[^/?]+) + # """, settings_admin.KITTYSTORE_URL, re.X) + #give_rights(dbspec.group("host"), + # dbspec.group("user"), + # dbspec.group("password"), + # dbspec.group("database") + # ) # HyperKitty give_rights( diff --git a/roles/mailman/files/post-update.sh b/roles/mailman/files/post-update.sh index 5bb3b28e1e..4e708abddb 100755 --- a/roles/mailman/files/post-update.sh +++ b/roles/mailman/files/post-update.sh @@ -8,13 +8,12 @@ export PATH=$PATH:$(dirname $(realpath $0)) # make yamlget available BASEDIR=`yamlget basedir $CONFFILE` CONFDIR=`yamlget confdir $CONFFILE` -INDEXDIR=$BASEDIR/kittystore_search_index +INDEXDIR=$BASEDIR/fulltext_index django-admin collectstatic --clear --noinput --verbosity 0 --pythonpath $CONFDIR --settings settings django-admin compress --pythonpath $CONFDIR --settings settings django-admin syncdb --pythonpath $CONFDIR --settings settings_admin --noinput --migrate django-admin loaddata /etc/postorius/sites/default/initial-user.json --pythonpath $CONFDIR --settings settings_admin -kittystore-updatedb --pythonpath $CONFDIR --settings settings_admin chown mailman:mailman -R $INDEXDIR chmod g+w -R $INDEXDIR @@ -27,7 +26,6 @@ restorecon -r "$BASEDIR" # Run unit tests django-admin test --pythonpath $CONFDIR --settings settings_test hyperkitty -nose2 kittystore # Reload Apache to flush the python cache systemctl reload httpd diff --git a/roles/mailman/files/settings_test.py b/roles/mailman/files/settings_test.py deleted file mode 100644 index a1ee01a76b..0000000000 --- a/roles/mailman/files/settings_test.py +++ /dev/null @@ -1,19 +0,0 @@ -#-*- coding: utf-8 -*- - -""" -Copy of the Django settings file, but with databases set for unit tests. -""" - -from settings import * -try: - from settings_local import * -except ImportError: - pass - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ':memory:', - } -} -KITTYSTORE_URL = 'sqlite:' diff --git a/roles/mailman/files/urls.py b/roles/mailman/files/urls.py index 3717bfd26d..eb35b984d1 100644 --- a/roles/mailman/files/urls.py +++ b/roles/mailman/files/urls.py @@ -10,11 +10,6 @@ from django.conf import settings from django.contrib import admin admin.autodiscover() -# Import hyperkitty urls and set urlpatterns if you want to hook -# hyperkitty into an existing django site. -# Otherwise set ROOT_URLCONF in settings.py to -# `hyperkitty.urls`. - from django.core.urlresolvers import reverse_lazy from django.views.generic import RedirectView @@ -23,6 +18,6 @@ urlpatterns = patterns('', #url(r'^$', 'postorius.views.list_index'), url(r'^admin/', include('postorius.urls')), url(r'^archives/', include('hyperkitty.urls')), - url(r'', include('social_auth.urls')), - url(r'', include('django_browserid.urls')), + url(r'', include('social_auth.urls'), {"SSL": True}), + url(r'', include('django_browserid.urls'), {"SSL": True}), ) diff --git a/roles/mailman/tasks/main.yml b/roles/mailman/tasks/main.yml index ea0523e666..0663fb0d82 100644 --- a/roles/mailman/tasks/main.yml +++ b/roles/mailman/tasks/main.yml @@ -13,7 +13,7 @@ command: semanage fcontext -a -t etc_t "{{ mailman_webui_confdir }}(/.*)?" - name: set the SELinux policy for the fulltext index - command: semanage fcontext -a -t httpd_sys_content_t "{{ mailman_webui_basedir }}/kittystore_search_index(/.*)?" + command: semanage fcontext -a -t httpd_sys_rw_content_t "{{ mailman_webui_basedir }}/fulltext_index(/.*)?" - name: set the SELinux policy for the static files directory command: semanage fcontext -a -t httpd_sys_content_t "{{ mailman_webui_basedir }}/static(/.*)?" @@ -21,6 +21,9 @@ - name: set the SELinux policy for the log directory command: semanage fcontext -a -t httpd_log_t "/var/log/hyperkitty(/.*)?" +- name: set the SELinux policy for the generated postfix databases + command: semanage fcontext -a -t etc_aliases_t "{{ mailman_webui_basedir }}/var/data/postfix_.*" + - name: allow Apache to remotely connect to PostgreSQL seboolean: name=httpd_can_network_connect_db state=yes persistent=yes @@ -50,15 +53,18 @@ - mailman3 - mailman3-selinux - python-psycopg2 - - python-storm-postgresql - - kittystore - hyperkitty - hyperkitty-selinux - postorius - memcached - python-pylibmc + - python-django-haystack-xapian - yum-plugin-post-transaction-actions - mailman3-fedmsg-plugin + # to run the test suite: + - python-beautifulsoup4 + - python-mock + - python-whoosh tags: - packages @@ -73,6 +79,13 @@ notify: - restart mailman3 +- name: add postfix to the mailman group + user: name=postfix groups=mailman append=yes + tags: + - config + notify: + - restart postfix + # for access to the full-text index - name: add apache to the mailman group user: name=apache groups=mailman append=yes @@ -130,13 +143,6 @@ tags: - config -- name: install the hyperkitty settings test file - copy: src=settings_test.py - dest="{{ mailman_webui_confdir }}/settings_test.py" - owner=root group=root mode=0640 - tags: - - config - - name: install the hyperkitty urls file copy: src=urls.py dest="{{ mailman_webui_confdir }}/urls.py" @@ -175,8 +181,8 @@ notify: - reload apache -- name: create the kittystore plain text index dir - file: path="{{ mailman_webui_basedir }}/kittystore_search_index" +- name: create the fulltext index dir + file: path="{{ mailman_webui_basedir }}/fulltext_index" state=directory owner=mailman group=mailman mode=0755 - name: create the hyperkitty static files dir @@ -187,8 +193,8 @@ # Plug HyperKitty into Mailman # - name: copy the mailman-hyperkitty conffile - template: src=mailman-hyperkitty.cfg.j2 - dest="{{ mailman_webui_confdir }}/mailman-hyperkitty.cfg" + copy: src=mailman-hyperkitty.cfg + dest="{{ mailman_webui_confdir }}/mailman-hyperkitty.cfg" tags: - config notify: @@ -228,18 +234,12 @@ - pg-give-rights.py - post-update.sh - import-mm2.py - - clean-sessions.sh - name: copy the initial user fixture copy: src=postorius.initial-user.json dest=/etc/postorius/sites/default/initial-user.json owner=root group=apache mode=0640 -- name: install the cronjobs - file: dest=/etc/cron.daily/hyperkitty-clean-sessions.sh - src="{{ mailman_webui_basedir }}/bin/clean-sessions.sh" - state=link - # Sync databases - name: install the post-transaction trigger diff --git a/roles/mailman/templates/crontab.j2 b/roles/mailman/templates/crontab.j2 new file mode 100644 index 0000000000..79a0847d18 --- /dev/null +++ b/roles/mailman/templates/crontab.j2 @@ -0,0 +1,10 @@ +# This goes in /etc/cron.d/. +# Replace "apache" by your webserver user ("www-data" on Debian systems) and +# set the path to the Django project directory + +@hourly apache django-admin runjobs hourly --pythonpath {{ mailman_webui_confdir }} --settings settings +@daily apache django-admin runjobs daily --pythonpath {{ mailman_webui_confdir }} --settings settings +@weekly apache django-admin runjobs weekly --pythonpath {{ mailman_webui_confdir }} --settings settings +@monthly apache django-admin runjobs monthly --pythonpath {{ mailman_webui_confdir }} --settings settings +@yearly apache django-admin runjobs yearly --pythonpath {{ mailman_webui_confdir }} --settings settings +2,17,32,47 * * * * apache django-admin runjobs quarter_hourly --pythonpath {{ mailman_webui_confdir }} --settings settings diff --git a/roles/mailman/templates/mailman.cfg.j2 b/roles/mailman/templates/mailman.cfg.j2 index 9005b85d5c..21bb8ffa3b 100644 --- a/roles/mailman/templates/mailman.cfg.j2 +++ b/roles/mailman/templates/mailman.cfg.j2 @@ -9,7 +9,13 @@ site_owner: root@localhost # Set the paths to be Fedora-compliant -layout: fhs +#layout: fhs +# Until the mailman3 rpm is back: +layout: dev + +[paths.dev] +var_dir = /srv/webui/var + [paths.fhs] bin_dir: /usr/libexec/mailman3 var_dir: /var/lib/mailman3 @@ -19,18 +25,18 @@ lock_dir: /run/lock/mailman3 ext_dir: /etc/mailman3.d pid_file: /run/mailman3/master.pid -[database] -class: mailman.database.postgresql.PostgreSQLDatabase -url: postgresql://mailmanadmin:{{ mailman_mailman_db_pass }}@{{ mailman_db_server }}/mailman +#[database] +#class: mailman.database.postgresql.PostgreSQLDatabase +#url: postgresql://mailmanadmin:{{ mailman_mailman_db_pass }}@{{ mailman_db_server }}/mailman [archiver.hyperkitty] -class: hyperkitty.archiver.Archiver +class: mailman_hyperkitty.Archiver enable: yes configuration: /etc/mailman3.d/hyperkitty.cfg -[archiver.fedmsg] -class: mailman3_fedmsg_plugin.Archiver -enable: yes +#[archiver.fedmsg] +#class: mailman3_fedmsg_plugin.Archiver +#enable: yes [archiver.prototype] enable: yes diff --git a/roles/mailman/templates/settings.py.j2 b/roles/mailman/templates/settings.py.j2 index f6eaffef7e..a501e08690 100644 --- a/roles/mailman/templates/settings.py.j2 +++ b/roles/mailman/templates/settings.py.j2 @@ -20,6 +20,8 @@ MANAGERS = ADMINS MAILMAN_REST_SERVER = MAILMAN_API_URL = 'http://localhost:8001' MAILMAN_API_USER = MAILMAN_USER = 'restadmin' MAILMAN_API_PASS = MAILMAN_PASS = 'restpass' +MAILMAN_ARCHIVER_KEY = 'SecretArchiverAPIKey' +MAILMAN_ARCHIVER_FROM = ("127.0.0.1", "::1") # CSS theme for postorius MAILMAN_THEME = "default" @@ -37,7 +39,12 @@ DATABASES = { # Hosts/domain names that are valid for this site; required if DEBUG is False # See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts -ALLOWED_HOSTS = [".fedoraproject.org", "discuss.arquillian.org", "127.0.0.1"] # 127.0.0.1: HAProxy ping +ALLOWED_HOSTS = [ + ".fedoraproject.org", + "discuss.arquillian.org", + "localhost", # Archiving API from Mailman + "127.0.0.1", # HAProxy ping +] # And for BrowserID too, see # http://django-browserid.rtfd.org/page/user/settings.html#django.conf.settings.BROWSERID_AUDIENCES BROWSERID_AUDIENCES = [ @@ -151,10 +158,8 @@ MIDDLEWARE_CLASSES = ( 'django.contrib.messages.middleware.MessageMiddleware', # Uncomment the next line for simple clickjacking protection: # 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'hyperkitty.middleware.KittyStoreDjangoMiddleware', 'hyperkitty.middleware.SSLRedirect', 'hyperkitty.middleware.TimezoneMiddleware', - 'hyperkitty.middleware.MailmanUserMetadata', ) ROOT_URLCONF = 'urls' @@ -196,20 +201,18 @@ INSTALLED_APPS = ( 'django_gravatar', 'south', 'crispy_forms', - 'compressor', 'paintstore', + 'compressor', 'django_browserid', + 'haystack', + 'django_extensions', 'postorius', ) -import django -if django.VERSION[:2] < (1, 6): - TEST_RUNNER = 'discover_runner.DiscoverRunner' -else: - # Django 1.6 defaults to a JSON serializer, but it won't work with django-openid, see - # https://bugs.launchpad.net/django-openid-auth/+bug/1252826 - SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer' +# Django 1.6 defaults to a JSON serializer, but it won't work with django-openid, see +# https://bugs.launchpad.net/django-openid-auth/+bug/1252826 +SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer' REST_FRAMEWORK = { @@ -273,6 +276,17 @@ COMPRESS_OFFLINE = True #INTERNAL_IPS = ('127.0.0.1',) +# +# Full-text search engine +# +HAYSTACK_CONNECTIONS = { + 'default': { + 'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine', + 'PATH': "{{ mailman_webui_basedir }}/fulltext_index", + }, +} + + # A sample logging configuration. The only tangible logging # performed by this configuration is to send an email to # the site admins on every HTTP 500 error when DEBUG=False. @@ -353,23 +367,12 @@ APP_NAME = 'Fedora list archives' # By default, only a login through Persona or your email provider is allowed. USE_INTERNAL_AUTH = False -# URL to the KittyStore database -KITTYSTORE_URL = 'postgresql://kittystoreapp:{{ mailman_kittystore_db_pass }}@{{ mailman_db_server }}/kittystore' -# Path to the KittyStore search index (writable directory) -KITTYSTORE_SEARCH_INDEX = '{{ mailman_webui_basedir }}/kittystore_search_index' - # Use SSL when logged in -#USE_SSL = True -# In the Fedora infra, SSL is handled by the proxy and we can't detect it -USE_SSL = False +USE_SSL = True # Only display mailing-lists from the same virtual host as the webserver FILTER_VHOST = False -# WARNING: the KITTYSTORE_DEBUG variable below will output every SQL query. -# That's a huge amount of text, don't enable it if you don't need to. -KITTYSTORE_DEBUG = False - # This is for development purposes USE_MOCKUPS = False diff --git a/roles/mailman/templates/settings_admin.py.j2 b/roles/mailman/templates/settings_admin.py.j2 index 92e9755a86..41f256743e 100644 --- a/roles/mailman/templates/settings_admin.py.j2 +++ b/roles/mailman/templates/settings_admin.py.j2 @@ -21,4 +21,3 @@ DATABASES = { 'PORT': '', } } -KITTYSTORE_URL = 'postgresql://kittystoreadmin:{{ mailman_kittystore_admin_db_pass }}@{{ mailman_db_server }}/kittystore'