Setup HyperKitty and Postorius in the same Django instance

files/mailman/mailman2-import.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+HKCONFDIR="/etc/hyperkitty/sites/default"
+MMDIR=$1
+DOMAIN=$2
+
+if [ -z "$MMDIR" ]; then
+    echo  "Usage: $0 <mailman-lib-directory>"
+    exit 2
+fi
+
+[ -z "$DOMAIN" ] && DOMAIN=lists.fedoraproject.org
+
+existinglists=`mktemp`
+trap "rm -f $existinglists" EXIT
+sudo -u mailman mailman3 lists -q > $existinglists
+
+for listname in `ls $MMDIR/lists`; do
+    listaddr="$listname@$DOMAIN"
+    if ! grep -qs $listaddr $existinglists; then
+        echo "sudo -u mailman mailman3 create -d $listaddr"
+        echo "sudo -u mailman PYTHONPATH=/usr/lib/mailman mailman3 import21 $listaddr $MMDIR/lists/$listname/config.pck"
+    fi
+    echo "sudo kittystore-import -p $HKCONFDIR -s settings_admin -l $listaddr --continue $MMDIR/archives/private/${listname}.mbox/${listname}.mbox"
+done

playbooks/groups/mailman.yml

@@ -74,24 +74,12 @@
       postgresql_user: name=kittystoreadmin password=$mailman_ks_admin_db_pass
     - name: kittystore DB user
       postgresql_user: name=kittystoreapp password=$mailman_ks_db_pass
-    - name: postorius DB admin user
-      postgresql_user: name=postoriusadmin password=$mailman_ps_admin_db_pass
-    - name: postorius DB user
-      postgresql_user: name=postoriusapp password=$mailman_ps_db_pass
     - name: databases creation
       postgresql_db: name=$item owner=${item}admin encoding=UTF-8
       with_items:
         - mailman
         - hyperkitty
         - kittystore
-        - postorius
-#    - name: database users permissions
-#      postgresql_privs: database=$item role=${item}app type=database
-#                        priv=SELECT,INSERT,UPDATE,DELETE
-#      with_items:
-#        - hyperkitty
-#        - kittystore
-#        - postorius
 
 
 # Real MM/HK-specific work
@@ -109,16 +97,20 @@
   - /srv/web/infra/ansible/roles/mailman
 
   tasks:
-  - name: server front page
+  - name: install more needed packages
-    copy: src=$files/mailman/index.html
+    yum: pkg=$item state=installed
-          dest=/var/www/html/index.html
+    with_items:
+    - tar
+    - mailman # transition from mailman2.1
+    tags:
+    - packages
 
   #- name: easy access to the postgresql databases
   #  template: src=$files/mailman/pgpass.j2 dest=/root/.pgpass
   #            owner=root group=root mode=0600
 
   - name: start services
-    service: state=started name=$item
+    service: state=started enabled=yes name=$item
     with_items:
       - httpd
       - mailman3

roles/mailman/files/hyperkitty.apache.conf

@@ -1,28 +0,0 @@
-#Alias /robots.txt  /etc/hyperkitty/sites/default/static/robots.txt
-#Alias /favicon.ico /etc/hyperkitty/sites/default/static/favicon.ico
-Alias /hyperkitty/static      /var/lib/hyperkitty/sites/default/static
-
-#ErrorLog /var/log/httpd/hyperkitty_error.log
-#CustomLog /var/log/httpd/hyperkitty_access.log combined
-
-WSGIScriptAlias /hyperkitty /etc/hyperkitty/sites/default/wsgi.py
-WSGIDaemonProcess hyperkitty display-name=hyperkitty maximum-requests=1000 processes=4 threads=4 python-path=/etc/hyperkitty/sites/default
-WSGISocketPrefix run/wsgi
-WSGIRestrictStdout On
-WSGIRestrictSignal Off
-WSGIPythonOptimize 1
-
-<Directory "/etc/hyperkitty/sites/default">
-    <Files wsgi.py>
-        Order deny,allow
-        Allow from all
-        Require all granted
-    </Files>
-    WSGIProcessGroup hyperkitty
-</Directory>
-
-<Directory "/var/lib/hyperkitty/sites/default/static">
-    Order deny,allow
-    Allow from all
-    Require all granted
-</Directory>

roles/mailman/files/hyperkitty.cfg

@@ -1,3 +0,0 @@
-[general]
-base_url: http://mailman01.stg.phx2.fedoraproject.org/
-django_settings: /etc/hyperkitty/sites/default/settings.py

roles/mailman/files/hyperkitty.urls.py

@@ -1,37 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (C) 1998-2012 by the Free Software Foundation, Inc.
-#
-# This file is part of HyperKitty.
-#
-# HyperKitty is free software: you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free
-# Software Foundation, either version 3 of the License, or (at your option)
-# any later version.
-#
-# HyperKitty is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-# more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# HyperKitty.  If not, see <http://www.gnu.org/licenses/>.
-
-import hyperkitty
-
-from django.conf.urls.defaults import *
-from django.conf import settings
-
-# Uncomment the next two lines to enable the admin:
-from django.contrib import admin
-admin.autodiscover()
-
-# Import mailman urls and set urlpatterns if you want to hook
-# mailman_django into an existing django site. 
-# Otherwise set ROOT_URLCONF in settings.py to
-# `mailman_django.urls`.
-# from mailman_django import urls as mailman_urls
-
-urlpatterns = patterns('',
-    url(r'^$', 'hyperkitty.views.pages.index'),
-    url(r'', include('hyperkitty.urls')),
-)

roles/mailman/files/postorius.apache.conf

@@ -1,28 +0,0 @@
-#Alias /robots.txt  /var/lib/postorius/sites/default/static/robots.txt
-#Alias /favicon.ico /var/lib/postorius/sites/default/static/favicon.ico
-Alias /postorius/static      /var/lib/postorius/sites/default/static
-
-#ErrorLog /var/log/httpd/postorius_error.log
-#CustomLog /var/log/httpd/postorius_access.log combined
-
-WSGIScriptAlias /postorius /etc/postorius/sites/default/srv/postorius.wsgi
-WSGIDaemonProcess postorius display-name=postorius maximum-requests=1000 processes=4 threads=4 python-path=/etc/postorius/sites/default
-WSGISocketPrefix run/wsgi
-WSGIRestrictStdout On
-WSGIRestrictSignal Off
-WSGIPythonOptimize 1
-
-<Directory "/etc/postorius/sites/default/srv">
-    <Files postorius.wsgi>
-        Order deny,allow
-        Allow from all
-        Require all granted
-    </Files>
-    WSGIProcessGroup postorius
-</Directory>
-
-<Directory "/var/lib/postorius/sites/default/static">
-    Order deny,allow
-    Allow from all
-    Require all granted
-</Directory>

roles/mailman/files/postorius.urls.py

@@ -1,38 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (C) 1998-2012 by the Free Software Foundation, Inc.
-#
-# This file is part of Postorius.
-#
-# Postorius is free software: you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free
-# Software Foundation, either version 3 of the License, or (at your option)
-# any later version.
-#
-# Postorius is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-# more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# Postorius.  If not, see <http://www.gnu.org/licenses/>.
-
-import postorius
-
-from django.conf.urls import *
-from django.conf import settings
-
-# Uncomment the next two lines to enable the admin:
-from django.contrib import admin
-admin.autodiscover()
-
-# Import mailman urls and set urlpatterns if you want to hook
-# mailman_django into an existing django site. 
-# Otherwise set ROOT_URLCONF in settings.py to
-# `mailman_django.urls`.
-# from mailman_django import urls as mailman_urls
-
-urlpatterns = patterns('',
-    url(r'^$', 'postorius.views.list_index'),
-    url(r'', include('postorius.urls')),
-	url(r'', include('social_auth.urls')),
-)

roles/mailman/files/urls.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+
+import hyperkitty
+import postorius
+
+from django.conf.urls.defaults import *
+from django.conf import settings
+
+# Uncomment the next two lines to enable the admin:
+from django.contrib import admin
+admin.autodiscover()
+
+# Import mailman urls and set urlpatterns if you want to hook
+# mailman_django into an existing django site. 
+# Otherwise set ROOT_URLCONF in settings.py to
+# `mailman_django.urls`.
+# from mailman_django import urls as mailman_urls
+
+urlpatterns = patterns('',
+    #url(r'^$', 'postorius.views.list_index'),
+    url(r'^admin/', include('postorius.urls')),
+    url(r'^$', 'hyperkitty.views.pages.index'),
+    url(r'^archives/', include('hyperkitty.urls')),
+    url(r'', include('social_auth.urls')),
+)

roles/mailman/files/webui.wsgi

@@ -0,0 +1,44 @@
+"""
+WSGI config for hyperkitty_standalone project.
+
+This module contains the WSGI application used by Django's development server
+and any production WSGI deployments. It should expose a module-level variable
+named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover
+this application via the ``WSGI_APPLICATION`` setting.
+
+Usually you will have the standard Django WSGI application here, but it also
+might make sense to replace the whole Django WSGI application with a custom one
+that later delegates to the Django one. For example, you could introduce WSGI
+middleware here, or combine a Django application with an application of another
+framework.
+
+"""
+import os
+import sys
+import site
+
+
+## For some unknown reason, sometimes mod_wsgi fails to set the python paths to
+## the virtualenv, with the 'python-path' option. You can do it here too.
+##
+## Remember original sys.path.
+#prev_sys_path = list(sys.path)
+## Add here, for the settings module
+#site.addsitedir(os.path.abspath(os.path.dirname(__file__)))
+## Add the virtualenv
+#venv = os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', 'lib', 'python2.6', 'site-packages')
+#site.addsitedir(venv)
+# Reorder sys.path so new directories at the front.
+#new_sys_path = []
+#for item in list(sys.path):
+#    if item not in prev_sys_path:
+#        new_sys_path.append(item)
+#        sys.path.remove(item)
+#        sys.path[:0] = new_sys_path
+
+site.addsitedir(os.path.abspath(os.path.dirname(__file__)))
+
+os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
+
+from django.core.wsgi import get_wsgi_application
+application = get_wsgi_application()

roles/mailman/tasks/main.yml

@@ -44,46 +44,52 @@
   copy: src=hyperkitty.logrotate.conf
         dest=/etc/logrotate.d/hyperkitty
 
-# settings / conf
+#
-- name: install the hyperkitty/postorius settings file
+# HyperKitty + Postorius setup
-  template: src=$item.settings_local.py.j2
+#
-            dest=/etc/$item/sites/default/settings_local.py
+
+- name: create the configuration directory
+  file: path=${mailman_webui_confdir} state=directory
+
+- name: install the hyperkitty settings file
+  template: src=settings.py.j2
+            dest=${mailman_webui_confdir}/settings.py
             owner=root group=apache mode=0640
-  with_items:
-    - hyperkitty
-    - postorius
   tags:
   - config
   notify:
     - reload apache
     - restart mailman3
 
-- name: install the hyperkitty/postorius settings admin file
+- name: install the hyperkitty settings admin file
-  template: src=$item.settings_admin.py.j2
+  template: src=settings_admin.py.j2
-            dest=/etc/$item/sites/default/settings_admin.py
+            dest=${mailman_webui_confdir}/settings_admin.py
             owner=root group=root mode=0600
-  with_items:
-    - hyperkitty
-    - postorius
   tags:
   - config
 
-- name: install the hyperkitty/postorius urls file
+- name: install the hyperkitty urls file
-  copy: src=$item.urls.py
+  copy: src=urls.py
-        dest=/etc/$item/sites/default/urls.py
+        dest=${mailman_webui_confdir}/urls.py
         owner=root group=root mode=0644
-  with_items:
-    - hyperkitty
-    - postorius
   tags:
   - config
   notify:
     - reload apache
     - restart mailman3
 
-- name: install the hyperkitty/postorius httpd conf file
+- name: install the hyperkitty wsgi file
-  copy: src=$item.apache.conf
+  copy: src=webui.wsgi
-        dest=/etc/httpd/conf.d/$item.conf
+        dest=${mailman_webui_confdir}/webui.wsgi
+        owner=root group=root mode=0644
+  tags:
+  - config
+  notify:
+    - reload apache
+
+- name: install the hyperkitty/postorius dummy httpd conf file
+  template: src=apache-dummy.conf.j2
+            dest=/etc/httpd/conf.d/$item.conf
   with_items:
     - hyperkitty
     - postorius
@@ -92,38 +98,80 @@
   notify:
     - reload apache
 
-- name: set the hyperkitty conffile in mailman
+- name: install the hyperkitty httpd conf file
-  copy: src=hyperkitty.cfg
+  template: src=apache.conf.j2
+            dest=${mailman_webui_confdir}/apache.conf
+  tags:
+  - config
+  notify:
+    - reload apache
+
+- name: install the hyperkitty httpd conf symlink
+  file: src=${mailman_webui_confdir}/apache.conf
+        dest=/etc/httpd/conf.d/mailman-webui.conf
+        state=link
+  tags:
+  - config
+  notify:
+    - reload apache
+
+- name: copy the manage.py script
+  command: /bin/cp /etc/hyperkitty/sites/default/manage.py ${mailman_webui_confdir}/manage.py
+           creates=${mailman_webui_confdir}/manage.py
+
+- name: create the kittystore plain text index dir
+  file: path=${mailman_webui_basedir}/kittystore_search_index
+        state=directory owner=mailman group=mailman mode=0755
+
+- name: create the hyperkitty static files dir
+  file: path=${mailman_webui_basedir}/static
+        state=directory owner=root group=root mode=0755
+
+
+# Plug HyperKitty into Mailman
+- name: copy the mailman-hyperkitty conffile
+  template: src=mailman-hyperkitty.cfg.j2
+            dest=${mailman_webui_confdir}/mailman-hyperkitty.cfg
+  tags:
+  - config
+  notify:
+    - restart mailman3
+
+- name: symlink the hyperkitty conffile in mailman directory
+  copy: src=${mailman_webui_confdir}/mailman-hyperkitty.cfg
         dest=/etc/mailman3.d/hyperkitty.cfg
+        state=link
   tags:
   - config
   notify:
     - restart mailman3
 
 
+# Post-update script
+- name: install the post-update script
+  template: src=post-update.sh.j2
+            dest=${mailman_webui_basedir}/bin/post-update.sh
+            owner=root group=root mode=0755
+
+- name: install the post-update database script
+  template: src=pg-give-rights.py.j2
+            dest=${mailman_webui_basedir}/bin/pg-give-rights.py
+            owner=root group=root mode=0755
+
+- name: copy the initial user fixture
+  copy: src=postorius.initial-user.json
+        dest=/etc/postorius/sites/default/initial-user.json
+        owner=root group=apache mode=0640
+
 # Sync databases
 # FIXME the db migrations should happen no matter when the pkgs are updated
 # so really this should be run once
 # and then stored in a yum-post-transaction-action on the system(s)
 # so no matter when it is run the right thing happens
 #
-- name: run django syncdb
+- name: run the post-update script
-  command: /usr/bin/django-admin syncdb --pythonpath=/etc/$item/sites/default --settings=settings_admin
+  command: ${mailman_webui_basedir}/bin/post-update.sh
-  with_items:
-    - hyperkitty
-    - postorius
-- name: run django migrate
-  command: /usr/bin/django-admin migrate --pythonpath=/etc/hyperkitty/sites/default --settings=settings_admin hyperkitty
-- name: kittystore schema update
-  command: /usr/bin/kittystore-updatedb --pythonpath=/etc/hyperkitty/sites/default --settings=settings_admin
 
-- name: copy the initial user fixture
-  copy: src=postorius.initial-user.json
-        dest=/etc/postorius/sites/default/initial-user.json
-        owner=root group=apache mode=0640
-
-- name: load the inital user
-  command: /usr/bin/django-admin loaddata --pythonpath=/etc/postorius/sites/default --settings=settings /etc/postorius/sites/default/initial-user.json
 
 
 # Postfix

roles/mailman/templates/apache-dummy.conf.j2

@@ -0,0 +1 @@
+# See ${mailman_webui_confdir}

roles/mailman/templates/apache.conf.j2

@@ -0,0 +1,28 @@
+Alias /robots.txt  ${mailman_webui_basedir}/static/hyperkitty/robots.txt
+Alias /favicon.ico ${mailman_webui_basedir}/static/hyperkitty/favicon.ico
+Alias /static      ${mailman_webui_basedir}/static
+
+#ErrorLog /var/log/httpd/webui_error.log
+#CustomLog /var/log/httpd/webui_access.log combined
+
+WSGIScriptAlias / ${mailman_webui_confdir}/webui.wsgi
+WSGIDaemonProcess webui display-name=webui maximum-requests=1000 processes=4 threads=4
+WSGISocketPrefix run/wsgi
+WSGIRestrictStdout On
+WSGIRestrictSignal Off
+WSGIPythonOptimize 1
+
+<Directory "${mailman_webui_confdir}">
+    <Files webui.wsgi>
+        Order deny,allow
+        Allow from all
+        Require all granted
+    </Files>
+    WSGIProcessGroup webui
+</Directory>
+
+<Directory "${mailman_webui_basedir}/static">
+    Order deny,allow
+    Allow from all
+    Require all granted
+</Directory>

roles/mailman/templates/hyperkitty.settings_local.py.j2

@@ -1,101 +0,0 @@
-#-*- coding: utf-8 -*-
-
-"""
-Django local settings for the hyperkitty project.
-"""
-
-DEBUG = False
-TEMPLATE_DEBUG = DEBUG
-
-ADMINS = ()
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.postgresql_psycopg2',
-        'NAME': 'hyperkitty',
-        'USER': 'hyperkittyapp',
-        'PASSWORD': '{{ mailman_hk_db_pass }}',
-        'HOST': 'db02.stg.phx2.fedoraproject.org',
-        'PORT': '',
-    }
-}
-# URL prefix for static files.
-# Example: "http://media.lawrence.com/static/"
-STATIC_URL = '/hyperkitty/static/'
-
-# Add Fedora to the authentication backends
-AUTHENTICATION_BACKENDS = (
-    'social_auth.backends.browserid.BrowserIDBackend',
-    'hyperkitty.lib.fedora.FedoraBackend',
-    'social_auth.backends.google.GoogleBackend',
-    'social_auth.backends.yahoo.YahooBackend',
-    #'social_auth.backends.OpenIDBackend',
-    'django.contrib.auth.backends.ModelBackend',
-)
-
-# Use the email address as the username (add associate_by_email).
-SOCIAL_AUTH_PIPELINE = (
-    'social_auth.backends.pipeline.social.social_auth_user',
-    'social_auth.backends.pipeline.associate.associate_by_email',
-    'social_auth.backends.pipeline.user.get_username',
-    'social_auth.backends.pipeline.user.create_user',
-    'social_auth.backends.pipeline.social.associate_user',
-    'social_auth.backends.pipeline.social.load_extra_data',
-    'social_auth.backends.pipeline.user.update_user_details'
-)
-
-# A sample logging configuration. The only tangible logging
-# performed by this configuration is to send an email to
-# the site admins on every HTTP 500 error.
-# See http://docs.djangoproject.com/en/dev/topics/logging for
-# more details on how to customize your logging configuration.
-LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': False,
-    'handlers': {
-        'mail_admins': {
-            'level': 'ERROR',
-            'class': 'django.utils.log.AdminEmailHandler'
-        },
-        'file':{
-            'level': 'DEBUG',
-            #'class': 'logging.handlers.RotatingFileHandler',
-            'class': 'logging.FileHandler',
-            'filename': '/var/log/hyperkitty/hyperkitty.log',
-            'formatter': 'verbose',
-        },
-    },
-    'loggers': {
-        'django.request': {
-            'handlers': ['file'],
-            'level': 'ERROR',
-            'propagate': True,
-        },
-        'django': {
-            'handlers': ['file'],
-            'level': 'ERROR',
-            'propagate': True,
-        },
-        'hyperkitty': {
-            'handlers': ['file'],
-            'level': 'DEBUG',
-            'propagate': True,
-        },
-    },
-    'formatters': {
-        'verbose': {
-            'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
-        },
-        'simple': {
-            'format': '%(levelname)s %(message)s'
-        },
-    },
-}
-
-# Make this unique, and don't share it with anybody.
-SECRET_KEY = '{{ mailman_hk_cookie_key }}'
-
-APP_NAME = 'Fedora list archives'
-KITTYSTORE_URL = 'postgres://kittystoreapp:{{ mailman_ks_db_pass }}@db02.stg.phx2.fedoraproject.org/kittystore'
-KITTYSTORE_SEARCH_INDEX = '/var/lib/hyperkitty/sites/default/kittystore_search_index'
-KITTYSTORE_DEBUG = False
-USE_MOCKUPS = False

roles/mailman/templates/mailman-hyperkitty.cfg.j2

@@ -0,0 +1,12 @@
+# This is the mailman extension configuration file to enable HyperKitty as an
+# archiver. Remember to add the following lines in the mailman.cfg file:
+#
+# [archiver.hyperkitty]
+# class: hyperkitty.lib.archiver.Archiver
+# enable: yes
+# configuration: /path/to/here/hyperkitty.cfg
+#
+
+[general]
+base_url: http://apps.stg.fedoraproject.org/lists
+django_settings: ${mailman_webui_confdir}/settings.py

roles/mailman/templates/pg-give-rights.py.j2

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# vim: et ts=4 sw=4 fileencoding=utf-8
+
+"""
+Give non-admin rights to the database app user.
+"""
+
+CONF_DIR = "{{ mailman_webui_confdir }}"
+
+
+import site
+import re
+import psycopg2
+
+
+def give_rights(dbhost, dbuser, dbpasswd, dbname):
+    conn = psycopg2.connect(host=dbhost, user=dbuser, password=dbpasswd,
+                            database=dbname)
+    cur = conn.cursor()
+    dbrightsquery = "GRANT CONNECT,TEMP ON DATABASE %s TO %sapp;" % (dbname, dbname)
+    print dbrightsquery
+    cur.execute(dbrightsquery)
+    cur.execute("""
+        SELECT 'GRANT SELECT,INSERT,UPDATE,DELETE,TRUNCATE ON ' || relname || ' TO %sapp;'
+        FROM pg_class
+        JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace
+        WHERE nspname = 'public' AND relkind IN ('r', 'v');
+    """ % dbname)
+    queries = [ q[0] for q in cur ]
+    for query in queries:
+        print query
+        cur.execute(query)
+    conn.commit()
+    cur.close()
+    conn.close()
+
+
+def main():
+    site.addsitedir(CONF_DIR)
+    import settings_admin
+
+    # KittyStore
+    dbspec = re.match("""
+        postgres://
+        (?P<user>[a-z]+)
+        :
+        (?P<password>[^@]+)
+        @
+        (?P<host>[^/]+)
+        /
+        (?P<database>[^/?]+)
+        """, settings_admin.KITTYSTORE_URL, re.X)
+    give_rights(dbspec.group("host"),
+                dbspec.group("user"),
+                dbspec.group("password"),
+                dbspec.group("database")
+                )
+
+    # HyperKitty
+    give_rights(
+        settings_admin.DATABASES["default"]["HOST"],
+        settings_admin.DATABASES["default"]["USER"],
+        settings_admin.DATABASES["default"]["PASSWORD"],
+        settings_admin.DATABASES["default"]["NAME"],
+    )
+
+
+if __name__ == "__main__": main()

roles/mailman/templates/post-update.sh.j2

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+BASEDIR={{ mailman_webui_basedir }}
+CONFDIR={{ mailman_webui_confdir }}
+
+set -e 
+
+django-admin collectstatic --clear --noinput --pythonpath $CONFDIR --settings settings
+django-admin assets build --parse-templates --pythonpath $CONFDIR --settings settings
+django-admin syncdb --pythonpath $CONFDIR --settings settings_admin
+django-admin migrate hyperkitty --pythonpath $CONFDIR --settings settings_admin
+django-admin loaddata /etc/postorius/sites/default/initial-user.json --pythonpath $CONFDIR --settings settings_admin
+kittystore-updatedb --pythonpath $CONFDIR --settings settings_admin
+
+# Give database rights to the non-admin user
+$CONFDIR/pg-give-rights.py

roles/mailman/templates/postorius.settings_admin.py.j2

@@ -1,20 +0,0 @@
-#-*- coding: utf-8 -*-
-
-"""
-Copy of the Django settings file, but with database admin credentials (for
-schema modifications)
-"""
-
-from settings import *
-from settings_local import *
-
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.postgresql_psycopg2',
-        'NAME': 'postorius',
-        'USER': 'postoriusadmin',
-        'PASSWORD': '{{ mailman_ps_admin_db_pass }}',
-        'HOST': 'db02.stg.phx2.fedoraproject.org',
-        'PORT': '',
-    }
-}

roles/mailman/templates/postorius.settings_local.py.j2

@@ -1,34 +0,0 @@
-#-*- coding: utf-8 -*-
-
-"""
-Django local settings for the postorious project.
-"""
-
-
-DEBUG = False
-TEMPLATE_DEBUG = DEBUG
-
-ADMINS = ()
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.postgresql_psycopg2',
-        'NAME': 'postorius',
-        'USER': 'postoriusapp',
-        'PASSWORD': '{{ mailman_ps_db_pass }}',
-        'HOST': 'db02.stg.phx2.fedoraproject.org',
-        'PORT': '',
-    }
-}
-
-# URL that handles the media served from STATIC_ROOT. Make sure to use a
-STATIC_URL = '/postorius/static/'
-# URL that handles the media served from MEDIA_ROOT. Make sure to use a
-MEDIA_URL = '/postorius/media/'
-
-# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
-# trailing slash.
-# Examples: "http://foo.com/media/", "/media/".
-ADMIN_MEDIA_PREFIX = '/postorius/static/admin/'
-
-# Make this unique, and don't share it with anybody.
-SECRET_KEY = '{{ mailman_ps_cookie_key }}'

roles/mailman/templates/settings.py.j2

@@ -0,0 +1,332 @@
+#-*- coding: utf-8 -*-
+"""
+Django settings for HyperKitty + Postorius
+"""
+
+import os
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+
+DEBUG = False
+TEMPLATE_DEBUG = DEBUG
+
+ADMINS = (
+     ('HyperKitty Admin', 'root@localhost'),
+)
+
+MANAGERS = ADMINS
+
+# Mailman API credentials
+MAILMAN_REST_SERVER = MAILMAN_API_URL = 'http://localhost:8001'
+MAILMAN_API_USER = MAILMAN_USER = 'restadmin'
+MAILMAN_API_PASS = MAILMAN_PASS = 'restpass'
+
+# CSS theme for postorius
+MAILMAN_THEME = "default"
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': 'hyperkitty',
+        'USER': 'hyperkittyapp',
+        'PASSWORD': '{{ mailman_hk_db_pass }}',
+        'HOST': '{{ mailman_dbserver }}',
+        'PORT': '',                      # Set to empty string for default.
+    }
+}
+
+# Hosts/domain names that are valid for this site; required if DEBUG is False
+# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
+ALLOWED_HOSTS = [".fedoraproject.org"]
+
+# We're behind a proxy, use the X-Forwarded-Host header
+# See https://docs.djangoproject.com/en/1.5/ref/settings/#use-x-forwarded-host
+USE_X_FORWARDED_HOST = True
+
+# Local time zone for this installation. Choices can be found here:
+# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
+# although not all choices may be available on all operating systems.
+# In a Windows environment this must be set to your system time zone.
+TIME_ZONE = 'America/Chicago'
+
+# Language code for this installation. All choices can be found here:
+# http://www.i18nguy.com/unicode/language-identifiers.html
+LANGUAGE_CODE = 'en-us'
+
+SITE_ID = 1
+
+# If you set this to False, Django will make some optimizations so as not
+# to load the internationalization machinery.
+USE_I18N = True
+
+# If you set this to False, Django will not format dates, numbers and
+# calendars according to the current locale.
+USE_L10N = True
+
+# If you set this to False, Django will not use timezone-aware datetimes.
+USE_TZ = True
+
+# Absolute filesystem path to the directory that will hold user-uploaded files.
+# Example: "/var/www/example.com/media/"
+MEDIA_ROOT = ''
+
+# URL that handles the media served from MEDIA_ROOT. Make sure to use a
+# trailing slash.
+# Examples: "http://example.com/media/", "http://media.example.com/"
+MEDIA_URL = ''
+
+# Absolute path to the directory static files should be collected to.
+# Don't put anything in this directory yourself; store your static files
+# in apps' "static/" subdirectories and in STATICFILES_DIRS.
+# Example: "/var/www/example.com/static/"
+#STATIC_ROOT = ''
+STATIC_ROOT = "{{ mailman_webui_basedir }}/static/"
+
+# URL prefix for static files.
+# Example: "http://example.com/static/", "http://static.example.com/"
+STATIC_URL = '/static/'
+
+# Additional locations of static files
+STATICFILES_DIRS = (
+    # Put strings here, like "/home/html/static" or "C:/www/django/static".
+    # Always use forward slashes, even on Windows.
+    # Don't forget to use absolute paths, not relative paths.
+    #BASE_DIR + '/static/',
+)
+
+# List of finder classes that know how to find static files in
+# various locations.
+STATICFILES_FINDERS = (
+    'django.contrib.staticfiles.finders.FileSystemFinder',
+    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
+    'django_assets.finders.AssetsFinder',
+)
+
+# Make this unique, and don't share it with anybody.
+SECRET_KEY = '{{ mailman_hk_cookie_key }}'
+
+# List of callables that know how to import templates from various sources.
+TEMPLATE_LOADERS = (
+    'django.template.loaders.filesystem.Loader',
+    'django.template.loaders.app_directories.Loader',
+#     'django.template.loaders.eggs.Loader',
+)
+
+
+TEMPLATE_CONTEXT_PROCESSORS = (
+    "django.contrib.auth.context_processors.auth",
+    "social_auth.context_processors.social_auth_login_redirect",
+    "django.contrib.messages.context_processors.messages",
+    "django.core.context_processors.debug",
+    "django.core.context_processors.i18n",
+    "django.core.context_processors.media",
+    "django.core.context_processors.static",
+    "django.core.context_processors.csrf",
+    "django.core.context_processors.request",
+    "django.core.context_processors.tz",
+    "django.contrib.messages.context_processors.messages",
+    "hyperkitty.context_processors.export_settings",
+    "postorius.context_processors.postorius",
+)
+
+
+MIDDLEWARE_CLASSES = (
+    'django.middleware.common.CommonMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    # Uncomment the next line for simple clickjacking protection:
+    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'hyperkitty.lib.store.KittyStoreDjangoMiddleware',
+    'hyperkitty.middleware.SSLRedirect',
+    'hyperkitty.middleware.TimezoneMiddleware',
+)
+
+ROOT_URLCONF = 'urls'
+
+TEMPLATE_DIRS = (
+    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
+    # Always use forward slashes, even on Windows.
+    # Don't forget to use absolute paths, not relative paths.
+#    BASE_DIR + '/templates',
+)
+
+AUTHENTICATION_BACKENDS = (
+    'social_auth.backends.browserid.BrowserIDBackend',
+    'hyperkitty.lib.fedora.FedoraBackend',
+    'social_auth.backends.google.GoogleBackend',
+    'social_auth.backends.yahoo.YahooBackend',
+    #'social_auth.backends.OpenIDBackend',
+    'django.contrib.auth.backends.ModelBackend',
+)
+
+INSTALLED_APPS = (
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.sites',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    # Uncomment the next line to enable the admin:
+    'django.contrib.admin',
+    # Uncomment the next line to enable admin documentation:
+    # 'django.contrib.admindocs',
+    'hyperkitty',
+    'social_auth',
+    'rest_framework',
+    'django_gravatar',
+    'south',
+    'crispy_forms',
+    'django_assets',
+    'paintstore',
+    'postorius',
+)
+
+REST_FRAMEWORK = {
+    'DEFAULT_RENDERER_CLASSES': (
+        'rest_framework.renderers.JSONRenderer',
+        'rest_framework.renderers.JSONPRenderer',
+        'rest_framework.renderers.XMLRenderer',
+        'rest_framework.renderers.BrowsableAPIRenderer',
+    )
+}
+
+LOGIN_URL          = '/accounts/login/'
+LOGIN_REDIRECT_URL = '/'
+LOGIN_ERROR_URL    = '/accounts/login/'
+SOCIAL_AUTH_COMPLETE_URL_NAME  = 'socialauth_complete'
+SOCIAL_AUTH_ASSOCIATE_URL_NAME = 'socialauth_associate_complete'
+SOCIAL_AUTH_DEFAULT_USERNAME = 'new_social_auth_user'
+SOCIAL_AUTH_UUID_LENGTH = 16
+SOCIAL_AUTH_LAST_LOGIN = 'social_auth_last_login_backend'
+GOOGLE_DISPLAY_NAME = 'HyperKitty'
+SOCIAL_AUTH_PIPELINE = (
+    'social_auth.backends.pipeline.social.social_auth_user',
+    'social_auth.backends.pipeline.associate.associate_by_email',
+    'social_auth.backends.pipeline.user.get_username',
+    'social_auth.backends.pipeline.user.create_user',
+    'social_auth.backends.pipeline.social.associate_user',
+    'social_auth.backends.pipeline.social.load_extra_data',
+    'social_auth.backends.pipeline.user.update_user_details'
+)
+SOCIAL_AUTH_USERNAME_IS_FULL_EMAIL = True
+
+AUTH_PROFILE_MODULE = 'hyperkitty.UserProfile'
+
+#
+# Gravatar
+# https://github.com/twaddington/django-gravatar
+#
+# Gravatar base url.
+#GRAVATAR_URL = 'http://www.gravatar.com/'
+# Gravatar base secure https url.
+#GRAVATAR_SECURE_URL = 'https://secure.gravatar.com/'
+# Gravatar size in pixels.
+#GRAVATAR_DEFAULT_SIZE = '80'
+# An image url or one of the following: 'mm', 'identicon', 'monsterid', 'wavatar', 'retro'.
+#GRAVATAR_DEFAULT_IMAGE = 'mm'
+# One of the following: 'g', 'pg', 'r', 'x'.
+#GRAVATAR_DEFAULT_RATING = 'g'
+# True to use https by default, False for plain http.
+#GRAVATAR_DEFAULT_SECURE = True
+
+#
+# django-assets
+# https://pypi.python.org/pypi/django-assets
+#
+ASSETS_DEBUG = DEBUG
+ASSETS_AUTO_BUILD = DEBUG
+
+
+# A sample logging configuration. The only tangible logging
+# performed by this configuration is to send an email to
+# the site admins on every HTTP 500 error when DEBUG=False.
+# See http://docs.djangoproject.com/en/dev/topics/logging for
+# more details on how to customize your logging configuration.
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'filters': {
+        'require_debug_false': {
+            '()': 'django.utils.log.RequireDebugFalse'
+        }
+    },
+    'handlers': {
+        'mail_admins': {
+            'level': 'ERROR',
+            'filters': ['require_debug_false'],
+            'class': 'django.utils.log.AdminEmailHandler'
+        },
+        'file':{
+            'level': 'INFO',
+            #'class': 'logging.handlers.RotatingFileHandler',
+            'class': 'logging.FileHandler',
+            'filename': '/var/log/hyperkitty/hyperkitty.log',
+            'formatter': 'verbose',
+        },
+    },
+    'loggers': {
+        #'django.request': {
+        #    'handlers': ['mail_admins'],
+        #    'level': 'ERROR',
+        #    'propagate': True,
+        #},
+        'django.request': {
+            'handlers': ['file'],
+            'level': 'ERROR',
+            'propagate': True,
+        },
+        'django': {
+            'handlers': ['file'],
+            'level': 'ERROR',
+            'propagate': True,
+        },
+        'hyperkitty': {
+            'handlers': ['file'],
+            'level': 'DEBUG',
+            'propagate': True,
+        },
+    },
+    'formatters': {
+        'verbose': {
+            'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
+        },
+        'simple': {
+            'format': '%(levelname)s %(message)s'
+        },
+    },
+}
+
+#
+# HyperKitty-specific
+#
+
+APP_NAME = 'Fedora list archives'
+
+# Allow authentication with the internal user database?
+# By default, only a login through Persona or your email provider is allowed.
+USE_INTERNAL_AUTH = False
+
+# URL to the KittyStore database
+#KITTYSTORE_URL = 'postgres://kittystore:kspass@localhost/kittystore'
+KITTYSTORE_URL = 'postgres://kittystoreapp:{{ mailman_ks_db_pass }}@{{ mailman_dbserver }}/kittystore'
+# Path to the KittyStore search index (writable directory)
+KITTYSTORE_SEARCH_INDEX = '{{ mailman_webui_basedir }}/kittystore_search_index'
+
+# Use SSL when logged in
+USE_SSL = True
+
+# WARNING: the KITTYSTORE_DEBUG variable below will output every SQL query.
+# That's a huge amount of text, don't enable it if you don't need to.
+KITTYSTORE_DEBUG = False
+
+# This is for development purposes
+USE_MOCKUPS = False
+
+
+try:
+    from settings_local import *
+except ImportError:
+    pass

roles/mailman/templates/settings_admin.py.j2

@@ -6,7 +6,10 @@ schema modifications)
 """
 
 from settings import *
-from settings_local import *
+try:
+    from settings_local import *
+except ImportError:
+    pass
 
 DATABASES = {
     'default': {
@@ -14,8 +17,8 @@ DATABASES = {
         'NAME': 'hyperkitty',
         'USER': 'hyperkittyadmin',
         'PASSWORD': '{{ mailman_hk_admin_db_pass }}',
-        'HOST': 'db02.stg.phx2.fedoraproject.org',
+        'HOST': '{{ mailman_dbserver }}',
         'PORT': '',
     }
 }
-KITTYSTORE_URL = 'postgres://kittystoreadmin:{{ mailman_ks_admin_db_pass }}@db02.stg.phx2.fedoraproject.org/kittystore'
+KITTYSTORE_URL = 'postgres://kittystoreadmin:{{ mailman_ks_admin_db_pass }}@{{ mailman_dbserver }}/kittystore'

roles/mailman/vars/main.yml

@@ -0,0 +1,4 @@
+---
+- mailman_webui_basedir: /srv/webui
+- mailman_webui_confdir: ${mailman_webui_basedir}/config
+- mailman_dbserver: db02.stg.phx2.fedoraproject.org