Update to the latest changes in HyperKitty

This commit is contained in:
Aurélien Bompard 2015-03-13 14:21:42 +00:00
parent 7914a6192f
commit 1e7869952d
11 changed files with 120 additions and 123 deletions

View file

@ -1,21 +1,28 @@
#!/usr/bin/env python
#!/usr/bin/env python2
from __future__ import unicode_literals, absolute_import, print_function
import os
import sys
import subprocess
import pickle
from optparse import OptionParser
from locale import getpreferredencoding
import yaml
MAILMAN_BIN = subprocess.check_output(["which", "mailman"]).decode("ascii").strip()
from mailman.commands.cli_import import Bouncer
sys.modules["Mailman.Bouncer"] = Bouncer
def call(command):
print "PYTHONPATH=%s" % os.environ["PYTHONPATH"], " ".join(command)
print(" ".join(command))
subprocess.check_call(command, env=os.environ)
def cmdget(command):
print "PYTHONPATH=%s" % os.environ["PYTHONPATH"], " ".join(command)
print(" ".join(command))
out = subprocess.check_output(command, env=os.environ)
return out.strip()
return out.decode(getpreferredencoding()).strip()
class Importer(object):
@ -26,10 +33,11 @@ class Importer(object):
self.index_path = self._get_index_path()
self.existing_lists = [ l.strip() for l in
cmdget(["sudo", "-u", "mailman",
"mailman3", "lists", "-q"]).split("\n") ]
MAILMAN_BIN, "lists", "-q"]).split("\n") ]
self.excluded = opts.exclude.strip().split(",")
def _get_index_path(self):
return None
sys.path.append(self.config["confdir"])
settings = __import__("settings")
sys.path.pop()
@ -43,40 +51,41 @@ class Importer(object):
for index, listname in enumerate(all_listnames):
listaddr = "%s@%s" % (listname, self.config["domain"])
if listname in self.excluded or listaddr in self.excluded:
print "Skipping excluded list %s" % listaddr
print("Skipping excluded list %s" % listaddr)
continue
print listaddr, "(%d/%d)" % (index+1, len(all_listnames))
print(listaddr, "(%d/%d)" % (index+1, len(all_listnames)))
confpickle = os.path.join(mm2libdir, 'lists', listname,
'config.pck')
if not os.path.exists(confpickle):
print "Missing configuration pickle:", confpickle
print("Missing configuration pickle:", confpickle)
continue
list_is_new = bool(listaddr not in self.existing_lists)
if list_is_new:
call(["sudo", "-u", "mailman", "mailman3", "create", "-d",
call(["sudo", "-u", "mailman", MAILMAN_BIN, "create", "-d",
listaddr])
call(["sudo", "-u", "mailman", "mailman3", "import21",
call(["sudo", "-u", "mailman", MAILMAN_BIN, "import21",
listaddr, confpickle])
if not self.opts.no_archives:
archivefile = os.path.join(
mm2libdir, "archives", "private",
"%s.mbox" % listname, "%s.mbox" % listname)
archive_policy = bool(pickle.load(
open(confpickle)).get('archive'))
archive_policy = bool(pickle.load(open(confpickle, "rb"),
encoding="utf-8", errors="ignore").get('archive'))
if not archive_policy:
print "List %s wants no archiving" % listname
print("List %s wants no archiving" % listname)
continue
if os.path.exists(archivefile) and \
(list_is_new or not self.opts.new_only):
call(["sudo", "kittystore-import", "-p",
self.config["confdir"], "-s", "settings_admin",
"-l", listaddr, "--continue", "--no-sync-mailman",
archivefile])
call(["sudo", "django-admin", "hyperkitty_import",
"--pythonpath", self.config["confdir"],
"--settings", "settings", "-l", listaddr,
"--no-sync-mailman", archivefile])
if self.index_path:
call(["sudo", "chown", "mailman:apache", "-R", self.index_path])
call(["sudo", "chmod", "g+w", self.index_path])
call(["sudo", "kittystore-sync-mailman", "-p",
self.config["confdir"], "-s", "settings_admin"])
call(["sudo", "django-admin", "mailman_sync",
"--pythonpath", self.config["confdir"],
"--settings", "settings"])
@ -101,10 +110,6 @@ def main():
with open(opts.config) as conffile:
config = yaml.safe_load(conffile)
sys.path.append(config["mm21codedir"])
# set the env var to propagate to subprocesses
os.environ["PYTHONPATH"] = config["mm21codedir"]
importer = Importer(opts, config)
importer.import_dir(mm2libdir)

View file

@ -8,5 +8,5 @@
#
[general]
base_url: http://{{ mailman_url }}
django_settings: {{ mailman_webui_confdir }}/settings.py
base_url: http://localhost/archives
api_key: SecretArchiverAPIKey

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python2
# vim: et ts=4 sw=4 fileencoding=utf-8
"""
@ -57,22 +57,22 @@ def main():
site.addsitedir(conf["confdir"])
import settings_admin
# KittyStore
dbspec = re.match("""
postgresql://
(?P<user>[a-z]+)
:
(?P<password>[^@]+)
@
(?P<host>[^/]+)
/
(?P<database>[^/?]+)
""", settings_admin.KITTYSTORE_URL, re.X)
give_rights(dbspec.group("host"),
dbspec.group("user"),
dbspec.group("password"),
dbspec.group("database")
)
## KittyStore
#dbspec = re.match("""
# postgresql://
# (?P<user>[a-z]+)
# :
# (?P<password>[^@]+)
# @
# (?P<host>[^/]+)
# /
# (?P<database>[^/?]+)
# """, settings_admin.KITTYSTORE_URL, re.X)
#give_rights(dbspec.group("host"),
# dbspec.group("user"),
# dbspec.group("password"),
# dbspec.group("database")
# )
# HyperKitty
give_rights(

View file

@ -8,13 +8,12 @@ export PATH=$PATH:$(dirname $(realpath $0)) # make yamlget available
BASEDIR=`yamlget basedir $CONFFILE`
CONFDIR=`yamlget confdir $CONFFILE`
INDEXDIR=$BASEDIR/kittystore_search_index
INDEXDIR=$BASEDIR/fulltext_index
django-admin collectstatic --clear --noinput --verbosity 0 --pythonpath $CONFDIR --settings settings
django-admin compress --pythonpath $CONFDIR --settings settings
django-admin syncdb --pythonpath $CONFDIR --settings settings_admin --noinput --migrate
django-admin loaddata /etc/postorius/sites/default/initial-user.json --pythonpath $CONFDIR --settings settings_admin
kittystore-updatedb --pythonpath $CONFDIR --settings settings_admin
chown mailman:mailman -R $INDEXDIR
chmod g+w -R $INDEXDIR
@ -27,7 +26,6 @@ restorecon -r "$BASEDIR"
# Run unit tests
django-admin test --pythonpath $CONFDIR --settings settings_test hyperkitty
nose2 kittystore
# Reload Apache to flush the python cache
systemctl reload httpd

View file

@ -1,19 +0,0 @@
#-*- coding: utf-8 -*-
"""
Copy of the Django settings file, but with databases set for unit tests.
"""
from settings import *
try:
from settings_local import *
except ImportError:
pass
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': ':memory:',
}
}
KITTYSTORE_URL = 'sqlite:'

View file

@ -10,11 +10,6 @@ from django.conf import settings
from django.contrib import admin
admin.autodiscover()
# Import hyperkitty urls and set urlpatterns if you want to hook
# hyperkitty into an existing django site.
# Otherwise set ROOT_URLCONF in settings.py to
# `hyperkitty.urls`.
from django.core.urlresolvers import reverse_lazy
from django.views.generic import RedirectView
@ -23,6 +18,6 @@ urlpatterns = patterns('',
#url(r'^$', 'postorius.views.list_index'),
url(r'^admin/', include('postorius.urls')),
url(r'^archives/', include('hyperkitty.urls')),
url(r'', include('social_auth.urls')),
url(r'', include('django_browserid.urls')),
url(r'', include('social_auth.urls'), {"SSL": True}),
url(r'', include('django_browserid.urls'), {"SSL": True}),
)

View file

@ -13,7 +13,7 @@
command: semanage fcontext -a -t etc_t "{{ mailman_webui_confdir }}(/.*)?"
- name: set the SELinux policy for the fulltext index
command: semanage fcontext -a -t httpd_sys_content_t "{{ mailman_webui_basedir }}/kittystore_search_index(/.*)?"
command: semanage fcontext -a -t httpd_sys_rw_content_t "{{ mailman_webui_basedir }}/fulltext_index(/.*)?"
- name: set the SELinux policy for the static files directory
command: semanage fcontext -a -t httpd_sys_content_t "{{ mailman_webui_basedir }}/static(/.*)?"
@ -21,6 +21,9 @@
- name: set the SELinux policy for the log directory
command: semanage fcontext -a -t httpd_log_t "/var/log/hyperkitty(/.*)?"
- name: set the SELinux policy for the generated postfix databases
command: semanage fcontext -a -t etc_aliases_t "{{ mailman_webui_basedir }}/var/data/postfix_.*"
- name: allow Apache to remotely connect to PostgreSQL
seboolean: name=httpd_can_network_connect_db state=yes persistent=yes
@ -50,15 +53,18 @@
- mailman3
- mailman3-selinux
- python-psycopg2
- python-storm-postgresql
- kittystore
- hyperkitty
- hyperkitty-selinux
- postorius
- memcached
- python-pylibmc
- python-django-haystack-xapian
- yum-plugin-post-transaction-actions
- mailman3-fedmsg-plugin
# to run the test suite:
- python-beautifulsoup4
- python-mock
- python-whoosh
tags:
- packages
@ -73,6 +79,13 @@
notify:
- restart mailman3
- name: add postfix to the mailman group
user: name=postfix groups=mailman append=yes
tags:
- config
notify:
- restart postfix
# for access to the full-text index
- name: add apache to the mailman group
user: name=apache groups=mailman append=yes
@ -130,13 +143,6 @@
tags:
- config
- name: install the hyperkitty settings test file
copy: src=settings_test.py
dest="{{ mailman_webui_confdir }}/settings_test.py"
owner=root group=root mode=0640
tags:
- config
- name: install the hyperkitty urls file
copy: src=urls.py
dest="{{ mailman_webui_confdir }}/urls.py"
@ -175,8 +181,8 @@
notify:
- reload apache
- name: create the kittystore plain text index dir
file: path="{{ mailman_webui_basedir }}/kittystore_search_index"
- name: create the fulltext index dir
file: path="{{ mailman_webui_basedir }}/fulltext_index"
state=directory owner=mailman group=mailman mode=0755
- name: create the hyperkitty static files dir
@ -187,8 +193,8 @@
# Plug HyperKitty into Mailman
#
- name: copy the mailman-hyperkitty conffile
template: src=mailman-hyperkitty.cfg.j2
dest="{{ mailman_webui_confdir }}/mailman-hyperkitty.cfg"
copy: src=mailman-hyperkitty.cfg
dest="{{ mailman_webui_confdir }}/mailman-hyperkitty.cfg"
tags:
- config
notify:
@ -228,18 +234,12 @@
- pg-give-rights.py
- post-update.sh
- import-mm2.py
- clean-sessions.sh
- name: copy the initial user fixture
copy: src=postorius.initial-user.json
dest=/etc/postorius/sites/default/initial-user.json
owner=root group=apache mode=0640
- name: install the cronjobs
file: dest=/etc/cron.daily/hyperkitty-clean-sessions.sh
src="{{ mailman_webui_basedir }}/bin/clean-sessions.sh"
state=link
# Sync databases
- name: install the post-transaction trigger

View file

@ -0,0 +1,10 @@
# This goes in /etc/cron.d/.
# Replace "apache" by your webserver user ("www-data" on Debian systems) and
# set the path to the Django project directory
@hourly apache django-admin runjobs hourly --pythonpath {{ mailman_webui_confdir }} --settings settings
@daily apache django-admin runjobs daily --pythonpath {{ mailman_webui_confdir }} --settings settings
@weekly apache django-admin runjobs weekly --pythonpath {{ mailman_webui_confdir }} --settings settings
@monthly apache django-admin runjobs monthly --pythonpath {{ mailman_webui_confdir }} --settings settings
@yearly apache django-admin runjobs yearly --pythonpath {{ mailman_webui_confdir }} --settings settings
2,17,32,47 * * * * apache django-admin runjobs quarter_hourly --pythonpath {{ mailman_webui_confdir }} --settings settings

View file

@ -9,7 +9,13 @@
site_owner: root@localhost
# Set the paths to be Fedora-compliant
layout: fhs
#layout: fhs
# Until the mailman3 rpm is back:
layout: dev
[paths.dev]
var_dir = /srv/webui/var
[paths.fhs]
bin_dir: /usr/libexec/mailman3
var_dir: /var/lib/mailman3
@ -19,18 +25,18 @@ lock_dir: /run/lock/mailman3
ext_dir: /etc/mailman3.d
pid_file: /run/mailman3/master.pid
[database]
class: mailman.database.postgresql.PostgreSQLDatabase
url: postgresql://mailmanadmin:{{ mailman_mailman_db_pass }}@{{ mailman_db_server }}/mailman
#[database]
#class: mailman.database.postgresql.PostgreSQLDatabase
#url: postgresql://mailmanadmin:{{ mailman_mailman_db_pass }}@{{ mailman_db_server }}/mailman
[archiver.hyperkitty]
class: hyperkitty.archiver.Archiver
class: mailman_hyperkitty.Archiver
enable: yes
configuration: /etc/mailman3.d/hyperkitty.cfg
[archiver.fedmsg]
class: mailman3_fedmsg_plugin.Archiver
enable: yes
#[archiver.fedmsg]
#class: mailman3_fedmsg_plugin.Archiver
#enable: yes
[archiver.prototype]
enable: yes

View file

@ -20,6 +20,8 @@ MANAGERS = ADMINS
MAILMAN_REST_SERVER = MAILMAN_API_URL = 'http://localhost:8001'
MAILMAN_API_USER = MAILMAN_USER = 'restadmin'
MAILMAN_API_PASS = MAILMAN_PASS = 'restpass'
MAILMAN_ARCHIVER_KEY = 'SecretArchiverAPIKey'
MAILMAN_ARCHIVER_FROM = ("127.0.0.1", "::1")
# CSS theme for postorius
MAILMAN_THEME = "default"
@ -37,7 +39,12 @@ DATABASES = {
# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = [".fedoraproject.org", "discuss.arquillian.org", "127.0.0.1"] # 127.0.0.1: HAProxy ping
ALLOWED_HOSTS = [
".fedoraproject.org",
"discuss.arquillian.org",
"localhost", # Archiving API from Mailman
"127.0.0.1", # HAProxy ping
]
# And for BrowserID too, see
# http://django-browserid.rtfd.org/page/user/settings.html#django.conf.settings.BROWSERID_AUDIENCES
BROWSERID_AUDIENCES = [
@ -151,10 +158,8 @@ MIDDLEWARE_CLASSES = (
'django.contrib.messages.middleware.MessageMiddleware',
# Uncomment the next line for simple clickjacking protection:
# 'django.middleware.clickjacking.XFrameOptionsMiddleware',
'hyperkitty.middleware.KittyStoreDjangoMiddleware',
'hyperkitty.middleware.SSLRedirect',
'hyperkitty.middleware.TimezoneMiddleware',
'hyperkitty.middleware.MailmanUserMetadata',
)
ROOT_URLCONF = 'urls'
@ -196,20 +201,18 @@ INSTALLED_APPS = (
'django_gravatar',
'south',
'crispy_forms',
'compressor',
'paintstore',
'compressor',
'django_browserid',
'haystack',
'django_extensions',
'postorius',
)
import django
if django.VERSION[:2] < (1, 6):
TEST_RUNNER = 'discover_runner.DiscoverRunner'
else:
# Django 1.6 defaults to a JSON serializer, but it won't work with django-openid, see
# https://bugs.launchpad.net/django-openid-auth/+bug/1252826
SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
# Django 1.6 defaults to a JSON serializer, but it won't work with django-openid, see
# https://bugs.launchpad.net/django-openid-auth/+bug/1252826
SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
REST_FRAMEWORK = {
@ -273,6 +276,17 @@ COMPRESS_OFFLINE = True
#INTERNAL_IPS = ('127.0.0.1',)
#
# Full-text search engine
#
HAYSTACK_CONNECTIONS = {
'default': {
'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
'PATH': "{{ mailman_webui_basedir }}/fulltext_index",
},
}
# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
@ -353,23 +367,12 @@ APP_NAME = 'Fedora list archives'
# By default, only a login through Persona or your email provider is allowed.
USE_INTERNAL_AUTH = False
# URL to the KittyStore database
KITTYSTORE_URL = 'postgresql://kittystoreapp:{{ mailman_kittystore_db_pass }}@{{ mailman_db_server }}/kittystore'
# Path to the KittyStore search index (writable directory)
KITTYSTORE_SEARCH_INDEX = '{{ mailman_webui_basedir }}/kittystore_search_index'
# Use SSL when logged in
#USE_SSL = True
# In the Fedora infra, SSL is handled by the proxy and we can't detect it
USE_SSL = False
USE_SSL = True
# Only display mailing-lists from the same virtual host as the webserver
FILTER_VHOST = False
# WARNING: the KITTYSTORE_DEBUG variable below will output every SQL query.
# That's a huge amount of text, don't enable it if you don't need to.
KITTYSTORE_DEBUG = False
# This is for development purposes
USE_MOCKUPS = False

View file

@ -21,4 +21,3 @@ DATABASES = {
'PORT': '',
}
}
KITTYSTORE_URL = 'postgresql://kittystoreadmin:{{ mailman_kittystore_admin_db_pass }}@{{ mailman_db_server }}/kittystore'