Update to the latest changes in HyperKitty
This commit is contained in:
parent
7914a6192f
commit
1e7869952d
11 changed files with 120 additions and 123 deletions
|
@ -1,21 +1,28 @@
|
|||
#!/usr/bin/env python
|
||||
#!/usr/bin/env python2
|
||||
|
||||
from __future__ import unicode_literals, absolute_import, print_function
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
import pickle
|
||||
from optparse import OptionParser
|
||||
from locale import getpreferredencoding
|
||||
import yaml
|
||||
|
||||
MAILMAN_BIN = subprocess.check_output(["which", "mailman"]).decode("ascii").strip()
|
||||
|
||||
from mailman.commands.cli_import import Bouncer
|
||||
sys.modules["Mailman.Bouncer"] = Bouncer
|
||||
|
||||
def call(command):
|
||||
print "PYTHONPATH=%s" % os.environ["PYTHONPATH"], " ".join(command)
|
||||
print(" ".join(command))
|
||||
subprocess.check_call(command, env=os.environ)
|
||||
|
||||
def cmdget(command):
|
||||
print "PYTHONPATH=%s" % os.environ["PYTHONPATH"], " ".join(command)
|
||||
print(" ".join(command))
|
||||
out = subprocess.check_output(command, env=os.environ)
|
||||
return out.strip()
|
||||
return out.decode(getpreferredencoding()).strip()
|
||||
|
||||
|
||||
class Importer(object):
|
||||
|
@ -26,10 +33,11 @@ class Importer(object):
|
|||
self.index_path = self._get_index_path()
|
||||
self.existing_lists = [ l.strip() for l in
|
||||
cmdget(["sudo", "-u", "mailman",
|
||||
"mailman3", "lists", "-q"]).split("\n") ]
|
||||
MAILMAN_BIN, "lists", "-q"]).split("\n") ]
|
||||
self.excluded = opts.exclude.strip().split(",")
|
||||
|
||||
def _get_index_path(self):
|
||||
return None
|
||||
sys.path.append(self.config["confdir"])
|
||||
settings = __import__("settings")
|
||||
sys.path.pop()
|
||||
|
@ -43,40 +51,41 @@ class Importer(object):
|
|||
for index, listname in enumerate(all_listnames):
|
||||
listaddr = "%s@%s" % (listname, self.config["domain"])
|
||||
if listname in self.excluded or listaddr in self.excluded:
|
||||
print "Skipping excluded list %s" % listaddr
|
||||
print("Skipping excluded list %s" % listaddr)
|
||||
continue
|
||||
print listaddr, "(%d/%d)" % (index+1, len(all_listnames))
|
||||
print(listaddr, "(%d/%d)" % (index+1, len(all_listnames)))
|
||||
confpickle = os.path.join(mm2libdir, 'lists', listname,
|
||||
'config.pck')
|
||||
if not os.path.exists(confpickle):
|
||||
print "Missing configuration pickle:", confpickle
|
||||
print("Missing configuration pickle:", confpickle)
|
||||
continue
|
||||
list_is_new = bool(listaddr not in self.existing_lists)
|
||||
if list_is_new:
|
||||
call(["sudo", "-u", "mailman", "mailman3", "create", "-d",
|
||||
call(["sudo", "-u", "mailman", MAILMAN_BIN, "create", "-d",
|
||||
listaddr])
|
||||
call(["sudo", "-u", "mailman", "mailman3", "import21",
|
||||
call(["sudo", "-u", "mailman", MAILMAN_BIN, "import21",
|
||||
listaddr, confpickle])
|
||||
if not self.opts.no_archives:
|
||||
archivefile = os.path.join(
|
||||
mm2libdir, "archives", "private",
|
||||
"%s.mbox" % listname, "%s.mbox" % listname)
|
||||
archive_policy = bool(pickle.load(
|
||||
open(confpickle)).get('archive'))
|
||||
archive_policy = bool(pickle.load(open(confpickle, "rb"),
|
||||
encoding="utf-8", errors="ignore").get('archive'))
|
||||
if not archive_policy:
|
||||
print "List %s wants no archiving" % listname
|
||||
print("List %s wants no archiving" % listname)
|
||||
continue
|
||||
if os.path.exists(archivefile) and \
|
||||
(list_is_new or not self.opts.new_only):
|
||||
call(["sudo", "kittystore-import", "-p",
|
||||
self.config["confdir"], "-s", "settings_admin",
|
||||
"-l", listaddr, "--continue", "--no-sync-mailman",
|
||||
archivefile])
|
||||
call(["sudo", "django-admin", "hyperkitty_import",
|
||||
"--pythonpath", self.config["confdir"],
|
||||
"--settings", "settings", "-l", listaddr,
|
||||
"--no-sync-mailman", archivefile])
|
||||
if self.index_path:
|
||||
call(["sudo", "chown", "mailman:apache", "-R", self.index_path])
|
||||
call(["sudo", "chmod", "g+w", self.index_path])
|
||||
call(["sudo", "kittystore-sync-mailman", "-p",
|
||||
self.config["confdir"], "-s", "settings_admin"])
|
||||
call(["sudo", "django-admin", "mailman_sync",
|
||||
"--pythonpath", self.config["confdir"],
|
||||
"--settings", "settings"])
|
||||
|
||||
|
||||
|
||||
|
@ -101,10 +110,6 @@ def main():
|
|||
with open(opts.config) as conffile:
|
||||
config = yaml.safe_load(conffile)
|
||||
|
||||
sys.path.append(config["mm21codedir"])
|
||||
# set the env var to propagate to subprocesses
|
||||
os.environ["PYTHONPATH"] = config["mm21codedir"]
|
||||
|
||||
importer = Importer(opts, config)
|
||||
importer.import_dir(mm2libdir)
|
||||
|
||||
|
|
|
@ -8,5 +8,5 @@
|
|||
#
|
||||
|
||||
[general]
|
||||
base_url: http://{{ mailman_url }}
|
||||
django_settings: {{ mailman_webui_confdir }}/settings.py
|
||||
base_url: http://localhost/archives
|
||||
api_key: SecretArchiverAPIKey
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python
|
||||
#!/usr/bin/env python2
|
||||
# vim: et ts=4 sw=4 fileencoding=utf-8
|
||||
|
||||
"""
|
||||
|
@ -57,22 +57,22 @@ def main():
|
|||
site.addsitedir(conf["confdir"])
|
||||
import settings_admin
|
||||
|
||||
# KittyStore
|
||||
dbspec = re.match("""
|
||||
postgresql://
|
||||
(?P<user>[a-z]+)
|
||||
:
|
||||
(?P<password>[^@]+)
|
||||
@
|
||||
(?P<host>[^/]+)
|
||||
/
|
||||
(?P<database>[^/?]+)
|
||||
""", settings_admin.KITTYSTORE_URL, re.X)
|
||||
give_rights(dbspec.group("host"),
|
||||
dbspec.group("user"),
|
||||
dbspec.group("password"),
|
||||
dbspec.group("database")
|
||||
)
|
||||
## KittyStore
|
||||
#dbspec = re.match("""
|
||||
# postgresql://
|
||||
# (?P<user>[a-z]+)
|
||||
# :
|
||||
# (?P<password>[^@]+)
|
||||
# @
|
||||
# (?P<host>[^/]+)
|
||||
# /
|
||||
# (?P<database>[^/?]+)
|
||||
# """, settings_admin.KITTYSTORE_URL, re.X)
|
||||
#give_rights(dbspec.group("host"),
|
||||
# dbspec.group("user"),
|
||||
# dbspec.group("password"),
|
||||
# dbspec.group("database")
|
||||
# )
|
||||
|
||||
# HyperKitty
|
||||
give_rights(
|
||||
|
|
|
@ -8,13 +8,12 @@ export PATH=$PATH:$(dirname $(realpath $0)) # make yamlget available
|
|||
|
||||
BASEDIR=`yamlget basedir $CONFFILE`
|
||||
CONFDIR=`yamlget confdir $CONFFILE`
|
||||
INDEXDIR=$BASEDIR/kittystore_search_index
|
||||
INDEXDIR=$BASEDIR/fulltext_index
|
||||
|
||||
django-admin collectstatic --clear --noinput --verbosity 0 --pythonpath $CONFDIR --settings settings
|
||||
django-admin compress --pythonpath $CONFDIR --settings settings
|
||||
django-admin syncdb --pythonpath $CONFDIR --settings settings_admin --noinput --migrate
|
||||
django-admin loaddata /etc/postorius/sites/default/initial-user.json --pythonpath $CONFDIR --settings settings_admin
|
||||
kittystore-updatedb --pythonpath $CONFDIR --settings settings_admin
|
||||
chown mailman:mailman -R $INDEXDIR
|
||||
chmod g+w -R $INDEXDIR
|
||||
|
||||
|
@ -27,7 +26,6 @@ restorecon -r "$BASEDIR"
|
|||
|
||||
# Run unit tests
|
||||
django-admin test --pythonpath $CONFDIR --settings settings_test hyperkitty
|
||||
nose2 kittystore
|
||||
|
||||
# Reload Apache to flush the python cache
|
||||
systemctl reload httpd
|
||||
|
|
|
@ -1,19 +0,0 @@
|
|||
#-*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
Copy of the Django settings file, but with databases set for unit tests.
|
||||
"""
|
||||
|
||||
from settings import *
|
||||
try:
|
||||
from settings_local import *
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
DATABASES = {
|
||||
'default': {
|
||||
'ENGINE': 'django.db.backends.sqlite3',
|
||||
'NAME': ':memory:',
|
||||
}
|
||||
}
|
||||
KITTYSTORE_URL = 'sqlite:'
|
|
@ -10,11 +10,6 @@ from django.conf import settings
|
|||
from django.contrib import admin
|
||||
admin.autodiscover()
|
||||
|
||||
# Import hyperkitty urls and set urlpatterns if you want to hook
|
||||
# hyperkitty into an existing django site.
|
||||
# Otherwise set ROOT_URLCONF in settings.py to
|
||||
# `hyperkitty.urls`.
|
||||
|
||||
from django.core.urlresolvers import reverse_lazy
|
||||
from django.views.generic import RedirectView
|
||||
|
||||
|
@ -23,6 +18,6 @@ urlpatterns = patterns('',
|
|||
#url(r'^$', 'postorius.views.list_index'),
|
||||
url(r'^admin/', include('postorius.urls')),
|
||||
url(r'^archives/', include('hyperkitty.urls')),
|
||||
url(r'', include('social_auth.urls')),
|
||||
url(r'', include('django_browserid.urls')),
|
||||
url(r'', include('social_auth.urls'), {"SSL": True}),
|
||||
url(r'', include('django_browserid.urls'), {"SSL": True}),
|
||||
)
|
||||
|
|
|
@ -13,7 +13,7 @@
|
|||
command: semanage fcontext -a -t etc_t "{{ mailman_webui_confdir }}(/.*)?"
|
||||
|
||||
- name: set the SELinux policy for the fulltext index
|
||||
command: semanage fcontext -a -t httpd_sys_content_t "{{ mailman_webui_basedir }}/kittystore_search_index(/.*)?"
|
||||
command: semanage fcontext -a -t httpd_sys_rw_content_t "{{ mailman_webui_basedir }}/fulltext_index(/.*)?"
|
||||
|
||||
- name: set the SELinux policy for the static files directory
|
||||
command: semanage fcontext -a -t httpd_sys_content_t "{{ mailman_webui_basedir }}/static(/.*)?"
|
||||
|
@ -21,6 +21,9 @@
|
|||
- name: set the SELinux policy for the log directory
|
||||
command: semanage fcontext -a -t httpd_log_t "/var/log/hyperkitty(/.*)?"
|
||||
|
||||
- name: set the SELinux policy for the generated postfix databases
|
||||
command: semanage fcontext -a -t etc_aliases_t "{{ mailman_webui_basedir }}/var/data/postfix_.*"
|
||||
|
||||
- name: allow Apache to remotely connect to PostgreSQL
|
||||
seboolean: name=httpd_can_network_connect_db state=yes persistent=yes
|
||||
|
||||
|
@ -50,15 +53,18 @@
|
|||
- mailman3
|
||||
- mailman3-selinux
|
||||
- python-psycopg2
|
||||
- python-storm-postgresql
|
||||
- kittystore
|
||||
- hyperkitty
|
||||
- hyperkitty-selinux
|
||||
- postorius
|
||||
- memcached
|
||||
- python-pylibmc
|
||||
- python-django-haystack-xapian
|
||||
- yum-plugin-post-transaction-actions
|
||||
- mailman3-fedmsg-plugin
|
||||
# to run the test suite:
|
||||
- python-beautifulsoup4
|
||||
- python-mock
|
||||
- python-whoosh
|
||||
tags:
|
||||
- packages
|
||||
|
||||
|
@ -73,6 +79,13 @@
|
|||
notify:
|
||||
- restart mailman3
|
||||
|
||||
- name: add postfix to the mailman group
|
||||
user: name=postfix groups=mailman append=yes
|
||||
tags:
|
||||
- config
|
||||
notify:
|
||||
- restart postfix
|
||||
|
||||
# for access to the full-text index
|
||||
- name: add apache to the mailman group
|
||||
user: name=apache groups=mailman append=yes
|
||||
|
@ -130,13 +143,6 @@
|
|||
tags:
|
||||
- config
|
||||
|
||||
- name: install the hyperkitty settings test file
|
||||
copy: src=settings_test.py
|
||||
dest="{{ mailman_webui_confdir }}/settings_test.py"
|
||||
owner=root group=root mode=0640
|
||||
tags:
|
||||
- config
|
||||
|
||||
- name: install the hyperkitty urls file
|
||||
copy: src=urls.py
|
||||
dest="{{ mailman_webui_confdir }}/urls.py"
|
||||
|
@ -175,8 +181,8 @@
|
|||
notify:
|
||||
- reload apache
|
||||
|
||||
- name: create the kittystore plain text index dir
|
||||
file: path="{{ mailman_webui_basedir }}/kittystore_search_index"
|
||||
- name: create the fulltext index dir
|
||||
file: path="{{ mailman_webui_basedir }}/fulltext_index"
|
||||
state=directory owner=mailman group=mailman mode=0755
|
||||
|
||||
- name: create the hyperkitty static files dir
|
||||
|
@ -187,8 +193,8 @@
|
|||
# Plug HyperKitty into Mailman
|
||||
#
|
||||
- name: copy the mailman-hyperkitty conffile
|
||||
template: src=mailman-hyperkitty.cfg.j2
|
||||
dest="{{ mailman_webui_confdir }}/mailman-hyperkitty.cfg"
|
||||
copy: src=mailman-hyperkitty.cfg
|
||||
dest="{{ mailman_webui_confdir }}/mailman-hyperkitty.cfg"
|
||||
tags:
|
||||
- config
|
||||
notify:
|
||||
|
@ -228,18 +234,12 @@
|
|||
- pg-give-rights.py
|
||||
- post-update.sh
|
||||
- import-mm2.py
|
||||
- clean-sessions.sh
|
||||
|
||||
- name: copy the initial user fixture
|
||||
copy: src=postorius.initial-user.json
|
||||
dest=/etc/postorius/sites/default/initial-user.json
|
||||
owner=root group=apache mode=0640
|
||||
|
||||
- name: install the cronjobs
|
||||
file: dest=/etc/cron.daily/hyperkitty-clean-sessions.sh
|
||||
src="{{ mailman_webui_basedir }}/bin/clean-sessions.sh"
|
||||
state=link
|
||||
|
||||
|
||||
# Sync databases
|
||||
- name: install the post-transaction trigger
|
||||
|
|
10
roles/mailman/templates/crontab.j2
Normal file
10
roles/mailman/templates/crontab.j2
Normal file
|
@ -0,0 +1,10 @@
|
|||
# This goes in /etc/cron.d/.
|
||||
# Replace "apache" by your webserver user ("www-data" on Debian systems) and
|
||||
# set the path to the Django project directory
|
||||
|
||||
@hourly apache django-admin runjobs hourly --pythonpath {{ mailman_webui_confdir }} --settings settings
|
||||
@daily apache django-admin runjobs daily --pythonpath {{ mailman_webui_confdir }} --settings settings
|
||||
@weekly apache django-admin runjobs weekly --pythonpath {{ mailman_webui_confdir }} --settings settings
|
||||
@monthly apache django-admin runjobs monthly --pythonpath {{ mailman_webui_confdir }} --settings settings
|
||||
@yearly apache django-admin runjobs yearly --pythonpath {{ mailman_webui_confdir }} --settings settings
|
||||
2,17,32,47 * * * * apache django-admin runjobs quarter_hourly --pythonpath {{ mailman_webui_confdir }} --settings settings
|
|
@ -9,7 +9,13 @@
|
|||
site_owner: root@localhost
|
||||
|
||||
# Set the paths to be Fedora-compliant
|
||||
layout: fhs
|
||||
#layout: fhs
|
||||
# Until the mailman3 rpm is back:
|
||||
layout: dev
|
||||
|
||||
[paths.dev]
|
||||
var_dir = /srv/webui/var
|
||||
|
||||
[paths.fhs]
|
||||
bin_dir: /usr/libexec/mailman3
|
||||
var_dir: /var/lib/mailman3
|
||||
|
@ -19,18 +25,18 @@ lock_dir: /run/lock/mailman3
|
|||
ext_dir: /etc/mailman3.d
|
||||
pid_file: /run/mailman3/master.pid
|
||||
|
||||
[database]
|
||||
class: mailman.database.postgresql.PostgreSQLDatabase
|
||||
url: postgresql://mailmanadmin:{{ mailman_mailman_db_pass }}@{{ mailman_db_server }}/mailman
|
||||
#[database]
|
||||
#class: mailman.database.postgresql.PostgreSQLDatabase
|
||||
#url: postgresql://mailmanadmin:{{ mailman_mailman_db_pass }}@{{ mailman_db_server }}/mailman
|
||||
|
||||
[archiver.hyperkitty]
|
||||
class: hyperkitty.archiver.Archiver
|
||||
class: mailman_hyperkitty.Archiver
|
||||
enable: yes
|
||||
configuration: /etc/mailman3.d/hyperkitty.cfg
|
||||
|
||||
[archiver.fedmsg]
|
||||
class: mailman3_fedmsg_plugin.Archiver
|
||||
enable: yes
|
||||
#[archiver.fedmsg]
|
||||
#class: mailman3_fedmsg_plugin.Archiver
|
||||
#enable: yes
|
||||
|
||||
[archiver.prototype]
|
||||
enable: yes
|
||||
|
|
|
@ -20,6 +20,8 @@ MANAGERS = ADMINS
|
|||
MAILMAN_REST_SERVER = MAILMAN_API_URL = 'http://localhost:8001'
|
||||
MAILMAN_API_USER = MAILMAN_USER = 'restadmin'
|
||||
MAILMAN_API_PASS = MAILMAN_PASS = 'restpass'
|
||||
MAILMAN_ARCHIVER_KEY = 'SecretArchiverAPIKey'
|
||||
MAILMAN_ARCHIVER_FROM = ("127.0.0.1", "::1")
|
||||
|
||||
# CSS theme for postorius
|
||||
MAILMAN_THEME = "default"
|
||||
|
@ -37,7 +39,12 @@ DATABASES = {
|
|||
|
||||
# Hosts/domain names that are valid for this site; required if DEBUG is False
|
||||
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
|
||||
ALLOWED_HOSTS = [".fedoraproject.org", "discuss.arquillian.org", "127.0.0.1"] # 127.0.0.1: HAProxy ping
|
||||
ALLOWED_HOSTS = [
|
||||
".fedoraproject.org",
|
||||
"discuss.arquillian.org",
|
||||
"localhost", # Archiving API from Mailman
|
||||
"127.0.0.1", # HAProxy ping
|
||||
]
|
||||
# And for BrowserID too, see
|
||||
# http://django-browserid.rtfd.org/page/user/settings.html#django.conf.settings.BROWSERID_AUDIENCES
|
||||
BROWSERID_AUDIENCES = [
|
||||
|
@ -151,10 +158,8 @@ MIDDLEWARE_CLASSES = (
|
|||
'django.contrib.messages.middleware.MessageMiddleware',
|
||||
# Uncomment the next line for simple clickjacking protection:
|
||||
# 'django.middleware.clickjacking.XFrameOptionsMiddleware',
|
||||
'hyperkitty.middleware.KittyStoreDjangoMiddleware',
|
||||
'hyperkitty.middleware.SSLRedirect',
|
||||
'hyperkitty.middleware.TimezoneMiddleware',
|
||||
'hyperkitty.middleware.MailmanUserMetadata',
|
||||
)
|
||||
|
||||
ROOT_URLCONF = 'urls'
|
||||
|
@ -196,20 +201,18 @@ INSTALLED_APPS = (
|
|||
'django_gravatar',
|
||||
'south',
|
||||
'crispy_forms',
|
||||
'compressor',
|
||||
'paintstore',
|
||||
'compressor',
|
||||
'django_browserid',
|
||||
'haystack',
|
||||
'django_extensions',
|
||||
'postorius',
|
||||
)
|
||||
|
||||
|
||||
import django
|
||||
if django.VERSION[:2] < (1, 6):
|
||||
TEST_RUNNER = 'discover_runner.DiscoverRunner'
|
||||
else:
|
||||
# Django 1.6 defaults to a JSON serializer, but it won't work with django-openid, see
|
||||
# https://bugs.launchpad.net/django-openid-auth/+bug/1252826
|
||||
SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
|
||||
# Django 1.6 defaults to a JSON serializer, but it won't work with django-openid, see
|
||||
# https://bugs.launchpad.net/django-openid-auth/+bug/1252826
|
||||
SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
|
||||
|
||||
|
||||
REST_FRAMEWORK = {
|
||||
|
@ -273,6 +276,17 @@ COMPRESS_OFFLINE = True
|
|||
#INTERNAL_IPS = ('127.0.0.1',)
|
||||
|
||||
|
||||
#
|
||||
# Full-text search engine
|
||||
#
|
||||
HAYSTACK_CONNECTIONS = {
|
||||
'default': {
|
||||
'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
|
||||
'PATH': "{{ mailman_webui_basedir }}/fulltext_index",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# A sample logging configuration. The only tangible logging
|
||||
# performed by this configuration is to send an email to
|
||||
# the site admins on every HTTP 500 error when DEBUG=False.
|
||||
|
@ -353,23 +367,12 @@ APP_NAME = 'Fedora list archives'
|
|||
# By default, only a login through Persona or your email provider is allowed.
|
||||
USE_INTERNAL_AUTH = False
|
||||
|
||||
# URL to the KittyStore database
|
||||
KITTYSTORE_URL = 'postgresql://kittystoreapp:{{ mailman_kittystore_db_pass }}@{{ mailman_db_server }}/kittystore'
|
||||
# Path to the KittyStore search index (writable directory)
|
||||
KITTYSTORE_SEARCH_INDEX = '{{ mailman_webui_basedir }}/kittystore_search_index'
|
||||
|
||||
# Use SSL when logged in
|
||||
#USE_SSL = True
|
||||
# In the Fedora infra, SSL is handled by the proxy and we can't detect it
|
||||
USE_SSL = False
|
||||
USE_SSL = True
|
||||
|
||||
# Only display mailing-lists from the same virtual host as the webserver
|
||||
FILTER_VHOST = False
|
||||
|
||||
# WARNING: the KITTYSTORE_DEBUG variable below will output every SQL query.
|
||||
# That's a huge amount of text, don't enable it if you don't need to.
|
||||
KITTYSTORE_DEBUG = False
|
||||
|
||||
# This is for development purposes
|
||||
USE_MOCKUPS = False
|
||||
|
||||
|
|
|
@ -21,4 +21,3 @@ DATABASES = {
|
|||
'PORT': '',
|
||||
}
|
||||
}
|
||||
KITTYSTORE_URL = 'postgresql://kittystoreadmin:{{ mailman_kittystore_admin_db_pass }}@{{ mailman_db_server }}/kittystore'
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue