Infrastructure/ansible
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

[mailman] Remove the mailman role from ansible repository

Browse source 
The mailman is no longer running and the new one is deployed by new mailman3
role. Let's do some cleaning.

Signed-off-by: Michal Konecny <mkonecny@redhat.com>
This commit is contained in:
Michal Konecny 2024-07-03 14:46:53 +02:00 • committed by kevin
parent 35902dad8b
commit 315f303956
54 changed files with 0 additions and 2977 deletions
Download patch file Download diff file Expand all files Collapse all files

12
roles/mailman/defaults/main.yml
View file

@ -1,12 +0,0 @@
---
mailman_webui_basedir: /srv/webui
mailman_webui_confdir: "{{ mailman_webui_basedir }}/config"
mailman_db_server: localhost
mailman_mailman_db_pass: changeme
mailman_hyperkitty_admin_db_pass: changeme
mailman_hyperkitty_db_pass: changeme
mailman_hyperkitty_cookie_key: changeme
mailman_domains:
- lists.example.com
- lists.example.org
mailman_social_login: []

15
roles/mailman/files/django_fedora.py
View file

@ -1,15 +0,0 @@
from django.http import UnreadablePostError
from pylibmc import Error as MemcachedError
EXCLUDED = (
UnreadablePostError,
MemcachedError,
)
def exclude_useless_errors(record):
if record.exc_info:
exc_type, exc_value = record.exc_info[:2]
for excluded_class in EXCLUDED:
if isinstance(exc_value, excluded_class):
return False
return True

14
roles/mailman/files/django_fedora_nosignup.py
View file

@ -1,14 +0,0 @@
# Disable local signup
from allauth.account.adapter import DefaultAccountAdapter
from allauth.socialaccount.adapter import DefaultSocialAccountAdapter
class NoLocalSignUpAdapter(DefaultAccountAdapter):
def is_open_for_signup(self, request):
return False
class SignUpEnabledSocialAdapter(DefaultSocialAccountAdapter):
def is_open_for_signup(self, request, sociallogin):
return True

BIN
roles/mailman/files/favicon.ico
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 31 KiB

11
roles/mailman/files/fedmsg-plugin-conf.py
View file

@ -1,11 +0,0 @@
config = {
# These are mailing lists that we don't publish to fedmsg (because spam or privacy)
'mailman.excluded_lists': [
'scm-commits', # too much traffic
'council-private', # private list
'cwg-private', # private list
'fesco', # private list
'security-private', # private list
'diversity-private', # private list
],
}

9
roles/mailman/files/headers.html
View file

@ -1,9 +0,0 @@
<style>
.socialaccount_provider.fedora {
padding: 10px 16px;
font-size: 18px;
line-height: 1.33333;
border-radius: 6px;
border-color: #aaa;
}
</style>

56
roles/mailman/files/hyperkitty-delete-list.py
View file

@ -1,56 +0,0 @@
#!/usr/bin/env python2
"""
Delete lists from HyperKitty
Author: Aurelien Bompard <abompard@fedoraproject.org>
"""
from __future__ import (
absolute_import, print_function, unicode_literals, division)
import os
import logging
import readline
import sys
from argparse import ArgumentParser
sys.path.insert(0, "/srv/webui/config")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
from django import setup
from hyperkitty.models import MailingList
def delete_list(address):
try:
ml = MailingList.objects.get(name=address)
except MailingList.DoesNotExist:
print("Could not find a list with address \"{}\".".format(address))
return
print("The mailing-list \"{}\" ({}) will be entirely deleted.".format(ml.name, ml.list_id))
print("It contains {} threads and {} emails.".format(ml.threads.count(), ml.emails.count()))
prompt = "Are you sure? [y/N] ".format(ml.name, ml.list_id)
response = raw_input(prompt)
if response != "y":
print("Not deleted.")
return
ml.delete()
print("Mailing-list successfully deleted.")
def parse_args():
parser = ArgumentParser()
parser.add_argument("list_address", nargs="+", help="The mailing-list address")
return parser.parse_args()
def main():
args = parse_args()
setup()
logging.basicConfig(level=logging.INFO, format="%(message)s")
for address in args.list_address:
delete_list(address)
if __name__ == "__main__":
main()

31
roles/mailman/files/hyperkitty.Fedora.repo
View file

@ -1,31 +0,0 @@
[hyperkitty]
name=HyperKitty archiver and its dependencies
baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/stable/fedora-$releasever/$basearch/
enabled=1
skip_if_unavailable=1
gpgcheck=1
gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc
[hyperkitty-source]
name=HyperKitty archiver and its dependencies - Source
baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/stable/fedora-$releasever/SRPMS
enabled=0
skip_if_unavailable=1
gpgcheck=1
gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc
[hyperkitty-devel]
name=HyperKitty archiver and its dependencies (devel versions)
baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/devel/fedora-$releasever/$basearch/
enabled=0
skip_if_unavailable=1
gpgcheck=1
gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc
[hyperkitty-devel-source]
name=HyperKitty archiver and its dependencies (devel versions) - Source
baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/devel/fedora-$releasever/SRPMS
enabled=0
skip_if_unavailable=1
gpgcheck=1
gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc

31
roles/mailman/files/hyperkitty.RedHat.repo
View file

@ -1,31 +0,0 @@
[hyperkitty]
name=HyperKitty archiver and its dependencies
baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/stable/el-$releasever/$basearch/
enabled=1
skip_if_unavailable=1
gpgcheck=1
gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc
[hyperkitty-source]
name=HyperKitty archiver and its dependencies - Source
baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/stable/el-$releasever/SRPMS
enabled=0
skip_if_unavailable=1
gpgcheck=1
gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc
[hyperkitty-devel]
name=HyperKitty archiver and its dependencies (devel versions)
baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/devel/el-$releasever/$basearch/
enabled=0
skip_if_unavailable=1
gpgcheck=1
gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc
[hyperkitty-devel-source]
name=HyperKitty archiver and its dependencies (devel versions) - Source
baseurl=https://repos.fedorapeople.org/repos/abompard/hyperkitty/devel/el-$releasever/SRPMS
enabled=0
skip_if_unavailable=1
gpgcheck=1
gpgkey=https://repos.fedorapeople.org/repos/abompard/abompard.asc

6
roles/mailman/files/hyperkitty.logrotate.conf
View file

@ -1,6 +0,0 @@
/var/log/hyperkitty/*.log {
missingok
notifempty
delaycompress
su root apache
}

148
roles/mailman/files/import-mm2.py
View file

@ -1,148 +0,0 @@
#!/usr/bin/env python3
from __future__ import unicode_literals, absolute_import, print_function
import os
import sys
import subprocess
import pickle
from optparse import OptionParser
from locale import getpreferredencoding
import yaml
MAILMAN_BIN = subprocess.check_output(["which", "mailman3"]).decode("ascii").strip()
from mailman.commands.cli_import import Bouncer
sys.modules["Mailman.Bouncer"] = Bouncer
def call(command):
print(" ".join(command))
subprocess.check_call(command, env=os.environ)
def cmdget(command):
print(" ".join(command))
out = subprocess.check_output(command, env=os.environ)
return out.decode(getpreferredencoding()).strip()
class Importer(object):
def __init__(self, opts, config):
self.opts = opts
self.config = config
self.index_path = self._get_index_path()
self.existing_lists = [ l.strip() for l in
cmdget(["sudo", "-u", "mailman",
MAILMAN_BIN, "lists", "-q"]).split("\n") ]
if opts.exclude:
self.excluded = opts.exclude.strip().split(",")
else:
self.excluded = []
if opts.include:
self.included = opts.include.strip().split(",")
else:
self.included = []
def _get_index_path(self):
return None
sys.path.append(self.config["confdir"])
settings = __import__("settings")
sys.path.pop()
return settings.KITTYSTORE_SEARCH_INDEX
def import_dir(self, mm2libdir):
all_listnames = [ d for d in os.listdir(
os.path.join(mm2libdir, 'lists'))
if not d.startswith(".") ]
all_listnames.sort()
for index, listname in enumerate(all_listnames):
listaddr = "%s@%s" % (listname, self.opts.domain.strip())
if listname in self.excluded or listaddr in self.excluded:
print("Skipping excluded list %s" % listaddr)
continue
if self.included and (
listname not in self.included and
listaddr not in self.included):
print("Skipping not included list %s" % listaddr)
continue
print(listaddr, "(%d/%d)" % (index+1, len(all_listnames)))
confpickle = os.path.join(mm2libdir, 'lists', listname,
'config.pck')
if not os.path.exists(confpickle):
print("Missing configuration pickle:", confpickle)
continue
list_is_new = bool(listaddr not in self.existing_lists)
if self.opts.recreate and not list_is_new:
call(["sudo", "-u", "mailman", MAILMAN_BIN, "remove",
listaddr])
list_is_new = True
if list_is_new:
call(["sudo", "-u", "mailman", MAILMAN_BIN, "create", "-d",
listaddr])
call(["sudo", "-u", "mailman", MAILMAN_BIN, "import21",
listaddr, confpickle])
if not self.opts.no_archives:
archivefile = os.path.join(
mm2libdir, "archives", "private",
"%s.mbox" % listname, "%s.mbox" % listname)
archive_policy = bool(pickle.load(open(confpickle, "rb"),
encoding="utf-8", errors="ignore").get('archive'))
if not archive_policy:
print("List %s wants no archiving" % listname)
continue
if os.path.exists(archivefile) and \
(list_is_new or not self.opts.new_only):
call(["sudo", "django-admin", "hyperkitty_import",
"--pythonpath", self.config["confdir"],
"--settings", "settings", "-l", listaddr,
"--no-sync-mailman", archivefile])
if self.index_path:
call(["sudo", "chown", "mailman:apache", "-R", self.index_path])
call(["sudo", "chmod", "g+w", self.index_path])
if not self.opts.no_sync:
call(["sudo", "django-admin", "mailman_sync",
"--pythonpath", self.config["confdir"],
"--settings", "settings"])
def main():
parser = OptionParser()
parser.add_option("-n", "--new-only", action="store_true",
help="Only import the archives when the list is new")
parser.add_option("-A", "--no-archives", action="store_true",
help="Don't import the archives, only import the list config")
parser.add_option("-c", "--config", default="/etc/mailman-migration.conf",
help="Configuration file (default: %defaults)")
parser.add_option("-d", "--domain",
help="Domain for the mailing-lists")
parser.add_option("-x", "--exclude", default="",
help="Comma-separated list of lists to exclude")
parser.add_option("-i", "--include", default="",
help="Comma-separated list of lists to include, no other "
"list will be imported")
parser.add_option("-R", "--recreate", action="store_true",
help="Recreate the lists and re-import their configuration")
parser.add_option("-S", "--no-sync", action="store_true",
help="Don't run the mailman_sync admin command")
opts, args = parser.parse_args()
if len(args) != 1:
parser.error("Only one arg: the Mailman 2.1 lib dir to import")
if opts.include and opts.exclude:
parser.error("Only one of 'include' or 'exclude' may be used")
if not opts.domain:
parser.error("You must provide a domain name for the lists (--domain)")
mm2libdir = args[0]
if not os.path.exists(mm2libdir):
parser.error("No such directory: %s" % mm2libdir)
with open(opts.config) as conffile:
config = yaml.safe_load(conffile)
importer = Importer(opts, config)
importer.import_dir(mm2libdir)
if __name__ == "__main__":
main()

6
roles/mailman/files/login_extra_top.html
View file

@ -1,6 +0,0 @@
<p>
By subscribing or posting to these mailing lists, you agree that any email
to the list is public and viewable worldwide (unless it's one of the few private mailing-lists).
For more details, see our
<a href="https://fedoraproject.org/wiki/Legal:PrivacyPolicy">Privacy policy</a>.
</p>

BIN
roles/mailman/files/logo-hyperkitty-fedora.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 4.5 KiB

20
roles/mailman/files/mailman-hyperkitty.cfg
View file

@ -1,20 +0,0 @@
# This is the mailman extension configuration file to enable HyperKitty as an
# archiver. Remember to add the following lines in the mailman.cfg file:
#
# [archiver.hyperkitty]
# class: mailman_hyperkitty.Archiver
# enable: yes
# configuration: /path/to/here/hyperkitty.cfg
#
[general]
# This is your HyperKitty installation, preferably on the localhost. This
# address will be used by Mailman to forward incoming emails to HyperKitty
# for archiving. It does not need to be publicly available, in fact it's
# better if it is not.
base_url: http://localhost/archives/
# Shared API key, must be the identical to the value in HyperKitty's
# settings.
api_key: SecretArchiverAPIKey

118
roles/mailman/files/mailman-sar.py
View file

@ -1,118 +0,0 @@
#!/usr/bin/env python
"""
GDPR SAR script for HyperKitty.
Extract all emails from a selected address and prints them in JSON to the
standard output.
"""
from __future__ import absolute_import, unicode_literals, print_function
import argparse
import json
import logging
import os
import sys
import requests
from six.moves.urllib.parse import urljoin
ENV_EMAIL = "SAR_EMAIL"
HYPERKITTY_INSTANCE = "http://localhost/archives/"
MAILMAN_INSTANCE = "http://localhost:8001/"
MAILMAN_AUTH = ("restadmin", "restpass")
log = logging.getLogger()
def get_emails(address):
url = urljoin(HYPERKITTY_INSTANCE, "api/sender/{}/emails/".format(address))
result = {"next": url}
count = None
email_urls = []
while result.get("next"):
url = result["next"]
response = requests.get(url)
if response.status_code >= 300:
log.error("Could not get URL %s: %d %s",
url, response.status_code, response.reason)
break
result = response.json()
if count is None:
count = result["count"]
email_urls.extend([e["url"] for e in result["results"]])
if count != len(email_urls):
log.error("Mismatch in the number of emails: got %s but there are "
"%s in total.", len(email_urls), count)
raise ValueError
emails = []
for url in email_urls:
response = requests.get(url)
result = response.json()
emails.append(result)
return emails
def get_subscriptions(address):
url = urljoin(MAILMAN_INSTANCE,
"3.1/members/find?subscriber={}".format(address))
response = requests.get(url, auth=MAILMAN_AUTH)
if response.status_code >= 300:
log.error("Could not get URL %s: %d %s",
url, response.status_code, response.reason)
return []
result = response.json()
subscriptions = []
for entry in result.get("entries", []):
subscription = {
"list_id": entry["list_id"],
"role": entry["role"],
"delivery_mode": entry["delivery_mode"],
}
# Get the subscription's preferences
member_id = entry["member_id"]
pref_url = urljoin(MAILMAN_INSTANCE,
"3.1/members/{}/preferences".format(member_id))
pref_response = requests.get(pref_url, auth=MAILMAN_AUTH)
pref_result = pref_response.json()
if pref_response.status_code >= 300:
log.error("Could not get URL %s: %d %s",
pref_url, pref_response.status_code,
pref_response.reason)
else:
subscription["preferences"] = dict([
(key, pref_result[key]) for key in pref_result
if key not in ("http_etag", "self_link")
])
subscriptions.append(subscription)
return subscriptions
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("--debug", action="store_true")
return parser.parse_args()
def main():
args = parse_args()
try:
email = os.environ[ENV_EMAIL]
except KeyError as e:
print("Missing environment variable. {}".format(e), file=sys.stderr)
sys.exit(1)
logging.basicConfig(
level=logging.DEBUG if args.debug else logging.WARNING,
stream=sys.stderr,
)
emails = get_emails(email)
subscriptions = get_subscriptions(email)
print(json.dumps(dict(
emails=emails,
subscriptions=subscriptions,
), indent=2))
if __name__ == "__main__":
main()

15
roles/mailman/files/mailman-template-list-admin-action-post.txt
View file

@ -1,15 +0,0 @@
As list administrator, your authorization is requested for the
following mailing list posting:
List: $listname
From: $sender_email
Subject: $subject
The message is being held because:
$reasons
At your convenience, visit your dashboard to approve or deny the
request:
https://lists.fedoraproject.org/admin/lists/$list_id/held_messages

8
roles/mailman/files/mailman-template-list-member-generic-footer.txt
View file

@ -1,8 +0,0 @@
--
_______________________________________________
$display_name mailing list -- $listname
To unsubscribe send an email to ${short_listname}-leave@${domain}
Fedora Code of Conduct: https://docs.fedoraproject.org/en-US/project/code-of-conduct/
List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines
List Archives: https://${domain}/archives/list/${listname}
Do not reply to spam, report it: https://pagure.io/fedora-infrastructure/new_issue

25
roles/mailman/files/mailman-template-users-list-user-notice-hold.txt
View file

@ -1,25 +0,0 @@
Your mail to '$listname' with the subject
$subject
Is being held until the list moderator can review it for approval.
The message is being held because:
$reasons
Either the message will get posted to the list, or you will receive
notification of the moderator's decision.
NOTE: If your message has been held due to size, please consider
this a rejection and a friendly request to reduce the size and
resend. The list moderators will not approve such messages.
If you are attaching logs or command output, please trim it to a
minimal amount (we can ask for more context, if needed).
If you are attaching an image, please crop it to reduce their size.
Also keep in mind that attachments will be encoded by your mail
client (usually via base64) and this encoding increases the size by
as much as 33%.

17
roles/mailman/files/mailman3-fedmsg-plugin.RedHat.repo
View file

@ -1,17 +0,0 @@
[mailman3-fedmsg-plugin]
name=fedmsg plugin for mailman3, and its deps
baseurl=https://copr-be.cloud.fedoraproject.org/results/ralph/fedmsg-python34/epel-7-$basearch/
enabled=1
skip_if_unavailable=1
gpgcheck=1
gpgkey=https://copr-be.cloud.fedoraproject.org/results/ralph/fedmsg-python34/pubkey.gpg
## COPR doesn't provide a source repo as far as I can tell.
## Please fix this file if I'm wrong.
#[mailman3-fedmsg-plugin-source]
#name=fedmsg plugin for mailman3, and its deps - Source
#baseurl=https://copr-be.cloud.fedoraproject.org/results/ralph/fedmsg-python34/epel-7-source/
#enabled=0
#skip_if_unavailable=1
#gpgcheck=1
#gpgkey=https://copr-be.cloud.fedoraproject.org/results/ralph/fedmsg-python34/pubkey.gpg

5
roles/mailman/files/mailman3_navbar-brand.html
View file

@ -1,5 +0,0 @@
{% load static %}
<a class="navbar-brand" href="{% url 'hk_root' %}"
title="{{ site_name|title }}" style="padding:8px 15px">
<img alt="{{ site_name|title }}" src="{% static 'logo-hyperkitty-fedora.png' %}" />
</a>

85
roles/mailman/files/mailman3_pg-give-rights.py
View file

@ -1,85 +0,0 @@
#!/usr/bin/env python3
# vim: et ts=4 sw=4 fileencoding=utf-8
"""
Give non-admin rights to the database app user.
"""
CONFFILE = "/etc/mailman-migration.conf"
import site
import yaml
import psycopg2
def give_rights(dbhost, dbuser, dbpasswd, dbname, dbreguser=None):
if dbreguser is None:
dbreguser = dbname + "app"
conn = psycopg2.connect(host=dbhost, user=dbuser, password=dbpasswd,
database=dbname)
cur = conn.cursor()
# Database permissions
dbrightsquery = "GRANT CONNECT,TEMP ON DATABASE %s TO %s;" % (dbname, dbreguser)
print(dbrightsquery)
cur.execute(dbrightsquery)
# Table permissions
cur.execute("""
SELECT 'GRANT SELECT,INSERT,UPDATE,DELETE,TRUNCATE ON "' || relname || '" TO %s;'
FROM pg_class
JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace
WHERE nspname = 'public' AND relkind IN ('r', 'v');
""" % dbreguser)
queries = [q[0] for q in cur]
for query in queries:
print(query)
cur.execute(query)
# Sequence permissions
cur.execute("""
SELECT 'GRANT USAGE,SELECT,UPDATE ON ' || relname || ' TO %s;'
FROM pg_class
JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace
WHERE nspname = 'public' AND relkind = 'S';
""" % dbreguser)
queries = [q[0] for q in cur]
for query in queries:
print(query)
cur.execute(query)
conn.commit()
cur.close()
conn.close()
def main():
with open(CONFFILE) as conffile:
conf = yaml.safe_load(conffile)
site.addsitedir(conf["confdir"])
import settings_admin
## KittyStore
#dbspec = re.match("""
# postgresql://
# (?P<user>[a-z]+)
# :
# (?P<password>[^@]+)
# @
# (?P<host>[^/]+)
# /
# (?P<database>[^/?]+)
# """, settings_admin.KITTYSTORE_URL, re.X)
#give_rights(dbspec.group("host"),
# dbspec.group("user"),
# dbspec.group("password"),
# dbspec.group("database")
# )
# HyperKitty
give_rights(
settings_admin.DATABASES["default"]["HOST"],
settings_admin.DATABASES["default"]["USER"],
settings_admin.DATABASES["default"]["PASSWORD"],
settings_admin.DATABASES["default"]["NAME"],
)
if __name__ == "__main__": main()

17
roles/mailman/files/mailman3_urls.py
View file

@ -1,17 +0,0 @@
# -*- coding: utf-8 -*-
from django.urls import include, re_path, reverse_lazy
from django.contrib import admin
from django.views.generic import RedirectView
urlpatterns = [
re_path(r'^$', RedirectView.as_view(
url=reverse_lazy('list_index'),
permanent=True)),
re_path(r'^admin/', include('postorius.urls')),
re_path(r'^archives/', include('hyperkitty.urls')),
re_path(r'', include('django_mailman3.urls')),
re_path(r'^accounts/', include('allauth.urls')),
re_path(r'^django-admin/', admin.site.urls),
]

110
roles/mailman/files/mailman3_yamlget
View file

@ -1,110 +0,0 @@
#!/usr/bin/env python
# vim: set fileencoding=utf-8 tabstop=4 shiftwidth=4 expandtab smartindent:
u"""
yamlget
-------
Output any key in a YAML-formatted file. The aim is to make such a
configuration file accessible to shell scripts.
.. :Authors:
Aurélien Bompard <aurelien@bompard.org> <http://aurelien.bompard.org>
.. :License:
GNU GPL v3 or later
"""
from __future__ import print_function
import os
import sys
from optparse import OptionParser
import yaml
def get_key(fullkey, data):
"""
Get the requested key from the parsed data.
:param fullkey: the key to get, nested values can be accessed by using a
colon (":") as the separator
:param data: the parsed data, from yaml.load()
Examples:
>>> data = {
... 'bool': [True, False, True, False],
... 'dict': {'hp': 13, 'sp': 5},
... 'float': 3.14159,
... 'int': 42,
... 'list': ['LITE', 'RES_ACID', 'SUS_DEXT'],
... 'none': [None, None],
... 'text': "The Set of Gauntlets 'Pauraegen'",
... }
>>> get_key('bool', data)
[True, False, True, False]
>>> get_key('bool:2', data)
False
>>> get_key('dict', data)
{'hp': 13, 'sp': 5}
>>> get_key('dict:hp', data)
13
>>> get_key('float', data)
3.14159
>>> get_key('int', data)
42
>>> get_key('list', data)
['LITE', 'RES_ACID', 'SUS_DEXT']
>>> get_key('list:2', data)
'RES_ACID'
>>> get_key('list:2:5', data)
'RES_ACID'
>>> get_key('none', data)
[None, None]
>>> get_key('none:1', data)
>>> get_key('text', data)
"The Set of Gauntlets 'Pauraegen'"
>>> get_key('2', ['item1', 'item2', 'item3'])
'item2'
"""
value = data
while value is not None:
key, _sep, fullkey = fullkey.partition(":")
if isinstance(value, list):
try:
key = int(key)
except TypeError:
print("Wrong key format: %s, it should be an integer" % key,
file=sys.stderr)
sys.exit(1)
value = value[key - 1] # start at 1, not 0
elif isinstance(value, dict):
value = value.get(key)
else:
break # we've got the value now
if not fullkey:
break # can't go any further
return value
def main():
parser = OptionParser(usage="%prog <key> <yaml-file>")
args = parser.parse_args()[1]
if len(args) != 2:
parser.error("wrong number of arguments")
fullkey, filepath = args
if not os.path.exists(filepath):
parser.error("no such file: %s" % filepath)
with open(filepath) as yamlfile:
data = next(yaml.safe_load_all(yamlfile))
#from pprint import pprint; pprint(data)
value = get_key(fullkey, data)
if value is not None:
print(value)
if __name__ == "__main__":
main()

5
roles/mailman/files/memcached.sysconfig
View file

@ -1,5 +0,0 @@
PORT="11211"
USER="memcached"
MAXCONN="1024"
CACHESIZE="655350"
OPTIONS=""

5
roles/mailman/files/navbar-brand.html
View file

@ -1,5 +0,0 @@
{% load staticfiles %}
<a class="navbar-brand" href="{% url 'hk_root' %}"
title="{{ site_name|title }}" style="padding:8px 15px">
<img alt="{{ site_name|title }}" src="{% static 'logo-hyperkitty-fedora.png' %}" />
</a>

23
roles/mailman/files/periodic.py
View file

@ -1,23 +0,0 @@
#!/usr/bin/python3
import os
import sys
from mailman.core.initialize import initialize
from mailman.config import config
from mailman.interfaces.pending import IPendings
from mailman.interfaces.requests import IListRequests, RequestType
from zope.component import getUtility
def clean_pended():
getUtility(IPendings).evict()
if __name__ == '__main__':
if os.getuid() == 0:
print("This script must be run as the mailman user", file=sys.stderr)
sys.exit(1)
initialize(config_path="/etc/mailman.cfg")
clean_pended()
config.db.commit()

86
roles/mailman/files/pg-give-rights.py
View file

@ -1,86 +0,0 @@
#!/usr/bin/env python2
# vim: et ts=4 sw=4 fileencoding=utf-8
"""
Give non-admin rights to the database app user.
"""
CONFFILE = "/etc/mailman-migration.conf"
import site
import re
import yaml
import psycopg2
def give_rights(dbhost, dbuser, dbpasswd, dbname, dbreguser=None):
if dbreguser is None:
dbreguser = dbname + "app"
conn = psycopg2.connect(host=dbhost, user=dbuser, password=dbpasswd,
database=dbname)
cur = conn.cursor()
# Database permissions
dbrightsquery = "GRANT CONNECT,TEMP ON DATABASE %s TO %s;" % (dbname, dbreguser)
print dbrightsquery
cur.execute(dbrightsquery)
# Table permissions
cur.execute("""
SELECT 'GRANT SELECT,INSERT,UPDATE,DELETE,TRUNCATE ON "' || relname || '" TO %s;'
FROM pg_class
JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace
WHERE nspname = 'public' AND relkind IN ('r', 'v');
""" % dbreguser)
queries = [ q[0] for q in cur ]
for query in queries:
print query
cur.execute(query)
# Sequence permissions
cur.execute("""
SELECT 'GRANT USAGE,SELECT,UPDATE ON ' || relname || ' TO %s;'
FROM pg_class
JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace
WHERE nspname = 'public' AND relkind = 'S';
""" % dbreguser)
queries = [ q[0] for q in cur ]
for query in queries:
print query
cur.execute(query)
conn.commit()
cur.close()
conn.close()
def main():
with open(CONFFILE) as conffile:
conf = yaml.safe_load(conffile)
site.addsitedir(conf["confdir"])
import settings_admin
## KittyStore
#dbspec = re.match("""
# postgresql://
# (?P<user>[a-z]+)
# :
# (?P<password>[^@]+)
# @
# (?P<host>[^/]+)
# /
# (?P<database>[^/?]+)
# """, settings_admin.KITTYSTORE_URL, re.X)
#give_rights(dbspec.group("host"),
# dbspec.group("user"),
# dbspec.group("password"),
# dbspec.group("database")
# )
# HyperKitty
give_rights(
settings_admin.DATABASES["default"]["HOST"],
settings_admin.DATABASES["default"]["USER"],
settings_admin.DATABASES["default"]["PASSWORD"],
settings_admin.DATABASES["default"]["NAME"],
)
if __name__ == "__main__": main()

43
roles/mailman/files/post-update.sh
View file

@ -1,43 +0,0 @@
#!/bin/bash
CONFFILE=/etc/mailman-migration.conf
set -e
export PATH=$PATH:$(dirname $(realpath $0)) # make yamlget available
BASEDIR=`yamlget basedir $CONFFILE`
CONFDIR=`yamlget confdir $CONFFILE`
INDEXDIR=$BASEDIR/fulltext_index
sleep $[ ( $RANDOM % 10 ) + 1 ]s # avoid simultaneous lockups on parallel servers. Yes, this is dirty.
echo "Stop services"
systemctl stop webui-qcluster crond httpd
echo "static files"
django-admin collectstatic --clear --noinput --verbosity 0 --pythonpath $CONFDIR --settings settings
django-admin compress --pythonpath $CONFDIR --settings settings
echo "db migration"
django-admin migrate --pythonpath $CONFDIR --settings settings_admin --noinput
echo "give database rights to the non-admin user"
# this must be done before loading initial data
$BASEDIR/bin/pg-give-rights.py > /dev/null
echo "load initial data"
django-admin loaddata $CONFDIR/initial-data.json --pythonpath $CONFDIR --settings settings
mkdir -p $INDEXDIR
chown apache:apache -R $INDEXDIR
# SELinux contexts
echo "SELinux contexts"
restorecon -r $BASEDIR/{bin,config,fulltext_index,static,templates}
# Run unit tests
echo "unit tests"
#django-admin test --pythonpath $CONFDIR --settings settings_test django_mailman3 hyperkitty postorius
# Restart services
echo "Start services"
systemctl start httpd crond webui-qcluster

211
roles/mailman/files/prod-to-stg.py
View file

@ -1,211 +0,0 @@
#!/usr/bin/python2
from __future__ import unicode_literals, print_function
import os
import sys
import psycopg2
from ConfigParser import ConfigParser
from subprocess import call
from urlparse import urlparse
if not os.getenv("DJANGO_SETTINGS_MODULE"):
os.environ["DJANGO_SETTINGS_MODULE"] = "settings_admin"
sys.path.insert(0, "/srv/webui/config")
import django
django.setup()
MAILMAN_TABLES_TO_REPLACE = [
("domain", "mail_host"),
("mailinglist", "mail_host"),
("mailinglist", "list_id"),
#("ban", "list_id"),
#("bounceevent", "list_id"),
#("member", "list_id"),
("template", "context"),
("acceptablealias", "alias"),
]
DJANGO_TABLES_TO_EMPTY = [
"account_emailconfirmation",
"openid_openidnonce",
"openid_openidstore",
"social_auth_association",
"social_auth_nonce",
"social_auth_usersocialauth",
"socialaccount_socialtoken",
]
DJANGO_INDICES_TO_RECREATE = [
("hyperkitty_thread", "hyperkitty_thread_mailinglist_id", "(mailinglist_id)"),
("hyperkitty_thread", "hyperkitty_thread_mailinglist_id_like", "(mailinglist_id varchar_pattern_ops)"),
("hyperkitty_email", "hyperkitty_email_mailinglist_id", "(mailinglist_id)"),
("hyperkitty_email", "hyperkitty_email_mailinglist_id_like", "(mailinglist_id varchar_pattern_ops)"),
]
DJANGO_CONSTRAINTS_TO_RECREATE = [
("hyperkitty_thread", "hyperkitty_thread_mailinglist_id_371b52d98485a593_uniq", "UNIQUE (mailinglist_id, thread_id)"),
("hyperkitty_thread", "mailinglist_id_refs_name_3725eec2", "FOREIGN KEY (mailinglist_id) REFERENCES hyperkitty_mailinglist(name) DEFERRABLE INITIALLY DEFERRED"),
("hyperkitty_email", "hyperkitty_email_mailinglist_id_57f04aace3f8ee5e_uniq", "UNIQUE (mailinglist_id, message_id)"),
("hyperkitty_email", "mailinglist_id_refs_name_654506d3", "FOREIGN KEY (mailinglist_id) REFERENCES hyperkitty_mailinglist(name) DEFERRABLE INITIALLY DEFERRED"),
]
def get_mapping(cursor, table, column):
ml_mapping = {}
query = "SELECT {c} FROM {t}".format(c=column, t=table)
print(query)
cursor.execute(query)
for row in cursor:
value = row[0]
orig_value = value.replace(
"lists.stg.fedoraproject.org", "lists.fedoraproject.org"
).replace(
"lists.stg.fedorahosted.org", "lists.fedorahosted.org")
changed_value = value.replace(
"lists.fedoraproject.org", "lists.stg.fedoraproject.org"
).replace(
"lists.fedorahosted.org", "lists.stg.fedorahosted.org")
if orig_value == changed_value:
continue
ml_mapping[orig_value] = changed_value
return ml_mapping
def update_col_1(connection, table, column, where=None, pk="id"):
cursor = connection.cursor()
cursor_2 = connection.cursor()
where = " WHERE {}".format(where) if where is not None else ""
#query = "SELECT COUNT(*) FROM {t} {w}".format(t=table, w=where)
#cursor.execute(query)
#count = cursor.fetchone()[0]
query = "SELECT {pk}, {c} FROM {t} {w}".format(
t=table, c=column, pk=pk, w=where)
#query += " LIMIT 10000"
print(query)
#print("{} lines".format(count))
updates = []
cursor.execute(query)
print(cursor.query)
for record_id, value in cursor:
changed_value = value.replace(
"lists.fedoraproject.org", "lists.stg.fedoraproject.org"
).replace(
"lists.fedorahosted.org", "lists.stg.fedorahosted.org")
if value == changed_value:
continue
if column == pk:
# Resilience: if the process is halted, there may be old and new
# values in the same table.
cursor_2.execute(
"SELECT 1 FROM {t} WHERE {pk} = %s".format(t=table, pk=pk),
[changed_value])
if cursor_2.fetchone():
print("Skipping {v} in {t}".format(t=table, v=changed_value))
continue
updates.append([changed_value, record_id])
cursor_2.close()
print("Doing {} updates now".format(len(updates)))
query = "UPDATE {t} SET {c} = %s WHERE {pk} = %s".format(
t=table, c=column, pk=pk)
print(query, "with %d params" % len(updates))
cursor.executemany(query, updates)
def update_col_2(ml_mapping, cursor, table, column, where=None):
query_where = " AND {}".format(where) if where is not None else ""
query = "SELECT COUNT(*) FROM {t} {w}".format(t=table, w=where)
cursor.execute(query)
count = cursor.fetchone()[0]
print("Updating {} rows.".format(count))
for name_orig, name_new in ml_mapping.items():
query = "UPDATE {t} SET {c} = %s WHERE {c} = %s {w}".format(
t=table, c=column, w=query_where)
params = (name_new, name_orig)
print(query % params)
cursor.execute(query, params)
def do_mailman():
config = ConfigParser()
config.read("/etc/mailman.cfg")
conn = psycopg2.connect(config.get("database", "url"))
#db_url = urlparse(config.get("database", "url"))
#conn = psycopg2.connect(
# "dbname={scheme} user={username} password={password} host={hostname}".format(db_url)
# )
with conn.cursor() as cursor:
ml_mapping = get_mapping(cursor, "mailinglist", "list_id")
for table, column in MAILMAN_TABLES_TO_REPLACE:
update_col_1(conn, table, column)
update_col_2(ml_mapping, cursor, "ban", "list_id")
update_col_2(ml_mapping, cursor, "member", "list_id")
update_col_2(ml_mapping, cursor, "bounceevent", "list_id")
update_col_1(conn, "pendedkeyvalue", "value",
""" "key" = 'list_id' OR "key" = '_mod_listid' """
""" OR "key" = 'envsender'""")
cursor.execute("UPDATE \"user\" SET password = 'INVALID'")
print(cursor.query)
cursor.execute("UPDATE \"mailinglist\" SET digests_enabled = FALSE")
print(cursor.query)
conn.commit()
conn.close()
call(["sudo", "-u", "mailman", "mailman3", "aliases"])
def do_django():
from django.db import connection, transaction
from django.core.management import call_command
with connection.cursor() as cursor:
cursor.execute("UPDATE auth_user SET password = '!INVALID'")
print(cursor.query)
# Empty tables that contain sensitive data
for table in DJANGO_TABLES_TO_EMPTY:
cursor.execute("DELETE FROM %s" % table)
print(cursor.query)
#for table, name, create in DJANGO_CONSTRAINTS_TO_RECREATE:
# cursor.execute(
# "ALTER TABLE {t} DROP CONSTRAINT IF EXISTS {n}".format(
# t=table, n=name))
# print(cursor.query)
for table, name, column in DJANGO_INDICES_TO_RECREATE:
cursor.execute("DROP INDEX IF EXISTS {n}".format(t=table, n=name))
print(cursor.query)
with transaction.atomic():
cursor.execute("SET CONSTRAINTS ALL DEFERRED")
ml_mapping = get_mapping(cursor, "hyperkitty_mailinglist", "name")
# Replace in tables with prod domains:
update_col_1(connection, "django_mailman3_maildomain", "mail_domain")
update_col_1(connection, "hyperkitty_mailinglist", "name", pk="name")
update_col_2(ml_mapping, cursor, "hyperkitty_thread", "mailinglist_id")
update_col_2(ml_mapping, cursor, "hyperkitty_email", "mailinglist_id")
cursor.execute("SET CONSTRAINTS ALL IMMEDIATE")
for table, name, column in DJANGO_INDICES_TO_RECREATE:
cursor.execute("CREATE INDEX {n} ON {t} {c}".format(
n=name, t=table, c=column))
print(cursor.query)
#for table, name, create in DJANGO_CONSTRAINTS_TO_RECREATE:
# cursor.execute("ALTER TABLE {t} ADD CONSTRAINT {n} {c}".format(
# n=name, t=table, c=create))
# print(cursor.query)
connection.commit()
call_command("loaddata", "/srv/webui/config/initial-data.json")
def main():
call(["systemctl", "stop", "webui-qcluster"])
call(["systemctl", "stop", "mailman3"])
call(["systemctl", "stop", "httpd"])
call(["systemctl", "stop", "crond"])
do_mailman()
do_django()
call(["systemctl", "start", "crond"])
call(["systemctl", "start", "httpd"])
call(["systemctl", "start", "mailman3"])
call(["systemctl", "start", "webui-qcluster"])
if __name__ == "__main__":
main()

4
roles/mailman/files/robots.txt
View file

@ -1,4 +0,0 @@
User-agent: *
Disallow: /accounts/
Disallow: /django-admin/
Disallow: /archives/*/vote$

85
roles/mailman/files/settings_test.py
View file

@ -1,85 +0,0 @@
#-*- coding: utf-8 -*-
"""
Copy of the Django settings file, but with the database set for unit tests.
"""
from settings import *
try:
from settings_local import *
except ImportError:
pass
TESTING = True
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': ':memory:',
}
}
# Mailman API credentials for testing Postorius
MAILMAN_REST_API_URL = 'http://localhost:9001'
MAILMAN_REST_API_USER = 'restadmin'
MAILMAN_REST_API_PASS = 'restpass'
VCR_RECORD_MODE = 'once'
USE_SSL = False
COMPRESS_ENABLED = False
# Empty the precompilers mapping for testing: django-compressor will run them
# even if compress_enabled is false, no idea why
COMPRESS_PRECOMPILERS = ()
#
# Full-text search engine
#
HAYSTACK_CONNECTIONS = {
'default': {
'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
'PATH': ':memory:',
'STORAGE': 'ram',
},
}
HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'
#
# Asynchronous tasks
#
Q_CLUSTER = {
'orm': 'default',
'sync': True,
}
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'console': {
'class': 'logging.StreamHandler',
},
},
'loggers': {
'hyperkitty': {
'handlers': ['console'],
'level': 'INFO',
'propagate': True,
},
'django_mailman3.lib.mailman': {
'handlers': ['console'],
'level': 'ERROR',
},
'django-q': {
'handlers': ['console'],
'level': 'WARNING',
},
},
}
# Disable caching
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
}
}

21
roles/mailman/files/signup_closed.html
View file

@ -1,21 +0,0 @@
{% extends "account/base.html" %}
{% load i18n %}
{% block head_title %}{% trans "Signup" %}{% endblock %}
{% block content %}
<h1>{% trans "Sign Up" %}</h1>
<p>
{% blocktrans %}
Account creation for the Fedora mailing-lists is done by
<a href="https://accounts.fedoraproject.org/">creating an account in FAS</a>.
{% endblocktrans %}
</p>
{% url LOGIN_URL as login_url %}
<p>{% blocktrans %}Already have an account? Then please <a href="{{login_url}}">sign in</a>.{% endblocktrans %}</p>
{% endblock %}

20
roles/mailman/files/syslog-logrotate
View file

@ -1,20 +0,0 @@
/var/log/cron
/var/log/maillog
/var/log/messages
/var/log/secure
/var/log/spooler
{
sharedscripts
postrotate
/bin/kill -HUP `cat /var/run/syslogd.pid 2> /dev/null` 2> /dev/null || true
endscript
daily
rotate 7
missingok
ifempty
compress
compresscmd /usr/bin/xz
uncompresscmd /usr/bin/xz
compressext .xz
dateext
}

10
roles/mailman/files/top.html
View file

@ -1,10 +0,0 @@
<!-- This markup will be inserted after the <body> tag -->
<!-- https://pagure.io/fedora-infrastructure/issue/10741 -->
<!--
<div style="margin: auto" class="container">
<a href="https://fedoraproject.limequery.com/2023" target="_blank">
<img src="https://pagure.io/fedora-infrastructure/issue/raw/files/b4d328e7ce730f610530822af4c9cdab1029c47dc2bdb15648d4008e1b304e43-contributor-survey-banner.png" style="max-width: 100%">
</a>
</div>
-->

18
roles/mailman/files/urls.py
View file

@ -1,18 +0,0 @@
# -*- coding: utf-8 -*-
from django.conf.urls import include, url
from django.contrib import admin
from django.core.urlresolvers import reverse_lazy
from django.views.generic import RedirectView
urlpatterns = [
url(r'^$', RedirectView.as_view(
url=reverse_lazy('hyperkitty.views.index.index'),
permanent=True)),
url(r'^admin/', include('postorius.urls')),
url(r'^archives/', include('hyperkitty.urls')),
url(r'', include('django_mailman3.urls')),
url(r'^accounts/', include('allauth.urls')),
url(r'^django-admin/', include(admin.site.urls)),
]

44
roles/mailman/files/webui.wsgi
View file

@ -1,44 +0,0 @@
"""
WSGI config for hyperkitty_standalone project.
This module contains the WSGI application used by Django's development server
and any production WSGI deployments. It should expose a module-level variable
named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover
this application via the ``WSGI_APPLICATION`` setting.
Usually you will have the standard Django WSGI application here, but it also
might make sense to replace the whole Django WSGI application with a custom one
that later delegates to the Django one. For example, you could introduce WSGI
middleware here, or combine a Django application with an application of another
framework.
"""
import os
import sys
import site
## For some unknown reason, sometimes mod_wsgi fails to set the python paths to
## the virtualenv, with the 'python-path' option. You can do it here too.
##
## Remember original sys.path.
#prev_sys_path = list(sys.path)
## Add here, for the settings module
#site.addsitedir(os.path.abspath(os.path.dirname(__file__)))
## Add the virtualenv
#venv = os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', 'lib', 'python2.6', 'site-packages')
#site.addsitedir(venv)
# Reorder sys.path so new directories at the front.
#new_sys_path = []
#for item in list(sys.path):
# if item not in prev_sys_path:
# new_sys_path.append(item)
# sys.path.remove(item)
# sys.path[:0] = new_sys_path
site.addsitedir(os.path.abspath(os.path.dirname(__file__)))
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()

110
roles/mailman/files/yamlget
View file

@ -1,110 +0,0 @@
#!/usr/bin/env python
# vim: set fileencoding=utf-8 tabstop=4 shiftwidth=4 expandtab smartindent:
u"""
yamlget
-------
Output any key in a YAML-formatted file. The aim is to make such a
configuration file accessible to shell scripts.
.. :Authors:
Aurélien Bompard <aurelien@bompard.org> <http://aurelien.bompard.org>
.. :License:
GNU GPL v3 or later
"""
from __future__ import print_function
import os
import sys
from optparse import OptionParser
import yaml
def get_key(fullkey, data):
"""
Get the requested key from the parsed data.
:param fullkey: the key to get, nested values can be accessed by using a
colon (":") as the separator
:param data: the parsed data, from yaml.load()
Examples:
>>> data = {
... 'bool': [True, False, True, False],
... 'dict': {'hp': 13, 'sp': 5},
... 'float': 3.14159,
... 'int': 42,
... 'list': ['LITE', 'RES_ACID', 'SUS_DEXT'],
... 'none': [None, None],
... 'text': "The Set of Gauntlets 'Pauraegen'",
... }
>>> get_key('bool', data)
[True, False, True, False]
>>> get_key('bool:2', data)
False
>>> get_key('dict', data)
{'hp': 13, 'sp': 5}
>>> get_key('dict:hp', data)
13
>>> get_key('float', data)
3.14159
>>> get_key('int', data)
42
>>> get_key('list', data)
['LITE', 'RES_ACID', 'SUS_DEXT']
>>> get_key('list:2', data)
'RES_ACID'
>>> get_key('list:2:5', data)
'RES_ACID'
>>> get_key('none', data)
[None, None]
>>> get_key('none:1', data)
>>> get_key('text', data)
"The Set of Gauntlets 'Pauraegen'"
>>> get_key('2', ['item1', 'item2', 'item3'])
'item2'
"""
value = data
while value is not None:
key, _sep, fullkey = fullkey.partition(":")
if isinstance(value, list):
try:
key = int(key)
except TypeError:
print("Wrong key format: %s, it should be an integer" % key,
file=sys.stderr)
sys.exit(1)
value = value[key - 1] # start at 1, not 0
elif isinstance(value, dict):
value = value.get(key)
else:
break # we've got the value now
if not fullkey:
break # can't go any further
return value
def main():
parser = OptionParser(usage="%prog <key> <yaml-file>")
args = parser.parse_args()[1]
if len(args) != 2:
parser.error("wrong number of arguments")
fullkey, filepath = args
if not os.path.exists(filepath):
parser.error("no such file: %s" % filepath)
with open(filepath) as yamlfile:
data = yaml.safe_load_all(yamlfile).next()
#from pprint import pprint; pprint(data)
value = get_key(fullkey, data)
if value is not None:
print(value)
if __name__ == "__main__":
main()

12
roles/mailman/handlers/main.yml
View file

@ -1,12 +0,0 @@
---
- name: restart mailman3
service: name=mailman3 state=restarted
- name: reload apache
service: name=httpd state=reloaded
- name: restart memcached
service: name=memcached state=restarted
- name: systemctl daemon-reload
command: /usr/bin/systemctl daemon-reload

717
roles/mailman/tasks/main.yml
View file

@ -1,717 +0,0 @@
---
# Configuration for Mailman 3
# PostgreSQL initialization must have been done already
#
# SELinux
#
- name: set the SELinux policy for the configuration directory
sefcontext:
target: "{{ mailman_webui_confdir }}(/.*)?"
setype: etc_t
state: present
tags:
- mailman
- selinux
- name: set the SELinux policy for the fulltext index
sefcontext:
target: "{{ mailman_webui_basedir }}/fulltext_index(/.*)?"
setype: httpd_sys_rw_content_t
state: present
tags:
- mailman
- selinux
- name: set the SELinux policy for the static files directory
sefcontext:
target: "{{ mailman_webui_basedir }}/static(/.*)?"
setype: httpd_sys_content_t
state: present
tags:
- mailman
- selinux
- name: set the SELinux policy for the templates override directory
sefcontext:
target: "{{ mailman_webui_basedir }}/templates(/.*)?"
setype: httpd_sys_content_t
state: present
tags:
- mailman
- selinux
- name: set the SELinux policy for the log directory
sefcontext:
target: "/var/log/hyperkitty(/.*)?"
setype: httpd_log_t
state: present
tags:
- mailman
- selinux
- name: set the SELinux policy for the generated postfix databases
sefcontext:
target: "{{ mailman_webui_basedir }}/var/data/postfix_.*"
setype: etc_aliases_t
state: present
tags:
- mailman
- selinux
- name: set the SELinux policy for the old static archives
sefcontext:
target: "{{ mailman_webui_basedir }}/old-archives/pipermail(/.*)?"
setype: httpd_sys_content_t
state: present
tags:
- mailman
- selinux
- name: allow Apache to remotely connect to PostgreSQL
seboolean: name=httpd_can_network_connect_db state=yes persistent=yes
tags:
- mailman
- selinux
- name: allow Apache to remotely connect to Mailman
seboolean: name=httpd_can_network_connect state=yes persistent=yes
tags:
- mailman
- selinux
- name: allow Apache to remotely connect to Memcached
seboolean: name=httpd_can_network_memcache state=yes persistent=yes
tags:
- mailman
- selinux
when: env == 'production'
#
# Packages
#
- name: setup the hyperkitty repo
copy: src={{item}} dest=/etc/yum.repos.d/hyperkitty.repo
with_first_found:
- hyperkitty.{{ansible_hostname}}.repo
- hyperkitty.{{ansible_distribution}}.repo
- hyperkitty.repo
tags: mailman
when: env == 'production'
- name: install GPG to validate the key
package: state=present name=gnupg
tags: mailman
when: env == 'production'
- name: add the GPG key
rpm_key: state=present key=https://repos.fedorapeople.org/repos/abompard/abompard.asc
tags: mailman
when: env == 'production'
- name: install needed packages
package: name={{ item }} state=present
with_items:
- python-psycopg2
- python34-psycopg2
- hyperkitty
- hyperkitty-selinux
- postorius
- memcached
- python-pylibmc
- python-django-haystack-xapian
- yum-plugin-post-transaction-actions
# to run the test suite:
- python-beautifulsoup4
- python-mock
- python-whoosh
- python-tox
- python-vcrpy
# scripts
- python34-PyYAML
# mailman soft dep to convert html to plaintext
- lynx
- fedora-messaging
tags:
- packages
- mailman
when: env == 'production'
- name: install needed packages
package: name={{ item }} state=present
with_items:
- mailman3
# - mailman3-hyperkitty
- hyperkitty
- postorius
- memcached
- fedora-messaging
- httpd
- python3-pylibmc
- python3-dnf-plugin-post-transaction-actions
- python3-psycopg2
- python3-mod_wsgi
- sassc
# to run the test suite:
- python-beautifulsoup4
- python3-whoosh
tags:
- packages
- mailman
when: env == 'staging'
- name: setup the mailman3-fedmsg repo
copy: src=mailman3-fedmsg-plugin.RedHat.repo
dest=/etc/yum.repos.d/mailman3-fedmsg-plugin.repo
when: ansible_distribution == 'RedHat' and env == 'production'
tags: mailman
- name: add the GPG key for the mailman3-fedmsg-plugin repo
rpm_key: state=present
key=https://copr-be.cloud.fedoraproject.org/results/ralph/fedmsg-python34/pubkey.gpg
when: ansible_distribution == 'RedHat' and env == 'production'
tags: mailman
- name: install the mailman3 fedmsg plugin rpm
package: name=mailman3-fedmsg-plugin
notify: restart mailman3
when: ansible_distribution == 'RedHat' and env == 'production'
tags: mailman
- name: copy in our fedmsg-plugin conf file
copy: src=fedmsg-plugin-conf.py dest=/etc/fedmsg.d/fedmsg-plugin-conf.py
tags:
- fedmsgdconfig
- mailman
notify: restart mailman3
when: env == 'production'
- name: install packages when not using source extracts
package: name={{ item }} state=present
with_items:
- mailman3
- mailman3-selinux
- mailman3-hyperkitty
tags:
- packages
- mailman
when: env == 'production'
## install hotfix for e-mails containing broken unicode
## see issue https://pagure.io/fedora-infrastructure/issue/8824
## see upstream merge request https://gitlab.com/mailman/mailman/-/merge_requests/350
- name: hotfix - let mailman cope with e-mails containing broken unicode
patch: src={{ files }}/hotfix/mailman3/mailman3-broken-unicode-emails.patch
dest=/usr/lib/python3.4/site-packages/mailman/email/message.py
notify:
- restart mailman3
- reload apache
tags:
- mailman
- hotfix
- patches
when: env == 'production'
#
# Initialize mailman (must be done after settings up the DBs)
#
# - name: add mailman to the apache group
# user: name=mailman groups=apache append=yes
# tags:
# - config
# - mailman
# #notify:
# # - restart mailman3
# access to the aliases files generated by mailman
- name: add postfix to the mailman group
user: name=postfix groups=mailman append=yes
tags:
- config
- mailman
notify:
- restart postfix
## for access to the full-text index
# - name: add apache to the mailman group
# user: name=apache groups=mailman append=yes
# tags:
# - config
# - mailman
# notify:
# - reload httpd
- name: set the mailman conffile
template: src={{ item }} dest=/etc/mailman.cfg
owner=root group=mailman mode=0640
with_first_found:
- mailman.cfg.{{ ansible_hostname }}.j2
- mailman.cfg.j2
tags:
- config
- mailman
notify:
- restart mailman3
- name: Create site/en dir
file: state=directory path=/var/lib/mailman3/templates/site/en
tags:
- config
- mailman
- name: set default list footer
copy: src=mailman-template-list-member-generic-footer.txt
dest=/var/lib/mailman3/templates/site/en/list:member:generic:footer.txt
owner=mailman group=mailman mode=0644
tags:
- config
- mailman
- name: set default list:admin:action:post template
copy: src=mailman-template-list-admin-action-post.txt
dest=/var/lib/mailman3/templates/site/en/list:admin:action:post.txt
owner=mailman group=mailman mode=0644
tags:
- config
- mailman
- name: Create lists/users.lists.fedoraproject.org/en template dir
file: state=directory path=/var/lib/mailman3/templates/lists/users.lists.fedoraproject.org/en
tags:
- config
- mailman
- name: set list:user:notice:hold template for users list
copy: src=mailman-template-users-list-user-notice-hold.txt
dest=/var/lib/mailman3/templates/lists/users.lists.fedoraproject.org/en/list:user:notice:hold.txt
owner=mailman group=mailman mode=0644
tags:
- config
- mailman
#
# Crontab
#
- name: set the hyperkitty crontab
template: src=crontab-webui.j2 dest=/etc/cron.d/hyperkitty
tags:
- config
- mailman
- name: set the mailman crontab
template: src=crontab-mailman3.j2 dest=/etc/cron.d/mailman3
tags:
- config
- mailman
#
# Logging
#
- name: hyperkitty logging -- directory
file: path=/var/log/hyperkitty state=directory
owner=root group=apache mode=2775
tags: mailman
- name: hyperkitty logging -- file creation
copy: content="" dest=/var/log/hyperkitty/hyperkitty.log
force=no
tags: mailman
- name: hyperkitty logging -- file permissions
file: path=/var/log/hyperkitty/hyperkitty.log state=file
owner=root group=apache mode=664
tags: mailman
- name: hyperkitty logging -- rotation
copy: src=hyperkitty.logrotate.conf
dest=/etc/logrotate.d/hyperkitty
tags: mailman
#
# HyperKitty + Postorius setup
#
- name: create the configuration directory
file: path={{ mailman_webui_confdir }} state=directory
tags: mailman
- name: install the hyperkitty settings file
template: src=settings.py.j2
dest="{{ mailman_webui_confdir }}/settings.py"
owner=root group=apache mode=0640
tags:
- config
- mailman
notify:
- reload apache
# - restart mailman3
- name: install the hyperkitty settings admin file
template: src=settings_admin.py.j2
dest="{{ mailman_webui_confdir }}/settings_admin.py"
owner=root group=root mode=0600
tags:
- config
- mailman
- name: install the hyperkitty settings test file
copy: src=settings_test.py
dest="{{ mailman_webui_confdir }}/settings_test.py"
owner=root group=root mode=0644
tags:
- config
- mailman
- name: install the fedora-specific modules
copy: src={{ item }}.py
dest="{{ mailman_webui_confdir }}/{{ item }}.py"
owner=root group=root mode=0644
with_items:
- django_fedora
- django_fedora_nosignup
tags:
- config
- mailman
notify:
- reload apache
- name: install the hyperkitty urls file
copy: src=mailman3_urls.py
dest="{{ mailman_webui_confdir }}/urls.py"
owner=root group=root mode=0644
tags:
- config
- mailman
notify:
- reload apache
when: env == 'staging'
- name: install the hyperkitty urls file
copy: src=urls.py
dest="{{ mailman_webui_confdir }}/urls.py"
owner=root group=root mode=0644
tags:
- config
- mailman
notify:
- reload apache
when: env == 'production'
- name: install the hyperkitty wsgi file
copy: src=webui.wsgi
dest="{{ mailman_webui_confdir }}/webui.wsgi"
owner=root group=root mode=0644
tags:
- config
- mailman
notify:
- reload apache
- name: install the hyperkitty/postorius dummy httpd conf file
template: src=apache-dummy.conf.j2
dest=/etc/httpd/conf.d/{{ item }}.conf
with_items:
- hyperkitty
- postorius
tags:
- config
- mailman
notify:
- reload apache
- name: install the hyperkitty httpd conf file
template: src=apache.conf.j2
dest=/etc/httpd/conf.d/mailman-webui.conf
tags:
- config
- mailman
notify:
- reload apache
- name: create the fulltext index dir
file: path="{{ mailman_webui_basedir }}/fulltext_index"
state=directory owner=apache group=apache mode=0755
tags: mailman
- name: create the hyperkitty static files dir
file: path="{{ mailman_webui_basedir }}/static"
state=directory owner=root group=root mode=0755
tags: mailman
- name: create the fedora-specific static files dir
file: path="{{ mailman_webui_basedir }}/static-fedora"
state=directory owner=root group=root mode=0755
tags: mailman
- name: create the fedora-specific hyperkitty img dir
file: path="{{ mailman_webui_basedir }}/static-fedora/hyperkitty/img"
state=directory owner=root group=root mode=0755
tags: mailman
- name: create the hyperkitty templates override dirs
file: path="{{ mailman_webui_basedir }}/templates/{{ item }}"
state=directory owner=root group=root mode=0755
with_items:
- hyperkitty
- django_mailman3
tags: mailman
# Add the Nest banner
# See issue https://pagure.io/fedora-infrastructure/issue/10103
- name: Replace the top.html template in hyperkitty to change banner
copy: src=top.html dest="{{ mailman_webui_basedir }}/templates/hyperkitty/top.html"
notify:
- reload apache
tags:
- mailman
- banner
- name: install our fedora-specific logo
copy: src=logo-hyperkitty-fedora.png
dest="{{ mailman_webui_basedir }}/static-fedora/logo-hyperkitty-fedora.png"
tags: mailman
- name: install our fedora-specific favicon
copy: src=favicon.ico
dest="{{ mailman_webui_basedir }}/static-fedora/favicon.ico"
tags: mailman
- name: install our fedora-specific robots.txt
copy: src=robots.txt
dest="{{ mailman_webui_basedir }}/static-fedora/robots.txt"
tags: mailman
- name: install our fedora-specific brand template override
copy: src=navbar-brand.html
dest="{{ mailman_webui_basedir }}/templates/hyperkitty/navbar-brand.html"
tags: mailman
when: env == 'production'
- name: install our fedora-specific brand template override
copy: src=mailman3_navbar-brand.html
dest="{{ mailman_webui_basedir }}/templates/hyperkitty/navbar-brand.html"
tags: mailman
when: env == 'staging'
- name: install our fedora-specific page headers
copy: src=headers.html
dest="{{ mailman_webui_basedir }}/templates/hyperkitty/headers.html"
tags: mailman
- name: install our fedora-specific login page extension
copy: src=login_extra_top.html
dest="{{ mailman_webui_basedir }}/templates/django_mailman3/login_extra_top.html"
tags: mailman
- name: Create dirs for signup_closed page
file: path="{{ mailman_webui_basedir }}/templates/account"
state=directory owner=root group=root mode=0755
tags: mailman
- name: install our fedora-specific signup_closed page
copy: src=signup_closed.html
dest="{{ mailman_webui_basedir }}/templates/account/signup_closed.html"
tags: mailman
- name: Install our fedmenu js hook
template: src=bottom.html
dest="{{mailman_webui_basedir }}/templates/hyperkitty/bottom.html"
tags: mailman
#
# Plug HyperKitty into Mailman
#
- name: copy the mailman-hyperkitty conffile
copy: src=mailman-hyperkitty.cfg
dest="/etc/mailman3.d/hyperkitty.cfg"
owner=root group=mailman mode=0640
tags:
- config
- mailman
notify:
- restart mailman3
#
# Scripts
#
- name: install the migration conffile
template: src=mailman-migration.conf.j2
dest=/etc/mailman-migration.conf
owner=root group=root mode=0644
tags: mailman
- name: create the scripts dir
file: path="{{ mailman_webui_basedir }}/bin"
state=directory owner=root group=root mode=0755
tags: mailman
- name: install the migration environment
template: src=mailman-migration-path.sh.j2
dest=/etc/profile.d/mailman-migration-path.sh
owner=root group=root mode=0644
tags: mailman
- name: install the scripts
copy: src={{ item }} dest="{{ mailman_webui_basedir }}/bin/{{ item }}"
owner=root group=root mode=0755
tags: mailman
with_items:
- yamlget
- pg-give-rights.py
- post-update.sh
- import-mm2.py
- periodic.py
- mailman-sar.py
- hyperkitty-delete-list.py
- name: install the updated yamlget script
copy: src=mailman3_yamlget dest="{{ mailman_webui_basedir }}/bin/yamlget"
owner=root group=root mode=0755
tags: mailman
when: env == 'staging'
- name: install the updated pg-give-rights script
copy: src=mailman3_pg-give-rights.py dest="{{ mailman_webui_basedir }}/bin/pg-give-rights.py"
owner=root group=root mode=0755
tags: mailman
when: env == 'staging'
- name: install the templatized scripts
template: src={{ item }}.j2 dest="{{ mailman_webui_basedir }}/bin/{{ item }}"
owner=root group=root mode=0755
tags: mailman
with_items:
- manage.py
- name: install the staging-sync script
copy: src=prod-to-stg.py dest="{{ mailman_webui_basedir }}/bin/prod-to-stg.py"
when: env == "staging"
tags: mailman
- name: copy the initial user fixture
template: src=initial-data.json.j2
dest={{ mailman_webui_basedir }}/config/initial-data.json
owner=root group=apache mode=0640
when: inventory_hostname.startswith('mailman01')
tags: mailman
# Sync databases and collect static files on RPM install/upgrade
- name: install the post-transaction trigger
template: src=post-transaction.action.j2
dest=/etc/dnf/plugins/post-transaction-actions.d/hyperkitty.action
tags: mailman
when: env == 'staging'
# Sync databases and collect static files on RPM install/upgrade
- name: install the post-transaction trigger
template: src=post-transaction.action.j2
dest=/etc/yum/post-actions/hyperkitty.action
tags: mailman
when: env == 'production'
# Systemd
- name: install the systemd service files
template: src={{ item }}.service.j2 dest=/etc/systemd/system/{{ item }}.service
with_items:
- webui-qcluster
- webui-warm-up-cache
notify:
- systemctl daemon-reload
tags:
- config
- mailman
#
# Only run this on mailman01 for now.
#
# The post-update scripts needs memcached to be up (django-compressor will
# store the timestamps there)
- name: start services
service: state=started enabled=yes name=memcached
tags: mailman
- name: run the post-update script
command: "{{ mailman_webui_basedir }}/bin/post-update.sh"
when: inventory_hostname.startswith('mailman01')
tags: mailman
## Postfix
# - name: create the postfix aliases
# command: su mailman -s /bin/sh -c "mailman3 aliases"
# creates=/var/lib/mailman3/data/postfix_lmtp.db
# Memcached
- name: set the memcached sysconfig file
copy: src=memcached.sysconfig dest=/etc/sysconfig/memcached
tags: mailman
notify:
- restart memcached
# SSL
- name: Letsencrypt for lists.pagure.org
include_role: name=letsencrypt
vars:
site_name: lists.pagure.io
when: env == 'production'
# Start services
- name: start services
service: state=started enabled=yes name={{ item }}
with_items:
- httpd
- mailman3
- postfix
- webui-qcluster
tags: mailman
when: inventory_hostname.startswith('mailman01.iad2')
- name: enable one-shot services
service: enabled=yes name={{ item }}
with_items:
- webui-warm-up-cache
tags: mailman
when: inventory_hostname.startswith('mailman01.iad2')
- name: Create /etc/pki/fedora-messaging
file:
dest: /etc/pki/fedora-messaging
mode: 0775
owner: root
group: root
state: directory
when: "deployment_type is defined"
tags:
- config
# FIXME: Need to create a mailman cert
- name: Deploy the Fedora mailman fedora-messaging cert
copy:
src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/mailman{{env_suffix}}.crt"
dest: /etc/pki/fedora-messaging/mailman{{env_suffix}}-cert.pem
mode: 0644
owner: root
group: root
when: "deployment_type is defined"
tags:
- config
- name: Deploy the Fedora infra fedora-messaging key
copy:
src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/mailman{{env_suffix}}.key"
dest: /etc/pki/fedora-messaging/mailman{{env_suffix}}-key.pem
mode: 0640
owner: root
group: root
when: "deployment_type is defined"
tags:
- config
- name: override the default syslog logrotate file
copy: src=syslog-logrotate dest=/etc/logrotate.d/syslog
tags:
- pagure
- logrotate

1
roles/mailman/templates/apache-dummy.conf.j2
View file

@ -1 +0,0 @@
# See {{ mailman_webui_confdir }}

43
roles/mailman/templates/apache.conf.j2
View file

@ -1,43 +0,0 @@
Alias /favicon.ico {{ mailman_webui_basedir }}/static/favicon.ico
Alias /robots.txt {{ mailman_webui_basedir }}/static/robots.txt
Alias /static {{ mailman_webui_basedir }}/static
#ErrorLog /var/log/httpd/webui_error.log
#CustomLog /var/log/httpd/webui_access.log combined
WSGIScriptAlias / {{ mailman_webui_confdir }}/webui.wsgi
WSGIDaemonProcess webui display-name=webui maximum-requests=1000 processes=4 threads=30
WSGISocketPrefix run/wsgi
WSGIRestrictStdout On
WSGIRestrictSignal Off
WSGIPythonOptimize 1
<Directory "{{ mailman_webui_confdir }}">
<Files webui.wsgi>
Order deny,allow
Allow from all
Require all granted
</Files>
WSGIProcessGroup webui
</Directory>
<Directory "{{ mailman_webui_basedir }}/static">
Order deny,allow
Allow from all
Require all granted
</Directory>
# Old static archives
Alias /pipermail/ {{ mailman_webui_basedir }}/old-archives/pipermail/
<Directory {{ mailman_webui_basedir }}/old-archives/pipermail>
Options FollowSymLinks
AllowOverride None
Require all granted
AddDefaultCharset Off
</Directory>
RedirectMatch ^/pipermail[/]*$ /
RedirectMatch ^/mailman/listinfo/$ /
RedirectMatch ^/mailman$ /

15
roles/mailman/templates/bottom.html
View file

@ -1,15 +0,0 @@
{% if env == 'staging' %}
<script src="https://apps.stg.fedoraproject.org/fedmenu/js/fedmenu.js"> </script>
{% else %}
<script src="https://apps.fedoraproject.org/fedmenu/js/fedmenu.js"> </script>
{% endif %}
<script>
fedmenu({
{% if env == 'staging' %}
'url': 'https://apps.stg.fedoraproject.org/js/data.js',
{% else %}
'url': 'https://apps.fedoraproject.org/js/data.js',
{% endif %}
'position': 'bottom-left'
});
</script>

1
roles/mailman/templates/crontab-mailman3.j2
View file

@ -1 +0,0 @@
42 * * * * mailman {{ mailman_webui_basedir }}/bin/periodic.py

26
roles/mailman/templates/crontab-webui.j2
View file

@ -1,26 +0,0 @@
# This goes in /etc/cron.d/.
# Replace "apache" by your webserver user ("www-data" on Debian systems) and
# set the path to the Django project directory
{% if env == 'staging' %}
MAILTO=""
{% endif %}
{% if ansible_hostname == 'mailman01' %}
@hourly apache django-admin runjobs hourly --pythonpath {{ mailman_webui_confdir }} --settings settings
@daily apache django-admin runjobs daily --pythonpath {{ mailman_webui_confdir }} --settings settings
@weekly apache django-admin runjobs weekly --pythonpath {{ mailman_webui_confdir }} --settings settings
@monthly apache django-admin runjobs monthly --pythonpath {{ mailman_webui_confdir }} --settings settings
@yearly apache django-admin runjobs yearly --pythonpath {{ mailman_webui_confdir }} --settings settings
2,17,32,47 * * * * apache django-admin runjobs quarter_hourly --pythonpath {{ mailman_webui_confdir }} --settings settings
* * * * * apache django-admin runjobs minutely --pythonpath {{ mailman_webui_confdir }} --settings settings
{% else %}
# These are only active on mailman01 (primary) server
#@hourly apache django-admin runjobs hourly --pythonpath {{ mailman_webui_confdir }} --settings settings
#@daily apache django-admin runjobs daily --pythonpath {{ mailman_webui_confdir }} --settings settings
#@weekly apache django-admin runjobs weekly --pythonpath {{ mailman_webui_confdir }} --settings settings
#@monthly apache django-admin runjobs monthly --pythonpath {{ mailman_webui_confdir }} --settings settings
#@yearly apache django-admin runjobs yearly --pythonpath {{ mailman_webui_confdir }} --settings settings
#2,17,32,47 * * * * apache django-admin runjobs quarter_hourly --pythonpath {{ mailman_webui_confdir }} --settings settings
#* * * * * apache django-admin runjobs minutely --pythonpath {{ mailman_webui_confdir }} --settings settings
{% endif %}

30
roles/mailman/templates/initial-data.json.j2
View file

@ -1,30 +0,0 @@
[
{% for host in mailman_domains %}
{
"fields": {
"domain": "{{ host }}",
"name": "Fedora mailing-lists"
},
"model": "sites.site",
"pk": {{ loop.index }}
},
{% endfor %}
{% for service_name, service_data in mailman_login.items() %}
{
"fields": {
"name": "{{ service_data.display_name }}",
"sites": [
{% for host in mailman_domains %}
{{ loop.index }}{% if not loop.last %},{% endif %}
{% endfor %}
],
"client_id": "{{ mailman_login_secrets[env][service_name].client_id }}",
"secret": "{{ mailman_login_secrets[env][service_name].secret }}",
"key": "",
"provider": "{{ service_data.provider }}"
},
"model": "socialaccount.socialapp",
"pk": {{ loop.index }}
}{% if not loop.last %},{% endif %}
{% endfor %}
]

3
roles/mailman/templates/mailman-migration-path.sh.j2
View file

@ -1,3 +0,0 @@
PATH=$PATH:{{ mailman_webui_basedir }}/bin
export PATH

2
roles/mailman/templates/mailman-migration.conf.j2
View file

@ -1,2 +0,0 @@
basedir: {{ mailman_webui_basedir }}
confdir: {{ mailman_webui_confdir }}

187
roles/mailman/templates/mailman.cfg.j2
View file

@ -1,187 +0,0 @@
# This is the absolute bare minimum base configuration file. User supplied
# configurations are pushed onto this.
[mailman]
# This address is the "site owner" address. Certain messages which must be
# delivered to a human, but which can't be delivered to a list owner (e.g. a
# bounce from a list owner), will be sent to this address. It should point to
# a human.
site_owner: admin@fedoraproject.org
# The local URL part to the administration interface (Postorius).
# The full URL will be constructed by prepending the domain URL set in the
# list's domain properties.
listinfo_url = /admin/
# Set the paths to be Fedora-compliant
layout: fhs
[paths.dev]
var_dir = {{ mailman_webui_basedir }}/var
[paths.fhs]
bin_dir: /usr/libexec/mailman3
var_dir: /var/lib/mailman3
queue_dir: /var/spool/mailman3
log_dir: /var/log/mailman3
lock_dir: /run/lock/mailman3
ext_dir: /etc/mailman3.d
pid_file: /run/mailman3/master.pid
[database]
class: mailman.database.postgresql.PostgreSQLDatabase
url: postgresql://mailmanadmin:{{ mailman_mailman_db_pass }}@{{ mailman_db_server }}/mailman
[archiver.hyperkitty]
class: mailman_hyperkitty.Archiver
enable: yes
configuration: /etc/mailman3.d/hyperkitty.cfg
[archiver.fedmsg]
class: mailman3_fedmsg_plugin.Archiver
enable: yes
[archiver.prototype]
enable: yes
[antispam]
# This section defines basic antispam detection settings.
# This value contains lines which specify RFC 822 headers in the email to
# check for spamminess. Each line contains a `key: value` pair, where the key
# is the header to check and the value is a Python regular expression to match
# against the header's value. E.g.:
#
# X-Spam: (yes|maybe)
#
# The header value and regular expression are always matched
# case-insensitively.
header_checks:
X-Spam: yes
X-Spam-Flag: Yes
X-Spam-Status: ^Yes,
# The chain to jump to if any of the header patterns matches. This must be
# the name of an existing chain such as 'discard', 'reject', 'hold', or
# 'accept', otherwise 'hold' will be used.
jump_chain: discard
[mta]
# Email is sent on the submission port to bypass spam checking.
smtp_port: 587
# Remove DKIM signatures in the email headers.
remove_dkim_headers: yes
# Don't process old stale bounces
verp_probes: yes
[language.en]
# Change the english language to be UTF-8 (it defaults to ascii).
description: English (USA)
charset: utf-8
enabled: yes
# http://www.lingoes.net/en/translator/langcode.htm
[language.pt]
description: Protuguese
charset: iso-8859-15
enabled: yes
[language.cs]
description: Czech
charset: utf-8
enabled: yes
[language.ca]
description: Catalan
charset: utf-8
enabled: yes
[language.ja]
description: Japanese
charset: utf-8
enabled: yes
[language.ar]
description: Arabic
charset: utf-8
enabled: yes
[language.nl]
description: Dutch
charset: utf-8
enabled: yes
[language.pl]
description: Polish
charset: utf-8
enabled: yes
[language.es]
description: Spanish
charset: utf-8
enabled: yes
[language.pt_BR]
description: Protuguese (Brazil)
charset: iso-8859-15
enabled: yes
[language.zh_CN]
description: Chinese (S)
charset: utf-8
enabled: yes
[language.zh_TW]
description: Chinese (T)
charset: utf-8
enabled: yes
[language.ru]
description: Russian
charset: utf-8
enabled: yes
[language.vi]
description: Vietnamese
charset: utf-8
enabled: yes
[language.it]
description: Italian
charset: utf-8
enabled: yes
[language.fr]
description: French
charset: utf-8
enabled: yes
[language.ro]
description: Romanian
charset: utf-8
enabled: yes
[language.de]
description: German
charset: utf-8
enabled: yes
[language.hu]
description: Hungarian
charset: utf-8
enabled: yes
[language.ko]
description: Korean
charset: utf-8
enabled: yes
[language.uk]
description: Ukrainian
charset: utf-8
enabled: yes

10
roles/mailman/templates/manage.py.j2
View file

@ -1,10 +0,0 @@
#!/usr/bin/env python
import os
import sys
if __name__ == "__main__":
sys.path.insert(0, "{{ mailman_webui_confdir }}")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
from django.core.management import execute_from_command_line
execute_from_command_line(sys.argv)

8
roles/mailman/templates/post-transaction.action.j2
View file

@ -1,8 +0,0 @@
# Run the post-update script
hyperkitty:install:{{ mailman_webui_basedir }}/bin/post-update.sh
hyperkitty:update:{{ mailman_webui_basedir }}/bin/post-update.sh
postorius:install:{{ mailman_webui_basedir }}/bin/post-update.sh
postorius:update:{{ mailman_webui_basedir }}/bin/post-update.sh
python-django-mailman3:install:{{ mailman_webui_basedir }}/bin/post-update.sh
python-django-mailman3:update:{{ mailman_webui_basedir }}/bin/post-update.sh

432
roles/mailman/templates/settings.py.j2
View file

@ -1,432 +0,0 @@
#-*- coding: utf-8 -*-
"""
Django settings for HyperKitty + Postorius
"""
import os
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
import django_fedora
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '{{ mailman_hyperkitty_cookie_key }}'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False
ADMINS = (
('HyperKitty Admin', 'abompard@fedoraproject.org'),
)
SERVER_EMAIL = 'root@fedoraproject.org'
DEFAULT_FROM_EMAIL = "admin@fedoraproject.org"
SITE_ID = 1
# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.8/ref/settings/#allowed-hosts
ALLOWED_HOSTS = [
{% for host in mailman_domains %}
"{{ host }}",
{% endfor %}
".fedoraproject.org",
"localhost", # Archiving API from Mailman
"127.0.0.1", # HAProxy ping
"{{ ansible_hostname }}", # Varnish ping
]
# Mailman API credentials
MAILMAN_REST_API_URL = 'http://localhost:8001'
MAILMAN_REST_API_USER = 'restadmin'
MAILMAN_REST_API_PASS = 'restpass'
MAILMAN_ARCHIVER_KEY = 'SecretArchiverAPIKey'
MAILMAN_ARCHIVER_FROM = ('127.0.0.1', '::1')
# Application definition
INSTALLED_APPS = (
# Uncomment the next line to enable the admin:
'django.contrib.admin',
# Uncomment the next line to enable admin documentation:
# 'django.contrib.admindocs',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.sites',
'django.contrib.messages',
'django.contrib.staticfiles',
'hyperkitty',
'rest_framework',
'django_gravatar',
{% if env != 'staging' %}
'paintstore',
{% endif %}
'compressor',
'haystack',
'django_extensions',
'postorius',
'django_mailman3',
'allauth',
'allauth.account',
'allauth.socialaccount',
'django_q',
'django_mailman3.lib.auth.fedora',
{% for service_name, service_data in mailman_login.items() %}
'allauth.socialaccount.providers.{{ service_data.provider }}',
{% endfor %}
'allauth.socialaccount.providers.openid',
)
{% if env == 'staging' %}
MIDDLEWARE = (
{% else %}
MIDDLEWARE_CLASSES = (
{% endif %}
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.middleware.locale.LocaleMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
{% if env == 'staging' %}
#'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
{% else %}
'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
{% endif %}
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'django.middleware.security.SecurityMiddleware',
#'hyperkitty.middleware.SSLRedirect',
'django_mailman3.middleware.TimezoneMiddleware',
'postorius.middleware.PostoriusMiddleware',
)
ROOT_URLCONF = 'urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [
'{{ mailman_webui_basedir }}/templates',
],
'OPTIONS': {
'loaders': [
# https://docs.djangoproject.com/en/1.8/ref/templates/api/#django.template.loaders.cached.Loader
('django.template.loaders.cached.Loader', [
'django.template.loaders.filesystem.Loader',
'django.template.loaders.app_directories.Loader',
]),
],
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.i18n',
'django.template.context_processors.media',
'django.template.context_processors.static',
'django.template.context_processors.tz',
'django.template.context_processors.csrf',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
'django_mailman3.context_processors.common',
'hyperkitty.context_processors.common',
'postorius.context_processors.postorius',
],
},
},
]
WSGI_APPLICATION = 'wsgi.application'
# Database
# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql_psycopg2',
'NAME': 'hyperkitty',
'USER': 'hyperkittyapp',
'PASSWORD': '{{ mailman_hyperkitty_db_pass }}',
'HOST': '{{ mailman_db_server }}',
'PORT': '', # Set to empty string for default.
}
}
# Security & production settings
# https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/
CSRF_COOKIE_SECURE = True
CSRF_COOKIE_HTTPONLY = True
SESSION_COOKIE_SECURE = True
SECURE_CONTENT_TYPE_NOSNIFF = True
SECURE_BROWSER_XSS_FILTER = True
X_FRAME_OPTIONS = 'DENY'
# We're behind a proxy, use the X-Forwarded-Host header
# See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host
USE_X_FORWARDED_HOST = True
# In the Fedora infra, requests are systematically redirected to HTTPS.
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_SCHEME', 'https')
#SECURE_SSL_REDIRECT = True
#SECURE_HSTS_SECONDS = 3600
#SECURE_HSTS_INCLUDE_SUBDOMAINS = True
# Internationalization
# https://docs.djangoproject.com/en/1.8/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'America/Chicago'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.8/howto/static-files/
# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/var/www/example.com/static/"
#STATIC_ROOT = ''
STATIC_ROOT = "{{ mailman_webui_basedir }}/static/"
# URL prefix for static files.
# Example: "http://example.com/static/", "http://static.example.com/"
STATIC_URL = '/static/'
# Additional locations of static files
STATICFILES_DIRS = (
# Put strings here, like "/home/html/static" or "C:/www/django/static".
# Always use forward slashes, even on Windows.
# Don't forget to use absolute paths, not relative paths.
"{{ mailman_webui_basedir }}/static-fedora/",
)
# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
'django.contrib.staticfiles.finders.FileSystemFinder',
'django.contrib.staticfiles.finders.AppDirectoriesFinder',
# 'django.contrib.staticfiles.finders.DefaultStorageFinder',
'compressor.finders.CompressorFinder',
)
# Sessions
# https://docs.djangoproject.com/en/1.8/topics/http/sessions/
SESSION_ENGINE = 'django.contrib.sessions.backends.cached_db'
# Compatibility with Bootstrap 3
from django.contrib.messages import constants as messages
MESSAGE_TAGS = {
messages.ERROR: 'danger'
}
#
# Authentication
#
LOGIN_URL = 'account_login'
LOGIN_REDIRECT_URL = 'hk_root'
LOGOUT_URL = 'account_logout'
AUTHENTICATION_BACKENDS = (
'django.contrib.auth.backends.ModelBackend',
'allauth.account.auth_backends.AuthenticationBackend',
)
# Django Allauth
ACCOUNT_AUTHENTICATION_METHOD = "username_email"
ACCOUNT_EMAIL_REQUIRED = True
ACCOUNT_EMAIL_VERIFICATION = "mandatory"
ACCOUNT_DEFAULT_HTTP_PROTOCOL = "https"
ACCOUNT_UNIQUE_EMAIL = True
# Disable local signup
ACCOUNT_ADAPTER = "django_fedora_nosignup.NoLocalSignUpAdapter"
SOCIALACCOUNT_ADAPTER = "django_fedora_nosignup.SignUpEnabledSocialAdapter"
SOCIALACCOUNT_PROVIDERS = {
'openid': {
'SERVERS': [
dict(id='yahoo',
name='Yahoo',
openid_url='http://me.yahoo.com'),
],
},
'google': {
'SCOPE': ['profile', 'email'],
'AUTH_PARAMS': {'access_type': 'online'},
},
'facebook': {
'METHOD': 'oauth2',
'SCOPE': ['email'],
'FIELDS': [
'email',
'name',
'first_name',
'last_name',
'locale',
'timezone',
],
'VERSION': 'v2.4',
},
'stackexchange': {
'SITE': 'stackoverflow',
},
}
#
# Gravatar
# https://github.com/twaddington/django-gravatar
#
# Gravatar base url.
GRAVATAR_URL = 'http://cdn.libravatar.org/'
# Gravatar base secure https url.
GRAVATAR_SECURE_URL = 'https://seccdn.libravatar.org/'
# Gravatar size in pixels.
#GRAVATAR_DEFAULT_SIZE = '80'
# An image url or one of the following: 'mm', 'identicon', 'monsterid', 'wavatar', 'retro'.
GRAVATAR_DEFAULT_IMAGE = 'retro'
# One of the following: 'g', 'pg', 'r', 'x'.
#GRAVATAR_DEFAULT_RATING = 'g'
# True to use https by default, False for plain http.
GRAVATAR_DEFAULT_SECURE = True
#
# django-compressor
# https://pypi.python.org/pypi/django_compressor
#
COMPRESS_PRECOMPILERS = (
('text/less', 'lessc {infile} {outfile}'),
('text/x-scss', 'sassc -t compressed {infile} {outfile}'),
('text/x-sass', 'sassc -t compressed {infile} {outfile}'),
)
COMPRESS_OFFLINE = True
# needed for debug mode
#INTERNAL_IPS = ('127.0.0.1',)
#
# Full-text search engine
#
HAYSTACK_CONNECTIONS = {
'default': {
'ENGINE': 'xapian_backend.XapianEngine',
'PATH': "{{ mailman_webui_basedir }}/fulltext_index",
},
}
#
# Asynchronous tasks
#
Q_CLUSTER = {
'timeout': 300,
'save_limit': 100000,
'orm': 'default',
}
#
# REST framework
#
REST_FRAMEWORK = {
'PAGE_SIZE': 10,
'DEFAULT_FILTER_BACKENDS': (
'rest_framework.filters.OrderingFilter',
),
}
# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'filters': {
'require_debug_false': {
'()': 'django.utils.log.RequireDebugFalse'
},
'exclude_useless_errors': {
'()': 'django.utils.log.CallbackFilter',
'callback': django_fedora.exclude_useless_errors,
}
},
'handlers': {
'mail_admins': {
'level': 'ERROR',
'filters': ['require_debug_false', 'exclude_useless_errors'],
'class': 'django.utils.log.AdminEmailHandler'
},
'file':{
'level': 'DEBUG',
#'class': 'logging.handlers.RotatingFileHandler',
'class': 'logging.handlers.WatchedFileHandler',
'filename': '/var/log/hyperkitty/hyperkitty.log',
'formatter': 'verbose',
},
'null': {
'class': 'logging.NullHandler',
},
},
'loggers': {
'django.request': {
'handlers': [
'file',
# Don't send an email on server errors, there's just too many
#{% if env == 'production' %}
#'mail_admins',
#{% endif %}
#
],
'level': 'DEBUG',
},
'django.security.DisallowedHost': {
'handlers': ['null'],
'propagate': False,
},
'requests.packages.urllib3': {
'level': 'WARNING',
},
},
'formatters': {
'verbose': {
'format': '%(levelname)s %(asctime)s %(process)d %(name)s %(message)s'
},
'simple': {
'format': '%(levelname)s %(message)s'
},
},
'root': {
'handlers': ['file'],
'level': 'INFO',
},
}
# Cache: use the local memcached server
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache',
'LOCATION': '127.0.0.1:11211',
}
}
# Only display mailing-lists from the same virtual host as the webserver
FILTER_VHOST = False

23
roles/mailman/templates/settings_admin.py.j2
View file

@ -1,23 +0,0 @@
#-*- coding: utf-8 -*-
"""
Copy of the Django settings file, but with database admin credentials (for
schema modifications)
"""
from settings import *
try:
from settings_local import *
except ImportError:
pass
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql_psycopg2',
'NAME': 'hyperkitty',
'USER': 'hyperkittyadmin',
'PASSWORD': '{{ mailman_hyperkitty_admin_db_pass }}',
'HOST': '{{ mailman_db_server }}',
'PORT': '',
}
}

11
roles/mailman/templates/webui-qcluster.service.j2
View file

@ -1,11 +0,0 @@
[Unit]
Description=Mailman web UI async tasks runner
After=network.target remote-fs.target
[Service]
ExecStart=/usr/bin/django-admin qcluster --pythonpath {{ mailman_webui_confdir }} --settings settings
User=apache
Restart=always
[Install]
WantedBy=multi-user.target

12
roles/mailman/templates/webui-warm-up-cache.service.j2
View file

@ -1,12 +0,0 @@
[Unit]
Description=Mailman web UI warm up cache
Requires=memcached.service
After=memcached.service
[Service]
Type=oneshot
ExecStart=/usr/bin/django-admin hyperkitty_warm_up_cache --pythonpath {{ mailman_webui_confdir }} --settings settings -v 2
User=apache
[Install]
WantedBy=multi-user.target