[mailman3] Create a new role mailman3

It seems that the mailman update is more complicated than I originally thought. So I
decided to start with clean role and will slowly start adding new changes.

Signed-off-by: Michal Konecny <mkonecny@redhat.com>

playbooks/groups/mailman.yml

@@ -100,6 +100,16 @@
     mailman_hyperkitty_admin_db_pass: "{{ mailman_hk_admin_db_pass }}"
     mailman_hyperkitty_db_pass: "{{ mailman_hk_db_pass }}"
     mailman_hyperkitty_cookie_key: "{{ mailman_hk_cookie_key }}"
+    when: env != "staging"
+  - role: mailman3
+    mailman_rest_api_user: "{{ mailman_stg_rest_api_user }}"
+    mailman_rest_api_pass: "{{ mailman_stg_rest_api_pass }}"
+    mailman_mailman_db_pass: "{{ mailman_mm_db_pass }}"
+    mailman_hyperkitty_admin_db_pass: "{{ mailman_hk_admin_db_pass }}"
+    mailman_hyperkitty_db_pass: "{{ mailman_hk_db_pass }}"
+    mailman_hyperkitty_cookie_key: "{{ mailman_hk_cookie_key }}"
+    mailman_hyperkitty_archiver_key: "{{ mailman_hk_stg_archiver_key }}"
+    when: env == "staging"
   - {role: fedmsg/base,
      when: env != "staging"}
   # Set up for fedora-messaging

roles/mailman3/defaults/main.yml

@@ -0,0 +1,36 @@
+---
+# Mailman core settings
+mailman_webui_basedir: /srv/webui
+mailman_webui_confdir: /etc/mailman3
+mailman_rest_api_user: restadmin
+mailman_rest_api_pass: changeme
+mailman_rest_api_workers: 2
+mailman_log_directory: /var/log/mailman3
+mailman_log_items:
+  - archiver
+  - bounce
+  - config
+  - database
+  - http
+  - runner
+  - smtp
+  - subscribe
+  - task
+
+# Mailman DB settings (the values should be provided by playbook)
+mailman_db_server: localhost
+mailman_mailman_db_pass: changeme
+
+# Hyperkitty/Postorius settings (the values should be provided by playbook)
+mailman_hyperkitty_admin_db_pass: changeme
+mailman_hyperkitty_db_pass: changeme
+mailman_hyperkitty_cookie_key: changeme
+mailman_hyperkitty_archiver_key: changeme
+mailman_domains:
+- lists.example.com
+- lists.example.org
+mailman_social_login: []
+
+# Apache configuration
+# Use mailman-web wsgi for mailman3
+wsgi_dir: /usr/lib/python3.9/site-packages/mailman_web

roles/mailman3/files/headers.html

@@ -0,0 +1,9 @@
+<style>
+.socialaccount_provider.fedora {
+    padding: 10px 16px;
+    font-size: 18px;
+    line-height: 1.33333;
+    border-radius: 6px;
+    border-color: #aaa;
+}
+</style>

roles/mailman3/files/login_extra_top.html

@@ -0,0 +1,6 @@
+<p>
+By subscribing or posting to these mailing lists, you agree that any email
+to the list is public and viewable worldwide (unless it's one of the few private mailing-lists).
+For more details, see our
+<a href="https://fedoraproject.org/wiki/Legal:PrivacyPolicy">Privacy policy</a>.
+</p>

roles/mailman3/files/mailman-hyperkitty.cfg

@@ -0,0 +1,20 @@
+# This is the mailman extension configuration file to enable HyperKitty as an
+# archiver. Remember to add the following lines in the mailman.cfg file:
+#
+# [archiver.hyperkitty]
+# class: mailman_hyperkitty.Archiver
+# enable: yes
+# configuration: /path/to/here/hyperkitty.cfg
+#
+
+[general]
+
+# This is your HyperKitty installation, preferably on the localhost. This
+# address will be used by Mailman to forward incoming emails to HyperKitty
+# for archiving. It does not need to be publicly available, in fact it's
+# better if it is not.
+base_url: http://localhost/archives/
+
+# Shared API key, must be the identical to the value in HyperKitty's
+# settings.
+api_key: {{ mailman_hypperkitty_archiver_key }}

roles/mailman3/files/mailman-template-list-admin-action-post.txt

@@ -0,0 +1,15 @@
+As list administrator, your authorization is requested for the
+following mailing list posting:
+
+    List:    $listname
+    From:    $sender_email
+    Subject: $subject
+
+The message is being held because:
+
+$reasons
+
+At your convenience, visit your dashboard to approve or deny the
+request:
+
+https://lists.fedoraproject.org/admin/lists/$list_id/held_messages

roles/mailman3/files/mailman-template-list-member-generic-footer.txt

@@ -0,0 +1,8 @@
+-- 
+_______________________________________________
+$display_name mailing list -- $listname
+To unsubscribe send an email to ${short_listname}-leave@${domain}
+Fedora Code of Conduct: https://docs.fedoraproject.org/en-US/project/code-of-conduct/
+List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines
+List Archives: https://${domain}/archives/list/${listname}
+Do not reply to spam, report it: https://pagure.io/fedora-infrastructure/new_issue

roles/mailman3/files/mailman-template-users-list-user-notice-hold.txt

@@ -0,0 +1,25 @@
+Your mail to '$listname' with the subject
+
+    $subject
+
+Is being held until the list moderator can review it for approval.
+
+The message is being held because:
+
+$reasons
+
+Either the message will get posted to the list, or you will receive
+notification of the moderator's decision.
+
+NOTE: If your message has been held due to size, please consider
+this a rejection and a friendly request to reduce the size and
+resend.  The list moderators will not approve such messages.
+
+If you are attaching logs or command output, please trim it to a
+minimal amount (we can ask for more context, if needed).
+
+If you are attaching an image, please crop it to reduce their size.
+
+Also keep in mind that attachments will be encoded by your mail
+client (usually via base64) and this encoding increases the size by
+as much as 33%.

roles/mailman3/files/mailman3.logrotate.conf

@@ -0,0 +1,6 @@
+/var/log/mailman3/*.log {
+    missingok
+    notifempty
+    delaycompress
+    su root apache
+}

roles/mailman3/files/memcached.sysconfig

@@ -0,0 +1,5 @@
+PORT="11211"
+USER="memcached"
+MAXCONN="1024"
+CACHESIZE="655350"
+OPTIONS=""

roles/mailman3/files/navbar-brand.html

@@ -0,0 +1,5 @@
+{% load static %}
+<a class="navbar-brand" href="{% url 'hk_root' %}"
+   title="{{ site_name|title }}" style="padding:8px 15px">
+    <img alt="{{ site_name|title }}" src="{% static 'logo-hyperkitty-fedora.png' %}" />
+</a>

roles/mailman3/files/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Disallow: /accounts/
+Disallow: /django-admin/
+Disallow: /archives/*/vote$

roles/mailman3/files/signup_closed.html

@@ -0,0 +1,21 @@
+{% extends "account/base.html" %}
+
+{% load i18n %}
+
+{% block head_title %}{% trans "Signup" %}{% endblock %}
+
+{% block content %}
+<h1>{% trans "Sign Up" %}</h1>
+
+<p>
+{% blocktrans %}
+Account creation for the Fedora mailing-lists is done by
+<a href="https://accounts.fedoraproject.org/">creating an account in FAS</a>.
+{% endblocktrans %}
+</p>
+
+{% url LOGIN_URL as login_url %}
+<p>{% blocktrans %}Already have an account? Then please <a href="{{login_url}}">sign in</a>.{% endblocktrans %}</p>
+
+
+{% endblock %}

roles/mailman3/files/top.html

@@ -0,0 +1,10 @@
+<!-- This markup will be inserted after the <body> tag -->
+
+<!-- https://pagure.io/fedora-infrastructure/issue/10741 -->
+<!--
+<div style="margin: auto" class="container">
+    <a href="https://fedoraproject.limequery.com/2023" target="_blank">
+        <img src="https://pagure.io/fedora-infrastructure/issue/raw/files/b4d328e7ce730f610530822af4c9cdab1029c47dc2bdb15648d4008e1b304e43-contributor-survey-banner.png" style="max-width: 100%">
+    </a>
+</div>
+-->

roles/mailman3/handlers/main.yml

@@ -0,0 +1,12 @@
+---
+- name: restart mailman3
+  service: name=mailman3 state=restarted
+
+- name: reload apache
+  service: name=httpd state=reloaded
+
+- name: restart memcached
+  service: name=memcached state=restarted
+
+- name: systemctl daemon-reload
+  command: /usr/bin/systemctl daemon-reload

roles/mailman3/tasks/main.yml

@@ -0,0 +1,452 @@
+---
+# Configuration for Mailman 3
+# PostgreSQL initialization must have been done already
+
+#
+# SELinux
+#
+- name: Set the SELinux policy for the fulltext index
+  community.general.sefcontext:
+    target: "{{ mailman_webui_basedir }}/fulltext_index(/.*)?"
+    setype: httpd_sys_rw_content_t
+    state: present
+  tags:
+    - mailman
+    - selinux
+
+- name: Set the SELinux policy for the static files directory
+  community.general.sefcontext:
+    target: "{{ mailman_webui_basedir }}/static(/.*)?"
+    setype: httpd_sys_content_t
+    state: present
+  tags:
+    - mailman
+    - selinux
+
+- name: Set the SELinux policy for the templates override directory
+  community.general.sefcontext:
+    target: "{{ mailman_webui_basedir }}/templates(/.*)?"
+    setype: httpd_sys_content_t
+    state: present
+  tags:
+    - mailman
+    - selinux
+
+- name: set the SELinux policy for the log directory
+  community.general.sefcontext:
+    target: "{{ mailman_log_directory }}(/.*)?"
+    setype: httpd_log_t
+    state: present
+  tags:
+    - mailman
+    - selinux
+
+- name: Set the SELinux policy for the generated postfix databases
+  community.general.sefcontext:
+    target: "{{ mailman_webui_basedir }}/var/data/postfix_.*"
+    setype: etc_aliases_t
+    state: present
+  tags:
+    - mailman
+    - selinux
+
+- name: Set the SELinux policy for the old static archives
+  community.general.sefcontext:
+    target: "{{ mailman_webui_basedir }}/old-archives/pipermail(/.*)?"
+    setype: httpd_sys_content_t
+    state: present
+  tags:
+    - mailman
+    - selinux
+
+- name: Allow Apache to remotely connect to PostgreSQL
+  ansible.posix.seboolean:
+    name: httpd_can_network_connect_db
+    state: yes
+    persistent: yes
+  tags:
+    - mailman
+    - selinux
+
+- name: Allow Apache to remotely connect to Mailman
+  ansible.posix.seboolean:
+    name: httpd_can_network_connect
+    state: yes
+    persistent: yes
+  tags:
+    - mailman
+    - selinux
+
+- name: Allow Apache to remotely connect to Memcached
+  ansible.posix.seboolean:
+    name: httpd_can_network_memcache
+    state: yes
+    persistent: yes
+  tags:
+    - mailman
+    - selinux
+
+
+- name: Install needed packages
+  ansible.builtin.package:
+    name: {{item}}
+    state: present
+  with_items:
+    - httpd
+    - hyperkitty
+    - fedora-messaging
+    - mailman3
+    - memcached
+    - postorius
+    - python3-mailman-hyperkitty
+    - python3-mailman-web
+    - python3-mod_wsgi
+    - python3-pylibmc
+    - python3-psycopg2
+    - python3-whoosh
+    - sassc
+  tags:
+    - packages
+    - mailman
+
+- name: Set the mailman conffile
+  ansible.builtin.template:
+    src: mailman.cfg.j2
+    dest: /etc/mailman.cfg
+    owner: mailman
+    group: mailman
+    mode: 0640
+  tags:
+    - config
+    - mailman
+  notify:
+    - restart mailman3
+
+#
+# Logging
+#
+- name: Ensuring log directory exists
+  ansible.builtin.file:
+    path: "{{ mailman3_log_directory }}"
+    state: directory
+    owner: mailman
+    group: mailman
+  tags:
+    - logging
+    - mailman
+
+- name: Mailman logging -- rotation
+  ansible.builtin.copy:
+    src: mailman3.logrotate.conf
+    dest: /etc/logrotate.d/mailman3
+  tags:
+    - logging
+    - mailman
+
+- name: Ensuring mailman-core is started
+  ansible.builtin.service:
+    name: mailman3
+    state: started
+    enabled: yes
+  tags:
+    - mailman
+
+#
+# HyperKitty + Postorius setup
+#
+- name: Create site/en dir
+  ansible.builtin.file:
+    state: directory
+    path: /var/lib/mailman3/templates/site/en
+  tags:
+    - config
+    - mailman
+
+- name: Set default list footer
+  ansible.builtin.copy:
+    src: mailman-template-list-member-generic-footer.txt
+    dest: /var/lib/mailman3/templates/site/en/list:member:generic:footer.txt
+    owner: mailman
+    group: mailman
+    mode: 0644
+  tags:
+    - config
+    - mailman
+
+- name: Set default list:admin:action:post template
+  ansible.builtin.copy:
+    src: mailman-template-list-admin-action-post.txt
+    dest: /var/lib/mailman3/templates/site/en/list:admin:action:post.txt
+    owner: mailman
+    group: mailman
+    mode: 0644
+  tags:
+    - config
+    - mailman
+
+- name: Create lists/users.lists.fedoraproject.org/en template dir
+  ansible.builtin.file:
+    state: directory
+    path: /var/lib/mailman3/templates/lists/users.lists.fedoraproject.org/en
+  tags:
+    - config
+    - mailman
+
+- name: Set list:user:notice:hold template for users list
+  anisble.builtin.copy:
+    src: mailman-template-users-list-user-notice-hold.txt
+    dest: /var/lib/mailman3/templates/lists/users.lists.fedoraproject.org/en/list:user:notice:hold.txt
+    owner: mailman
+    group: mailman
+    mode: 0644
+  tags:
+    - config
+    - mailman
+
+- name: Install the hyperkitty/postorius settings file
+  ansible.builtin.template:
+    src: settings.py.j2
+    dest: "{{ mailman_webui_confdir }}/settings.py"
+    owner: root
+    group: apache
+    mode: 0640
+  tags:
+    - config
+    - mailman
+  notify:
+    - reload apache
+
+- name: Install the hyperkitty/postorius httpd conf file
+  ansible.builtin.template:
+    src: apache.conf.j2
+    dest: /etc/httpd/conf.d/mailman-webui.conf
+  tags:
+    - config
+    - mailman
+  notify:
+    - reload apache
+
+- name: Create the fulltext index dir
+  ansible.builtin.file:
+    path: "{{ mailman_webui_basedir }}/fulltext_index"
+    state: directory
+    owner: apache
+    group: apache
+    mode: 0755
+  tags: mailman
+
+- name: Create the hyperkitty static files dirs
+  ansible.builtin.file:
+    path: "{{ mailman_webui_basedir }}/{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+  with_items:
+    - static
+    - static-fedora
+  register: static_root_dir
+  tags: mailman
+
+- name: Create the fedora-specific hyperkitty img dir
+  ansible.builtin.file:
+    path: "{{ mailman_webui_basedir }}/static-fedora/hyperkitty/img"
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+  tags:
+    - mailman
+    - hyperkitty
+
+- name: Create the hyperkitty templates override dirs
+  ansible.builtin.file:
+    path: "{{ mailman_webui_basedir }}/templates/{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+  with_items:
+    - hyperkitty
+    - django_mailman3
+  tags:
+    - mailman
+    - hypperkitty
+
+# Add the Nest banner
+# See issue https://pagure.io/fedora-infrastructure/issue/10103
+- name: Replace the top.html template in hyperkitty/postorius to change banner
+  ansible.builtin.copy:
+    src: top.html
+    dest: "{{ mailman_webui_basedir }}/templates/hyperkitty/top.html"
+  notify:
+    - reload apache
+  tags:
+    - mailman
+    - banner
+    - hyperkitty
+
+- name: Install our fedora-specific logo
+  ansible.builtin.copy:
+    src: logo-hyperkitty-fedora.png
+    dest: "{{ mailman_webui_basedir }}/static-fedora/logo-hyperkitty-fedora.png"
+  tags:
+    - mailman
+    - hyperkitty
+
+- name: Install our fedora-specific favicon
+  ansible.builtin.copy:
+    src: favicon.ico
+    dest: "{{ mailman_webui_basedir }}/static-fedora/favicon.ico"
+  tags:
+    - mailman
+    - hyperkitty
+
+- name: Install our fedora-specific robots.txt
+  ansible.builtin.copy:
+    src: robots.txt
+    dest: "{{ mailman_webui_basedir }}/static-fedora/robots.txt"
+  tags:
+    - mailman
+    - hyperkitty
+
+- name: Install our fedora-specific brand template override
+  ansible.builtin.copy:
+    src: navbar-brand.html
+    dest: "{{ mailman_webui_basedir }}/templates/hyperkitty/navbar-brand.html"
+  tags:
+    - mailman
+    - hyperkitty
+
+- name: Install our fedora-specific page headers
+  ansible.builtin.copy:
+    src: headers.html
+    dest: "{{ mailman_webui_basedir }}/templates/hyperkitty/headers.html"
+  tags:
+    - mailman
+    - hyperkitty
+
+- name: Install our fedora-specific login page extension
+  ansible.builtin.copy:
+    src: login_extra_top.html
+    dest: "{{ mailman_webui_basedir }}/templates/django_mailman3/login_extra_top.html"
+  tags:
+    - mailman
+    - hyperkitty
+
+- name: Create dirs for signup_closed page
+  ansible.builtin.file:
+    path: "{{ mailman_webui_basedir }}/templates/account"
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+  tags:
+    - mailman
+    - hyperkitty
+
+- name: Install our fedora-specific signup_closed page
+  ansible.builtin.copy:
+    src: signup_closed.html
+    dest: "{{ mailman_webui_basedir }}/templates/account/signup_closed.html"
+  tags:
+    - mailman
+    - hyperkitty
+
+- name: Install our fedmenu js hook
+  ansible.builtin.template:
+    src: bottom.html
+    dest: "{{mailman_webui_basedir }}/templates/hyperkitty/bottom.html"
+  tags:
+    - mailman
+    - hyperkitty
+
+
+- name: Ensuring we have correct DB schema for django apps
+  ansible.builtin.command:
+    cmd: mailman-web migrate
+    chdir: /etc/mailman3
+  become_user: mailman
+  tags:
+    - mailman
+    - db
+
+#
+# Plug HyperKitty into Mailman
+#
+- name: Copy the mailman-hyperkitty conffile
+  ansible.builtin.copy:
+    src: mailman-hyperkitty.cfg
+    dest: "/etc/mailman3.d/hyperkitty.cfg"
+    owner: mailman
+    group: mailman
+    mode: 0640
+  tags:
+    - config
+    - mailman
+    - hyperkitty
+  notify:
+    - restart mailman3
+
+- name: Populating static_root dir with django files/css
+  ansible.builtin.command:
+    cmd: mailman-web collectstatic --no-input
+    chdir: /etc/mailman3
+  become_user: mailman
+  when:
+    - static_root_dir is changed
+
+#
+# Scripts
+#
+
+- name: Install the staging-sync script
+  ansible.builtin.copy:
+    src: prod-to-stg.py
+    dest: "{{ mailman_webui_basedir }}/bin/prod-to-stg.py"
+  when: env == "staging"
+  tags: mailman
+
+#
+# Systemd
+#
+- name: Install the systemd service files
+  ansible.builtin.template:
+    src: "{{ item }}.service.j2"
+    dest: "/etc/systemd/system/{{ item }}.service"
+  with_items:
+    - webui-qcluster
+    - webui-warm-up-cache
+  notify:
+    - systemctl daemon-reload
+  tags:
+    - config
+    - mailman
+    - systemd
+
+# Memcached
+- name: Set the memcached sysconfig file
+  ansible.builtin.copy:
+    src: memcached.sysconfig
+    dest: /etc/sysconfig/memcached
+  notify:
+    - restart memcached
+  tags:
+    - mailman
+    - memcached
+
+# Start services
+- name: Start services
+  ansible.builtin.systemd:
+    state: started
+    enabled: yes
+    name: {{item}}
+  with_items:
+    - httpd
+    - mailman3
+    - postfix
+    - webui-qcluster
+    - webui-warm-up-cache
+  tags:
+    - mailman
+    - hyperkitty

roles/mailman3/templates/apache.conf.j2

@@ -0,0 +1,43 @@
+Alias /favicon.ico {{ mailman_webui_basedir }}/static/favicon.ico
+Alias /robots.txt  {{ mailman_webui_basedir }}/static/robots.txt
+Alias /static      {{ mailman_webui_basedir }}/static
+
+#ErrorLog /var/log/httpd/webui_error.log
+#CustomLog /var/log/httpd/webui_access.log combined
+
+WSGIScriptAlias / {{ wsgi_dir }}/wsgi.py
+WSGIDaemonProcess webui display-name=webui maximum-requests=1000 processes=4 threads=30 python-path={{ wsgi_dir }}
+WSGIProcessGroup webui
+WSGISocketPrefix run/wsgi
+WSGIRestrictStdout On
+WSGIRestrictSignal Off
+WSGIPythonOptimize 1
+
+<Directory "{{ wsgi_dir }}">
+    <Files wsgi.py>
+        Order deny,allow
+        Allow from all
+        Require all granted
+    </Files>
+</Directory>
+
+<Directory "{{ mailman_webui_basedir }}/static">
+    Order deny,allow
+    Allow from all
+    Require all granted
+</Directory>
+
+
+# Old static archives
+
+Alias /pipermail/ {{ mailman_webui_basedir }}/old-archives/pipermail/
+<Directory {{ mailman_webui_basedir }}/old-archives/pipermail>
+    Options FollowSymLinks
+    AllowOverride None
+    Require all granted
+    AddDefaultCharset Off
+</Directory>
+
+RedirectMatch ^/pipermail[/]*$ /
+RedirectMatch ^/mailman/listinfo/$ /
+RedirectMatch ^/mailman$ /

roles/mailman3/templates/bottom.html

@@ -0,0 +1,15 @@
+{% if env == 'staging' %}
+<script src="https://apps.stg.fedoraproject.org/fedmenu/js/fedmenu.js"> </script>
+{% else %}
+<script src="https://apps.fedoraproject.org/fedmenu/js/fedmenu.js"> </script>
+{% endif %}
+<script>
+ fedmenu({
+{% if env == 'staging' %}
+   'url': 'https://apps.stg.fedoraproject.org/js/data.js',
+{% else %}
+   'url': 'https://apps.fedoraproject.org/js/data.js',
+{% endif %}
+   'position': 'bottom-left'
+ });
+</script>

roles/mailman3/templates/mailman.cfg.j2

@@ -0,0 +1,191 @@
+# This is the absolute bare minimum base configuration file.  User supplied
+# configurations are pushed onto this.
+
+[mailman]
+# This address is the "site owner" address.  Certain messages which must be
+# delivered to a human, but which can't be delivered to a list owner (e.g. a
+# bounce from a list owner), will be sent to this address.  It should point to
+# a human.
+site_owner: admin@fedoraproject.org
+
+# The local URL part to the administration interface (Postorius).
+# The full URL will be constructed by prepending the domain URL set in the
+# list's domain properties.
+listinfo_url = /admin/
+
+# Set the paths to be Fedora-compliant
+layout: fhs
+
+[paths.fhs]
+bin_dir: /usr/libexec/mailman3
+var_dir: /var/lib/mailman3
+queue_dir: /var/spool/mailman3
+log_dir: /var/log/mailman3
+lock_dir: /run/lock/mailman3
+ext_dir: /etc/mailman3.d
+pid_file: /run/mailman3/master.pid
+
+[database]
+class: mailman.database.postgresql.PostgreSQLDatabase
+url: postgresql://mailmanadmin:{{ mailman_mailman_db_pass }}@{{ mailman_db_server }}/mailman
+
+[archiver.hyperkitty]
+class: mailman_hyperkitty.Archiver
+enable: yes
+configuration: /etc/mailman3.d/hyperkitty.cfg
+
+[webservice]
+hostname: localhost
+port: 8001
+admin_user: {{ mailman3_rest_api_user }}
+admin_pass: {{ mailman3_rest_api_pass }}
+workers: {{ mailman3_rest_api_workers }}
+
+{% for item in mailman3_log_items %}
+
+[loggin.{{ item }}]
+level: {{ mailman3_log_level }}
+path: {{ mailman3_log_directory }}/{{ item }}.log
+
+{% endfor %}
+
+[antispam]
+# This section defines basic antispam detection settings.
+
+# This value contains lines which specify RFC 822 headers in the email to
+# check for spamminess.  Each line contains a `key: value` pair, where the key
+# is the header to check and the value is a Python regular expression to match
+# against the header's value.  E.g.:
+#
+# X-Spam: (yes|maybe)
+#
+# The header value and regular expression are always matched
+# case-insensitively.
+header_checks:
+    X-Spam: yes
+    X-Spam-Flag: Yes
+    X-Spam-Status: ^Yes,
+
+# The chain to jump to if any of the header patterns matches.  This must be
+# the name of an existing chain such as 'discard', 'reject', 'hold', or
+# 'accept', otherwise 'hold' will be used.
+jump_chain: discard
+
+
+[mta]
+# Email is sent on the submission port to bypass spam checking.
+smtp_port: 587
+# Remove DKIM signatures in the email headers.
+remove_dkim_headers: yes
+# Don't process old stale bounces
+verp_probes: yes
+
+
+[language.en]
+# Change the english language to be UTF-8 (it defaults to ascii).
+description: English (USA)
+charset: utf-8
+enabled: yes
+
+
+# http://www.lingoes.net/en/translator/langcode.htm
+
+[language.pt]
+description: Protuguese
+charset: iso-8859-15
+enabled: yes
+
+[language.cs]
+description: Czech
+charset: utf-8
+enabled: yes
+
+[language.ca]
+description: Catalan
+charset: utf-8
+enabled: yes
+
+[language.ja]
+description: Japanese
+charset: utf-8
+enabled: yes
+
+[language.ar]
+description: Arabic
+charset: utf-8
+enabled: yes
+
+[language.nl]
+description: Dutch
+charset: utf-8
+enabled: yes
+
+[language.pl]
+description: Polish
+charset: utf-8
+enabled: yes
+
+[language.es]
+description: Spanish
+charset: utf-8
+enabled: yes
+
+[language.pt_BR]
+description: Protuguese (Brazil)
+charset: iso-8859-15
+enabled: yes
+
+[language.zh_CN]
+description: Chinese (S)
+charset: utf-8
+enabled: yes
+
+[language.zh_TW]
+description: Chinese (T)
+charset: utf-8
+enabled: yes
+
+[language.ru]
+description: Russian
+charset: utf-8
+enabled: yes
+
+[language.vi]
+description: Vietnamese
+charset: utf-8
+enabled: yes
+
+[language.it]
+description: Italian
+charset: utf-8
+enabled: yes
+
+[language.fr]
+description: French
+charset: utf-8
+enabled: yes
+
+[language.ro]
+description: Romanian
+charset: utf-8
+enabled: yes
+
+[language.de]
+description: German
+charset: utf-8
+enabled: yes
+
+[language.hu]
+description: Hungarian
+charset: utf-8
+enabled: yes
+
+[language.ko]
+description: Korean
+charset: utf-8
+enabled: yes
+
+[language.uk]
+description: Ukrainian
+charset: utf-8
+enabled: yes

roles/mailman3/templates/settings.py.j2

@@ -0,0 +1,406 @@
+#-*- coding: utf-8 -*-
+"""
+Django settings for HyperKitty + Postorius
+"""
+
+import os
+
+# Import mailman_web default settings
+from mailman_web.settings.base import *  # noqa: F403
+from mailman_web.settings.mailman import *  # noqa: F403
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = '{{ mailman_hyperkitty_cookie_key }}'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = False
+
+ADMINS = (
+     ('HyperKitty Admin', 'abompard@fedoraproject.org'),
+)
+
+SERVER_EMAIL = 'root@fedoraproject.org'
+DEFAULT_FROM_EMAIL = "admin@fedoraproject.org"
+
+SITE_ID = 1
+
+# Hosts/domain names that are valid for this site; required if DEBUG is False
+# See https://docs.djangoproject.com/en/1.8/ref/settings/#allowed-hosts
+ALLOWED_HOSTS = [
+{% for host in mailman_domains %}
+    "{{ host }}",
+{% endfor %}
+    ".fedoraproject.org",
+    "localhost", # Archiving API from Mailman
+    "127.0.0.1", # HAProxy ping
+    "{{ ansible_hostname }}", # Varnish ping
+]
+
+# Mailman API credentials
+MAILMAN_REST_API_URL = 'http://localhost:8001'
+MAILMAN_REST_API_USER = '{{ mailman_rest_api_user }}'
+MAILMAN_REST_API_PASS = '{{ mailman_rest_api_pass }}'
+MAILMAN_ARCHIVER_KEY = '{{ mailman_hyperkitty_archiver_key }}'
+MAILMAN_ARCHIVER_FROM = ('127.0.0.1', '::1')
+
+ROOT_URLCONF = 'mailman_web.urls'
+
+# Application definition
+
+INSTALLED_APPS = (
+    'allauth',
+    'allauth.account',
+    'allauth.socialaccount',
+    'compressor',
+    # Uncomment the next line to enable the admin:
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.sites',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'django.contrib.humanize',
+    'django_extensions',
+    'django_gravatar',
+    'django_mailman3',
+    'django_mailman3.lib.auth.fedora',
+    'django_q',
+    'hyperkitty',
+    'postorius',
+    {% for service_name, service_data in mailman_login.items() %}
+    'allauth.socialaccount.providers.{{ service_data.provider }}',
+    {% endfor %}
+    'allauth.socialaccount.providers.openid',
+)
+
+
+MIDDLEWARE = (
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django_mailman3.middleware.TimezoneMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.middleware.locale.LocaleMiddleware',
+    'django.middleware.security.SecurityMiddleware',
+    'postorius.middleware.PostoriusMiddleware',
+)
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [
+            '{{ mailman_webui_basedir }}/templates',
+        ],
+        'OPTIONS': {
+            'loaders': [
+                # https://docs.djangoproject.com/en/1.8/ref/templates/api/#django.template.loaders.cached.Loader
+                ('django.template.loaders.cached.Loader', [
+                    'django.template.loaders.filesystem.Loader',
+                    'django.template.loaders.app_directories.Loader',
+                ]),
+            ],
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.i18n',
+                'django.template.context_processors.media',
+                'django.template.context_processors.static',
+                'django.template.context_processors.tz',
+                'django.template.context_processors.csrf',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+                'django_mailman3.context_processors.common',
+                'hyperkitty.context_processors.common',
+                'postorius.context_processors.postorius',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': 'hyperkitty',
+        'USER': 'hyperkittyapp',
+        'PASSWORD': '{{ mailman_hyperkitty_db_pass }}',
+        'HOST': '{{ mailman_db_server }}',
+        'PORT': '',                      # Set to empty string for default.
+    }
+}
+
+
+
+# Security & production settings
+# https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/
+
+CSRF_COOKIE_SECURE = True
+CSRF_COOKIE_HTTPONLY = True
+SESSION_COOKIE_SECURE = True
+SECURE_CONTENT_TYPE_NOSNIFF = True
+SECURE_BROWSER_XSS_FILTER = True
+X_FRAME_OPTIONS = 'DENY'
+# We're behind a proxy, use the X-Forwarded-Host header
+# See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host
+USE_X_FORWARDED_HOST = True
+# In the Fedora infra, requests are systematically redirected to HTTPS.
+SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_SCHEME', 'https')
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/1.8/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'America/Chicago'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/1.8/howto/static-files/
+
+# Absolute path to the directory static files should be collected to.
+# Don't put anything in this directory yourself; store your static files
+# in apps' "static/" subdirectories and in STATICFILES_DIRS.
+# Example: "/var/www/example.com/static/"
+STATIC_ROOT = "{{ mailman_webui_basedir }}/static/"
+
+# URL prefix for static files.
+# Example: "http://example.com/static/", "http://static.example.com/"
+STATIC_URL = '/static/'
+
+# Additional locations of static files
+STATICFILES_DIRS = (
+    # Put strings here, like "/home/html/static" or "C:/www/django/static".
+    # Always use forward slashes, even on Windows.
+    # Don't forget to use absolute paths, not relative paths.
+    "{{ mailman_webui_basedir }}/static-fedora/",
+)
+
+# List of finder classes that know how to find static files in
+# various locations.
+STATICFILES_FINDERS = (
+    'django.contrib.staticfiles.finders.FileSystemFinder',
+    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+    'compressor.finders.CompressorFinder',
+)
+
+
+# Compatibility with Bootstrap 3
+from django.contrib.messages import constants as messages
+MESSAGE_TAGS = {
+    messages.ERROR: 'danger'
+}
+
+
+#
+# Authentication
+#
+
+LOGIN_URL          = 'account_login'
+LOGIN_REDIRECT_URL = 'hk_root'
+LOGOUT_URL         = 'account_logout'
+
+AUTHENTICATION_BACKENDS = (
+    'django.contrib.auth.backends.ModelBackend',
+    'allauth.account.auth_backends.AuthenticationBackend',
+)
+
+# Django Allauth
+ACCOUNT_AUTHENTICATION_METHOD = "username_email"
+ACCOUNT_EMAIL_REQUIRED = True
+ACCOUNT_EMAIL_VERIFICATION = "mandatory"
+ACCOUNT_DEFAULT_HTTP_PROTOCOL = "https"
+ACCOUNT_UNIQUE_EMAIL  = True
+# Disable local signup
+ACCOUNT_ADAPTER = "django_fedora_nosignup.NoLocalSignUpAdapter"
+SOCIALACCOUNT_ADAPTER = "django_fedora_nosignup.SignUpEnabledSocialAdapter"
+
+SOCIALACCOUNT_PROVIDERS = {
+    'openid': {
+        'SERVERS': [
+            dict(id='yahoo',
+                 name='Yahoo',
+                 openid_url='http://me.yahoo.com'),
+        ],
+    },
+    'google': {
+        'SCOPE': ['profile', 'email'],
+        'AUTH_PARAMS': {'access_type': 'online'},
+    },
+    'facebook': {
+       'METHOD': 'oauth2',
+       'SCOPE': ['email'],
+       'FIELDS': [
+           'email',
+           'name',
+           'first_name',
+           'last_name',
+           'locale',
+           'timezone',
+           ],
+       'VERSION': 'v2.4',
+    },
+    'stackexchange': {
+        'SITE': 'stackoverflow',
+    },
+}
+
+
+#
+# Gravatar
+# https://github.com/twaddington/django-gravatar
+#
+# Gravatar base url.
+GRAVATAR_URL = 'http://cdn.libravatar.org/'
+# Gravatar base secure https url.
+GRAVATAR_SECURE_URL = 'https://seccdn.libravatar.org/'
+# Gravatar size in pixels.
+#GRAVATAR_DEFAULT_SIZE = '80'
+# An image url or one of the following: 'mm', 'identicon', 'monsterid', 'wavatar', 'retro'.
+GRAVATAR_DEFAULT_IMAGE = 'retro'
+# One of the following: 'g', 'pg', 'r', 'x'.
+#GRAVATAR_DEFAULT_RATING = 'g'
+# True to use https by default, False for plain http.
+GRAVATAR_DEFAULT_SECURE = True
+
+#
+# django-compressor
+# https://pypi.python.org/pypi/django_compressor
+#
+COMPRESS_PRECOMPILERS = (
+   ('text/less', 'lessc {infile} {outfile}'),
+   ('text/x-scss', 'sassc -t compressed {infile} {outfile}'),
+   ('text/x-sass', 'sassc -t compressed {infile} {outfile}'),
+)
+COMPRESS_OFFLINE = True
+# needed for debug mode
+#INTERNAL_IPS = ('127.0.0.1',)
+
+
+#
+# Full-text search engine
+#
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
+        'PATH': "{{ mailman_webui_basedir }}/fulltext_index",
+    },
+}
+
+
+#
+# Asynchronous tasks
+#
+Q_CLUSTER = {
+    'timeout': 300,
+    'save_limit': 100000,
+    'orm': 'default',
+}
+
+
+#
+# REST framework
+#
+REST_FRAMEWORK = {
+    'PAGE_SIZE': 10,
+    'DEFAULT_FILTER_BACKENDS': (
+        'rest_framework.filters.OrderingFilter',
+    ),
+}
+
+
+# A sample logging configuration. The only tangible logging
+# performed by this configuration is to send an email to
+# the site admins on every HTTP 500 error when DEBUG=False.
+# See http://docs.djangoproject.com/en/dev/topics/logging for
+# more details on how to customize your logging configuration.
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'filters': {
+        'require_debug_false': {
+            '()': 'django.utils.log.RequireDebugFalse'
+        },
+        'exclude_useless_errors': {
+            '()': 'django.utils.log.CallbackFilter',
+            'callback': django_fedora.exclude_useless_errors,
+        }
+    },
+    'handlers': {
+        'mail_admins': {
+            'level': 'ERROR',
+            'filters': ['require_debug_false', 'exclude_useless_errors'],
+            'class': 'django.utils.log.AdminEmailHandler'
+        },
+        'file':{
+            'level': 'DEBUG',
+            #'class': 'logging.handlers.RotatingFileHandler',
+            'class': 'logging.handlers.WatchedFileHandler',
+            'filename': '/var/log/hyperkitty/hyperkitty.log',
+            'formatter': 'verbose',
+        },
+        'null': {
+            'class': 'logging.NullHandler',
+        },
+    },
+    'loggers': {
+        'django.request': {
+            'handlers': [
+                'file',
+                # Don't send an email on server errors, there's just too many
+                #{% if env == 'production' %}
+                #'mail_admins',
+                #{% endif %}
+                #
+
+            ],
+            'level': 'DEBUG',
+        },
+        'django.security.DisallowedHost': {
+            'handlers': ['null'],
+            'propagate': False,
+        },
+        'requests.packages.urllib3': {
+            'level': 'WARNING',
+        },
+    },
+    'formatters': {
+        'verbose': {
+            'format': '%(levelname)s %(asctime)s %(process)d %(name)s %(message)s'
+        },
+        'simple': {
+            'format': '%(levelname)s %(message)s'
+        },
+    },
+    'root': {
+        'handlers': ['file'],
+        'level': 'INFO',
+    },
+}
+
+
+# Cache: use the local memcached server
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache',
+        'LOCATION': '127.0.0.1:11211',
+    }
+}
+
+
+# Only display mailing-lists from the same virtual host as the webserver
+FILTER_VHOST = False

roles/mailman3/templates/webui-qcluster.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=Mailman web UI async tasks runner
+After=network.target remote-fs.target
+
+[Service]
+ExecStart=/usr/bin/mailman-web qcluster
+ExecStop=/bin/kill -s TERM $MAINPID
+User=mailman
+Group=mailman
+WorkingDirectory=/etc/mailman3
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

roles/mailman3/templates/webui-warm-up-cache.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=Mailman web UI warm up cache
+Requires=memcached.service
+After=memcached.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/mailman-web hyperkitty_warm_up_cache
+User=mailman
+Group=mailman
+WorkingDirectory=/etc/mailman3
+
+[Install]
+WantedBy=multi-user.target