diff --git a/roles/mailman3/files/mailman-sar.py b/roles/mailman3/files/mailman-sar.py new file mode 100644 index 0000000000..980b6dcfcd --- /dev/null +++ b/roles/mailman3/files/mailman-sar.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +""" +GDPR SAR script for HyperKitty. + +Extract all emails from a selected address and prints them in JSON to the +standard output. +""" + +from __future__ import absolute_import, unicode_literals, print_function + +import argparse +import json +import logging +import os +import sys + +import requests +from six.moves.urllib.parse import urljoin + + +ENV_EMAIL = "SAR_EMAIL" +HYPERKITTY_INSTANCE = "http://localhost/archives/" +MAILMAN_INSTANCE = "http://localhost:8001/" +MAILMAN_AUTH = ("restadmin", "restpass") + +log = logging.getLogger() + + +def get_emails(address): + url = urljoin(HYPERKITTY_INSTANCE, "api/sender/{}/emails/".format(address)) + result = {"next": url} + count = None + email_urls = [] + while result.get("next"): + url = result["next"] + response = requests.get(url) + if response.status_code >= 300: + log.error("Could not get URL %s: %d %s", + url, response.status_code, response.reason) + break + result = response.json() + if count is None: + count = result["count"] + email_urls.extend([e["url"] for e in result["results"]]) + if count != len(email_urls): + log.error("Mismatch in the number of emails: got %s but there are " + "%s in total.", len(email_urls), count) + raise ValueError + emails = [] + for url in email_urls: + response = requests.get(url) + result = response.json() + emails.append(result) + return emails + + +def get_subscriptions(address): + url = urljoin(MAILMAN_INSTANCE, + "3.1/members/find?subscriber={}".format(address)) + response = requests.get(url, auth=MAILMAN_AUTH) + if response.status_code >= 300: + log.error("Could not get URL %s: %d %s", + url, response.status_code, response.reason) + return [] + result = response.json() + subscriptions = [] + for entry in result.get("entries", []): + subscription = { + "list_id": entry["list_id"], + "role": entry["role"], + "delivery_mode": entry["delivery_mode"], + } + # Get the subscription's preferences + member_id = entry["member_id"] + pref_url = urljoin(MAILMAN_INSTANCE, + "3.1/members/{}/preferences".format(member_id)) + pref_response = requests.get(pref_url, auth=MAILMAN_AUTH) + pref_result = pref_response.json() + if pref_response.status_code >= 300: + log.error("Could not get URL %s: %d %s", + pref_url, pref_response.status_code, + pref_response.reason) + else: + subscription["preferences"] = dict([ + (key, pref_result[key]) for key in pref_result + if key not in ("http_etag", "self_link") + ]) + subscriptions.append(subscription) + return subscriptions + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--debug", action="store_true") + return parser.parse_args() + + +def main(): + args = parse_args() + try: + email = os.environ[ENV_EMAIL] + except KeyError as e: + print("Missing environment variable. {}".format(e), file=sys.stderr) + sys.exit(1) + logging.basicConfig( + level=logging.DEBUG if args.debug else logging.WARNING, + stream=sys.stderr, + ) + emails = get_emails(email) + subscriptions = get_subscriptions(email) + print(json.dumps(dict( + emails=emails, + subscriptions=subscriptions, + ), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/roles/mailman3/tasks/main.yml b/roles/mailman3/tasks/main.yml index dbcff1f6cb..53382fb42c 100644 --- a/roles/mailman3/tasks/main.yml +++ b/roles/mailman3/tasks/main.yml @@ -39,6 +39,16 @@ notify: - restart mailman3 +- name: Install configuration for fedmsg-plugin + ansible.builtin.template: + src: fedmsg-plugin.toml.j2 + dest: "/etc/fedora-messaging/config.toml" + tags: + - config + - mailman + notify: + - restart mailman3 + # # Logging # @@ -306,8 +316,9 @@ tags: - config - mailman + - hyperkitty notify: - - reload apache + - reload mailmanweb # # Plug HyperKitty into Mailman @@ -383,6 +394,17 @@ mode: 0755 tags: mailman +- name: Install the scripts + ansible.builtin.copy: + src: {{ item }} + dest: "{{ mailman_webui_basedir }}/bin/{{ item }}" + owner: root + group: root + mode: 0755 + tags: mailman + with_items: + - mailman-sar.py + - name: Install the staging-sync script ansible.builtin.copy: src: prod-to-stg.py diff --git a/roles/mailman3/templates/fedmsg-plugin.toml.j2 b/roles/mailman3/templates/fedmsg-plugin.toml.j2 new file mode 100644 index 0000000000..d1a781b07b --- /dev/null +++ b/roles/mailman3/templates/fedmsg-plugin.toml.j2 @@ -0,0 +1,69 @@ +# A sample configuration for fedora-messaging. This file is in the TOML format. +amqp_url = "amqp://" +passive_declares = false +publish_exchange = "amq.topic" +topic_prefix = "" + +[tls] +ca_cert = "/etc/fedora-messaging/cacert.pem" +keyfile = "/etc/fedora-messaging/mailman3-key.pem" +certfile = "/etc/fedora-messaging/mailman3-cert.pem" + +[client_properties] +app = "Mailman3" + +[exchanges."amq.topic"] +type = "topic" +durable = true +auto_delete = false +arguments = {} + +[consumer_config] +# List-Ids that will not be relayed on the bus +excluded_lists = [ + 'scm-commits', # too much traffic + 'council-private', # private list + 'cwg-private', # private list + 'fesco', # private list + 'security-private', # private list + 'diversity-private', # private list +] +# URL of the HyperKitty instance +archive_base_url = "https://lists{{ env_suffix }}.fedoraproject.org/archives/" +# Domains where we can extract the username from the address +owned_domains = ["fedoraproject.org", "centos.org"] + +[log_config] +version = 1 +disable_existing_loggers = true + +[log_config.formatters.simple] +format = "[%(levelname)s %(name)s] %(message)s" + +[log_config.handlers.console] +class = "logging.StreamHandler" +formatter = "simple" +stream = "ext://sys.stdout" + +[log_config.loggers.fedora_messaging] +level = "INFO" +propagate = false +handlers = ["console"] + +# Twisted is the asynchronous framework that manages the TCP/TLS connection, as well +# as the consumer event loop. When debugging you may want to lower this log level. +[log_config.loggers.twisted] +level = "INFO" +propagate = false +handlers = ["console"] + +# Pika is the underlying AMQP client library. When debugging you may want to +# lower this log level. +[log_config.loggers.pika] +level = "WARNING" +propagate = false +handlers = ["console"] + +[log_config.root] +level = "ERROR" +handlers = ["console"]