From 93cf23a2aeab6e383820415b04085350a89870f7 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 30 Apr 2020 18:29:50 +0200 Subject: [PATCH] mirror_pagure_ansible: Add a role for this service Signed-off-by: Pierre-Yves Chibon --- playbooks/groups/batcave.yml | 10 ++ roles/mirror_pagure_ansible/tasks/main.yml | 115 ++++++++++++++++++ .../templates/mirror_from_pagure_bus.py | 115 ++++++++++++++++++ .../templates/mirror_pagure_ansible.cfg | 84 +++++++++++++ .../templates/mirror_pagure_ansible.service | 15 +++ 5 files changed, 339 insertions(+) create mode 100644 roles/mirror_pagure_ansible/tasks/main.yml create mode 100644 roles/mirror_pagure_ansible/templates/mirror_from_pagure_bus.py create mode 100644 roles/mirror_pagure_ansible/templates/mirror_pagure_ansible.cfg create mode 100644 roles/mirror_pagure_ansible/templates/mirror_pagure_ansible.service diff --git a/playbooks/groups/batcave.yml b/playbooks/groups/batcave.yml index fe0f450ec5..7eda27fbaf 100644 --- a/playbooks/groups/batcave.yml +++ b/playbooks/groups/batcave.yml @@ -30,6 +30,16 @@ certname: "{{wildcard_cert_name}}" SSLCertificateChainFile: "{{wildcard_int_file}}" - openvpn/client + - role: rabbit/user + username: "mirror_pagure_ansible{{ env_suffix }}" + - role: rabbit/queue + username: "mirror_pagure_ansible{{ env_suffix }}" + queue_name: "mirror_pagure_ansible{{ env_suffix }}" + routing_keys: + - "io.pagure.*.pagure.git.receive" + thresholds: + warning: 10 + critical: 100 - batcave - { role: nfs/client, when: inventory_hostname.startswith('batcave'), mnt_dir: '/srv/web/pub', nfs_src_dir: 'fedora_ftp/fedora.redhat.com/pub' } - { role: nfs/client, when: inventory_hostname.startswith('batcave01'), mnt_dir: '/mnt/fedora/app', nfs_src_dir: 'fedora_app/app' } diff --git a/roles/mirror_pagure_ansible/tasks/main.yml b/roles/mirror_pagure_ansible/tasks/main.yml new file mode 100644 index 0000000000..e13e02e983 --- /dev/null +++ b/roles/mirror_pagure_ansible/tasks/main.yml @@ -0,0 +1,115 @@ +- name: Install packages + package: state=present name={{ item }} + with_items: + - fedora-messaging + - git + tags: + - packages + - mirror_pagure_ansible + + +# Create the user the service will run under + +- name: Setup pagure user + user: + name: mirror_pagure_ansible + shell: /sbin/nologin + comment: "mirror_pagure_ansible User" + tags: + - mirror_pagure_ansible + - mirror_pagure_ansible/user + +# configure all the fedora-messaging files + +- name: Create /etc/pki/fedora-messaging + file: + dest: /etc/pki/fedora-messaging + mode: 0775 + owner: root + group: root + state: directory + tags: + - config + - mirror_pagure_ansible + +- name: deploy pagure/rabbitmq certificate + copy: src={{ item.src }} + dest=/etc/pki/fedora-messaging/{{ item.dest }} + owner={{ item.owner }} group={{ item.group}} mode={{ item.mode }} + with_items: + - src: "{{private}}/files/rabbitmq/production/pki/issued/mirror_pagure_ansible{{env_suffix}}.crt" + dest: mirror_pagure_ansible.crt + owner: mirror_pagure_ansible + group: mirror_pagure_ansible + mode: "0644" + - src: "{{private}}/files/rabbitmq/production/pki/private/mirror_pagure_ansible{{env_suffix}}.key" + dest: mirror_pagure_ansible.key + owner: mirror_pagure_ansible + group: mirror_pagure_ansible + mode: "0600" + - src: "{{private}}/files/rabbitmq/production/pki/ca.crt" + dest: cacert.pem + owner: mirror_pagure_ansible + group: mirror_pagure_ansible + mode: "0644" + tags: + - pagure + - fedora-messaging + +- name: Setup mirror_pagure_ansible fedora-messaging config + template: + src: mirror_pagure_ansible.cfg + dest: /etc/fedora-messaging/mirror_pagure_ansible.toml + owner: mirror_pagure_ansible + group: mirror_pagure_ansible + mode: 0640 + tags: + - config + - mirror_pagure_ansible + +# Install the script + +- name: Create /usr/local/libexec/mirror_pagure_ansible + file: + dest: /usr/local/libexec/mirror_pagure_ansible + mode: 0775 + owner: root + group: root + state: directory + tags: + - config + - mirror_pagure_ansible + +- name: Install the consumer + template: + src: mirror_from_pagure_bus.py + dest: /usr/local/libexec/mirror_pagure_ansible/mirror_from_pagure_bus.py + tags: + - packages + - mirror_pagure_ansible + + +# Install and start the service + +- name: Install the dedicated service file for mirror_pagure_ansible + template: + src: mirror_pagure_ansible.service + dest: /etc/systemd/system/mirror_pagure_ansible.service + owner: root + group: root + mode: 0755 + notify: + - reload systemd + tags: + - config + - mirror_pagure_ansible + +- name: Enable and started the service + service: + name: mirror_pagure_ansible.service + enabled: yes + state: started + tags: + - config + - mirror_pagure_ansible + diff --git a/roles/mirror_pagure_ansible/templates/mirror_from_pagure_bus.py b/roles/mirror_pagure_ansible/templates/mirror_from_pagure_bus.py new file mode 100644 index 0000000000..f545107349 --- /dev/null +++ b/roles/mirror_pagure_ansible/templates/mirror_from_pagure_bus.py @@ -0,0 +1,115 @@ +""" +This script runs in a loop and clone or update the clone of the ansible repo +hosted in pagure.io +""" +from __future__ import print_function + +import datetime +import logging +import os +import sched +import subprocess +import sys +import time + +from fedora_messaging import api, config + + +_log = logging.getLogger(__name__) + + + +def run_command(command, cwd=None): + """ Run the specified command in a specific working directory if one + is specified. + + :arg command: the command to run + :type command: list + :kwarg cwd: the working directory in which to run this command + :type cwd: str or None + """ + output = None + try: + output = subprocess.check_output(command, cwd=cwd, stderr=subprocess.PIPE) + except subprocess.CalledProcessError as e: + _log.error("Command `%s` return code: `%s`", " ".join(command), e.returncode) + _log.error("stdout:\n-------\n%s", e.stdout) + _log.error("stderr:\n-------\n%s", e.stderr) + raise + + return output + + +class MirrorFromPagure(object): + """ + A fedora-messaging consumer update a local mirror of a repo hosted on + pagure.io. + + Three configuration key is used from fedora-messaging's + "consumer_config" key: + - "mirror_folder", which indicates where mirrors should be store + - "urls", which is a list of mirrors to keep up to date + - "triggers_name", the fullname of the project (ie: name or namespace/name) + that we want to trigger a refresh of our clone on + + :: + + [consumer_config] + mirror_folder = "mirrors" + trigger_names = ["Fedora-Infra/ansible"] + urls = ["https://pagure.io/Fedora-Infra/ansible.git"] + """ + + def __init__(self): + """Perform some one-time initialization for the consumer.""" + self.path = config.conf["consumer_config"]["mirror_folder"] + self.urls = config.conf["consumer_config"]["urls"] + self.trigger_names = config.conf["consumer_config"]["trigger_names"] + + if not os.path.exists(self.path): + raise OSError("No folder %s found on disk" % self.path) + + def __call__(self, message, cnt=0): + """ + Invoked when a message is received by the consumer. + + Args: + message (fedora_messaging.api.Message): The message from AMQP. + """ + _log.info("Received topic: %s", message.topic) + if message.topic == "io.pagure.prod.pagure.git.receive": + repo_name = message.body.get("repo", {}).get("fullname") + if repo_name not in self.trigger_names: + _log.info("Not the pagure repo of interest, bailing") + return + elif message.topic == "org.fedoraproject.prod.infragit.receive": + pass + else: + _log.info("Unexpected topic received: %s", message.topic) + return + + try: + for url in self.urls: + _log.info("Syncing %s", url) + name = url.rsplit("/", 1)[-1] + + dest_folder = os.path.join(self.path, name) + if not os.path.exists(dest_folder): + _log.info(" Cloning as new %s", url) + cmd = ["git", "clone", "--mirror", url] + run_command(cmd, cwd=self.path) + + _log.info( + " Running git fetch with transfer.fsckObjects=1 in %s", + dest_folder + ) + cmd = ["git", "-c", "transfer.fsckObjects=1", "fetch"] + run_command(cmd, cwd=dest_folder) + + except Exception: + _log.exception("Something happened while calling git") + if cnt >= 3: + raise + _log.info(" Re-running in 10 seconds") + time.sleep(10) + self.__call__(message, cnt=cnt+1) diff --git a/roles/mirror_pagure_ansible/templates/mirror_pagure_ansible.cfg b/roles/mirror_pagure_ansible/templates/mirror_pagure_ansible.cfg new file mode 100644 index 0000000000..5d8cd2e482 --- /dev/null +++ b/roles/mirror_pagure_ansible/templates/mirror_pagure_ansible.cfg @@ -0,0 +1,84 @@ +amqp_url = "amqps://mirror_pagure_ansible{{ env_suffix }}:@rabbitmq{{ env_suffix }}.fedoraproject.org/%2Fpubsub" + +publish_exchange = "amq.topic" +passive_declares = true + +callback = "mirror_from_pagure_bus:MirrorFromPagure" + +# Don't use topic_prefix, since outgoing message topics are derived from incoming messages. +# topic_prefix = "" + +# Note the double brackets below. +# To add another binding, add another [[bindings]] section. +[[bindings]] +queue = "mirror_pagure_ansible{{ env_suffix }}" +exchange = "amq.topic" +routing_keys = [ + "io.pagure.*.pagure.git.receive", +] + +[tls] +ca_cert = "/etc/pki/fedora-messaging/cacert.pem" +keyfile = "/etc/pki/fedora-messaging/mirror_pagure_ansible-key.pem" +certfile = "/etc/pki/fedora-messaging/mirror_pagure_ansible-cert.pem" + +[client_properties] +app = "RoboSignatory" +app_url = "https://pagure.io/Fedora-Infra/mirror_from_pagure" +app_contacts_email = ["pingou@fedoraproject.org"] + +[queues."mirror_pagure_ansible{{ env_suffix }}"] +durable = true +auto_delete = false +exclusive = false +arguments = {} + +[consumer_config] +mirror_folder = "/srv/" +trigger_names = ["fedora-infrastructure"] +urls = [ + "https://pagure.io/fedora-infrastructure.git", +] + +[qos] +prefetch_size = 0 +prefetch_count = 25 + +[log_config] +version = 1 +disable_existing_loggers = true + +[log_config.formatters.simple] +format = "[%(levelname)s %(name)s] %(message)s" + +[log_config.handlers.console] +class = "logging.StreamHandler" +formatter = "simple" +stream = "ext://sys.stdout" + +[log_config.loggers.fedora_messaging] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.loggers.twisted] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.loggers.pika] +level = "WARNING" +propagate = false +handlers = ["console"] + +# If your consumer sets up a logger, you must add a configuration for it +# here in order for the messages to show up. e.g. if it set up a logger +# called 'example_printer', you could do: +[log_config.loggers.mirror_from_pagure_bus] +level = "DEBUG" +propagate = false +handlers = ["console"] + +[log_config.root] +level = "ERROR" +handlers = ["console"] diff --git a/roles/mirror_pagure_ansible/templates/mirror_pagure_ansible.service b/roles/mirror_pagure_ansible/templates/mirror_pagure_ansible.service new file mode 100644 index 0000000000..bd216152cf --- /dev/null +++ b/roles/mirror_pagure_ansible/templates/mirror_pagure_ansible.service @@ -0,0 +1,15 @@ +[Unit] +Description=Fedora Messaging consumer +Documentation=http://fedora-messaging.readthedocs.io/ + +[Service] +Type=simple +Environment="PYTHONPATH=/usr/local/libexec/mirror_pagure_ansible" +ExecStart=/usr/bin/fedora-messaging --conf /etc/fedora-messaging/mirror_pagure_ansible.toml consume +Restart=on-failure +User=mirror_pagure_ansible +Group=mirror_pagure_ansible + +[Install] +WantedBy=multi-user.target +