bodhi/backend: Port the koji-sync script from fedmsg to fedora-messaging

Fixes https://pagure.io/fedora-infrastructure/issue/8967

Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>

playbooks/groups/bodhi-backend.yml

@@ -84,20 +84,25 @@
     service: bodhi
     host: "bodhi.fedoraproject.org"
     when: env == "production"
+
   - role: keytab/service
     owner_user: apache
     owner_group: apache
     service: bodhi
     host: "bodhi.stg.fedoraproject.org"
     when: env == "staging"
+
   - bodhi2/backend
+
   - role: push-container-registry
     cert_dest_dir: "/etc/docker/certs.d/registry{{ env_suffix }}.fedoraproject.org"
     cert_src: "{{private}}/files/docker-registry/{{env}}/pki/issued/containerstable.crt"
     key_src: "{{private}}/files/docker-registry/{{env}}/pki/private/containerstable.key"
     certs_group: apache
+
   - role: rabbit/user
     username: "bodhi{{ env_suffix }}"
+
   - role: rabbit/queue
     username: "bodhi{{ env_suffix }}"
     queue_name: "{{ bodhi_message_queue_name }}"
@@ -106,6 +111,17 @@
       warning: 10
       critical: 100
 
+  - role: rabbit/user
+    username: "koji_sync_listener{{ env_suffix }}"
+
+  - role: rabbit/queue
+    username: "koji_sync_listener{{ env_suffix }}"
+    queue_name: "koji_sync_listener{{ env_suffix }}"
+    routing_keys:
+      - "io.pagure.*.pagure.issue.edit"
+    thresholds:
+      warning: 10
+      critical: 100
 
   tasks:
   - name: create secondary volume dir for stg bodhi

roles/bodhi2/backend/files/koji-sync-listener.py

@@ -1,74 +0,0 @@
-#!/usr/bin/env python
-""" This is a glue script to run /usr/local/bin/owner-sync-pagure on a given
-package anytime a ticket gets closed at
-https://pagure.io/releng/fedora-scm-requests
-
-Author: Ralph Bean <rbean@redhat.com>
-"""
-
-import json
-import logging.config
-import subprocess as sp
-import sys
-
-import fedmsg
-import fedmsg.meta
-
-
-def handle(content):
-    body = content.strip('`').strip()
-    body = json.loads(body)
-    package = body['repo']
-    print("Operating on {package}".format(package=package))
-    sys.stdout.flush()
-    # XXX If you modify this taglist.  Please also modify the other copy in
-    # bodhi2/backend/tasks/main.yml
-    taglist = 'f33 f33-container f33-modular f33-flatpak f32 f32-container f32-modular f32-flatpak f31 f31-container f31-flatpak f31-modular epel8 epel8-playground epel8-modular epel7 dist-6E-epel module-package-list modular'
-    cmd = [
-        '/usr/local/bin/owner-sync-pagure',
-        '--package', package,
-        '--verbose',
-    ] + taglist.split()
-    print("Running %r" % cmd)
-    proc = sp.Popen(cmd)
-    status = proc.wait()
-    if status:
-        raise RuntimeError("%r gave return code %r" % (cmd, status))
-
-
-def main(fullname, fields, content):
-    if fullname != 'releng/fedora-scm-requests':
-        print("Dropping %r.  Not scm request." % fullname)
-        return False
-    if 'close_status' not in fields:
-        print("Dropping %r %r.  Not closed." % (fullname, fields))
-        return False
-
-    handle(content)
-
-
-if __name__ == '__main__':
-    config = fedmsg.config.load_config()
-    logging.config.dictConfig(config['logging'])
-    fedmsg.meta.make_processors(**config)
-    topic = 'io.pagure.prod.pagure.issue.edit'
-    for _, _, topic, msg in fedmsg.tail_messages(topic=topic):
-        # Extract some useful information for debugging
-        title, subtitle, link, idx = [None] * 4
-        try:
-            title = fedmsg.meta.msg2title(msg, **config)
-            subtitle = fedmsg.meta.msg2subtitle(msg, **config)
-            link = fedmsg.meta.msg2link(msg, **config)
-            idx = msg.get('msg_id')
-        except Exception as e:
-            print("!! Failed to determine title, subtitle, link")
-        print("Inspecting {title}, {subtitle}, {link}, {idx}".format(
-            title=title, subtitle=subtitle, link=link, idx=idx))
-
-        # Extract values we need to actually process the message
-        fullname = msg['msg']['project']['fullname']
-        fields = msg['msg']['fields']
-        content = msg['msg']['issue']['content']
-
-        # Do the work.
-        main(fullname, fields, content)

roles/bodhi2/backend/files/koji-sync-listener.service

@@ -1,15 +0,0 @@
-[Unit]
-Description=Sync the package list to koji tags as new repo tickets close
-After=syslog.target network.target
-
-[Service]
-ExecStart=/usr/local/bin/koji-sync-listener.py
-User=fedmsg
-Group=fedmsg
-Restart=on-failure
-
-# Use a different ccname so we don't share tickets with bodhi.
-Environment=KRB5CCNAME=/var/tmp/owner-sync-krbcc
-
-[Install]
-WantedBy=multi-user.target

roles/bodhi2/backend/files/koji_sync_listener.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+""" This is a glue script to run /usr/local/bin/owner-sync-pagure on a given
+package anytime a ticket gets closed at
+https://pagure.io/releng/fedora-scm-requests
+
+Author: Ralph Bean <rbean@redhat.com>
+        Pierre-Yves Chibon <pingou@pingoured.fr>
+"""
+
+import logging
+import json
+import subprocess as sp
+import sys
+
+from fedora_messaging import config
+
+_log = logging.getLogger("koji_sync_listener")
+
+
+class KojiSyncListener(object):
+    """
+    A fedora-messaging consumer that calls the owner-sync-pagure script
+    upon notifications coming from pagure.
+
+    A single configuration key is used from fedora-messaging's
+    "consumer_config" key, "path", which is where the consumer will save
+    the messages::
+
+        [consumer_config]
+        path = "/tmp/fedora-messaging/messages.txt"
+    """
+    def __init__(self):
+        """Perform some one-time initialization for the consumer."""
+        self.taglist = config.conf["consumer_config"]["taglist"]
+
+    def __call__(self, message):
+        """
+        Invoked when a message is received by the consumer.
+
+        Args:
+            message (fedora_messaging.api.Message): The message from AMQP.
+        """
+        body = message.body
+
+        fullname = body.get("project", {}).get("fullname")
+        fields = body.get("fields")
+        content = body.get('issue', {}).get('content')
+
+        if fullname != 'releng/fedora-scm-requests':
+            _log.info("Dropping %r.  Not scm request." % fullname)
+            return False
+        if 'close_status' not in fields:
+            _log.info("Dropping %r %r.  Not closed." % (fullname, fields))
+            return False
+
+        try:
+            body = content.strip('`').strip()
+            body = json.loads(body)
+        except Exception:
+            _log.info("Failed to decode JSON in the issue's initial comment")
+            return False
+
+        package = content['repo']
+        _log.info("Operating on {package}".format(package=package))
+        sys.stdout.flush()
+
+        cmd = [
+            '/usr/local/bin/owner-sync-pagure',
+            '--package', package,
+            '--verbose',
+        ] + self.taglist
+
+        _log.info("Running %r" % cmd)
+        proc = sp.Popen(cmd)
+        status = proc.wait()
+        if status:
+            raise RuntimeError("%r gave return code %r" % (cmd, status))
+
+        _log.info("Done.")

roles/bodhi2/backend/tasks/main.yml

@@ -18,6 +18,7 @@
       - bodhi-composer
       - python3-pyramid_sawing
       - sigul
+      - fedora-messaging
       # This next one is not strictly needed since bodhi-composer depends on bodhi-server which
       # depends on python3-celery, but since we're depoying a service file calling celery in this
       # role I think it makes sense to require it here instead of assuming a dependency chain.
@@ -83,20 +84,19 @@
 - name: put the koji sync listener script in place
   copy:
       src: koji-sync-listener.py
-      dest: /usr/local/bin/koji-sync-listener.py
+      dest: /usr/lib/python3.8/site-packages/koji_sync_listener.py
-      mode: 0755
+      mode: 0644
   when: env == "production"
   notify:
-  - restart koji-sync-listener
+  - restart fedora-messaging@koji_sync_listener
   tags:
   - bodhi
   - koji-sync
 
-- name: put the koji sync listener systemd file in place.
+- name: put the koji sync listener config file in place
-  copy:
+  template:
-      src: koji-sync-listener.service
+      src: koji_sync_listener.toml
-      dest: /usr/lib/systemd/system/koji-sync-listener.service
+      dest: /etc/fedora-messaging/koji_sync_listener.toml
-      mode: 0755
   when: env == "production"
   notify:
   - reload systemd
@@ -106,7 +106,7 @@
   - koji-sync
 
 - name: start the koji sync listener
-  service: name=koji-sync-listener enabled=yes state=started
+  service: name=fedora-messaging@koji_sync_listener enabled=yes state=started
   when: env == "production"
   tags:
   - bodhi

roles/bodhi2/backend/templates/koji-sync-listener.toml

@@ -0,0 +1,101 @@
+amqp_url = "amqps://koji_sync_listener{{ env_suffix }}:@rabbitmq{{ env_suffix }}.fedoraproject.org/%2Fpubsub"
+
+publish_exchange = "amq.topic"
+passive_declares = true
+
+callback = "koji_sync_listener:KojiSyncListener"
+
+# Don't use topic_prefix, since outgoing message topics are derived from incoming messages.
+# topic_prefix = ""
+
+[[bindings]]
+queue = "koji_sync_listener{{ env_suffix }}"
+
+exchange = "amq.topic"
+routing_keys = [
+    "io.pagure.*.pagure.issue.edit",
+]
+
+[tls]
+ca_cert = "/etc/pki/fedora-messaging/cacert.pem"
+keyfile = "/etc/pki/fedora-messaging/bodhi-key.pem"
+certfile = "/etc/pki/fedora-messaging/bodhi-cert.pem"
+
+[client_properties]
+app = "koji_sync_listener"
+app_contacts_email = ["pingou@fedoraproject.org"]
+
+[queues."koji_sync_listener{{ env_suffix }}"]
+durable = true
+auto_delete = false
+exclusive = false
+arguments = {}
+
+[consumer_config]
+
+# XXX If you modify this taglist.  Please also modify the other copy in
+# bodhi2/backend/tasks/main.yml
+taglist = [
+    "f33",
+    "f33-container",
+    "f33-modular",
+    "f33-flatpak",
+    "f32",
+    "f32-container",
+    "f32-modular",
+    "f32-flatpak",
+    "f31",
+    "f31-container",
+    "f31-flatpak",
+    "f31-modular",
+    "epel8",
+    "epel8-playground",
+    "epel8-modular",
+    "epel7",
+    "dist-6E-epel",
+    "module-package-list",
+    "modular",
+]
+
+[qos]
+prefetch_size = 0
+prefetch_count = 25
+
+[log_config]
+version = 1
+disable_existing_loggers = true
+
+[log_config.formatters.simple]
+format = "[%(levelname)s %(name)s] %(message)s"
+
+[log_config.handlers.console]
+class = "logging.StreamHandler"
+formatter = "simple"
+stream = "ext://sys.stdout"
+
+[log_config.loggers.fedora_messaging]
+level = "INFO"
+propagate = false
+handlers = ["console"]
+
+[log_config.loggers.twisted]
+level = "INFO"
+propagate = false
+handlers = ["console"]
+
+[log_config.loggers.pika]
+level = "WARNING"
+propagate = false
+handlers = ["console"]
+
+# If your consumer sets up a logger, you must add a configuration for it
+# here in order for the messages to show up. e.g. if it set up a logger
+# called 'example_printer', you could do:
+[log_config.loggers.koji_sync_listener]
+level = "DEBUG"
+propagate = false
+handlers = ["console"]
+
+[log_config.root]
+level = "ERROR"
+handlers = ["console"]