First stab at reorganizing fedmsg for anitya into its own bus.

playbooks/groups/anitya.yml

@@ -65,8 +65,15 @@
   - include: "{{ tasks }}/apache.yml"
 
   roles:
+  - anitya/fedmsg
   - anitya/frontend
 
+  tasks:
+  - name: install fedmsg-relay
+    yum: pkg=fedmsg-relay state=installed
+  - name: and start it
+    service: name=fedmsg-relay state=started
+
   tags:
   - anitya
 
@@ -85,6 +92,7 @@
    - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
 
   roles:
+  - anitya/fedmsg
   - anitya/backend
 
   tags:
@@ -92,21 +100,3 @@
 
   handlers:
   - include: "{{ handlers }}/restart_services.yml"
-
-# This comes last because it requires usernames from some of the previous plays.
-- name: finish up by setting up fedmsg
-  hosts: anitya
-  user: root
-  gather_facts: True
-  accelerate: "{{ accelerated }}"
-
-  vars_files:
-   - /srv/web/infra/ansible/vars/global.yml
-   - "{{ private }}/vars.yml"
-   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
-
-  roles:
-  - fedmsg/base
-
-  handlers:
-  - include: "{{ handlers }}/restart_services.yml"

roles/anitya/fedmsg/files/selinux/fedmsg.te

@@ -0,0 +1,21 @@
+
+module fedmsg 1.1;
+
+require {
+	type anon_inodefs_t;
+	type httpd_t;
+	class file write;
+}
+
+require {
+	type ptmx_t;
+	type httpd_t;
+	class chr_file getattr;
+}
+
+#============= httpd_t ==============
+# For basic port binding
+allow httpd_t anon_inodefs_t:file write;
+# So that psutil can work from /etc/fedmsg.d/logging.py
+allow httpd_t ptmx_t:chr_file getattr;
+

roles/anitya/fedmsg/tasks/main.yml

@@ -0,0 +1,118 @@
+---
+# tasklist for setting up fedmsg
+# This is the base set of files needed for fedmsg
+
+- name: install needed packages
+  yum: pkg={{ item }} state=installed
+  with_items:
+  - fedmsg
+  - libsemanage-python
+  - python-psutil
+  - policycoreutils-python  # This is in the kickstart now.  Here for old hosts.
+  tags:
+  - packages
+
+# We use setgid here so that the monitoring sockets created by fedmsg services
+# are accessible to the nrpe group.
+- name: create a /var/run/fedmsg dir with setgid for monitoring.
+  file: >
+    dest=/var/run/fedmsg
+    mode=2775
+    owner=fedmsg
+    group=nrpe
+    state=directory
+
+- name: setup /etc/fedmsg.d directory
+  file: path=/etc/fedmsg.d owner=root group=root mode=0755 state=directory
+  tags:
+  - config
+
+# Any files that change need to restart any services that depend on them.  A
+# trick here is that some hosts have an httpd that uses fedmsg, while others do
+# not.  Some hosts have a fedmsg-hub that uses this config, while others do not.
+# Our handlers in handlers/restart_services.yml are smart enough to
+# *conditionally* restart these services, only if they are installed on the
+# system.
+- name: setup basic /etc/fedmsg.d/ contents
+  template: >
+    src="{{ item }}.j2"
+    dest="/etc/fedmsg.d/{{ item }}"
+    owner=root
+    group=root
+    mode=644
+  with_items:
+  - ssl.py
+  - endpoints.py
+  - relay.py
+  - logging.py
+  - base.py
+  tags:
+  - config
+  - fedmsgdconfig
+  notify:
+  - restart httpd
+  - restart fedmsg-relay
+
+- name: Remove unwanted files
+  file: dest=/etc/fedmsg.d/{{item}} state=absent
+  with_items:
+  - endpoints.py 
+  tags:
+  - config
+  - fedmsgdconfig
+  notify:
+  - restart httpd
+  - restart fedmsg-relay
+
+- name: setup /etc/pki/fedmsg directory
+  file: path=/etc/pki/fedmsg owner=root group=root mode=0755 state=directory
+  tags:
+  - config
+
+- name: install fedmsg ca.cert
+  copy: >
+    src="{{ puppet_private }}/fedmsg-certs/keys/ca.crt"
+    dest=/etc/pki/fedmsg/ca.crt
+    owner=root
+    group=root
+    mode=0644
+  tags:
+  - config
+
+- name: fedmsg certs
+  copy: >
+    src="{{ private }}/files/fedmsg-certs/keys/{{item['service']}}-{{fedmsg_fqdn | default(ansible_fqdn)}}.crt"
+    dest=/etc/pki/fedmsg/
+    mode=644
+    owner={{item['owner']}}
+    group={{item['group']}}
+  with_items:
+  - "{{ fedmsg_certs }}"
+  when: fedmsg_certs != []
+  tags:
+  - config
+
+- name: fedmsg keys
+  copy: >
+    src="{{ private }}/files/fedmsg-certs/keys/{{item['service']}}-{{fedmsg_fqdn | default(ansible_fqdn)}}.key"
+    dest=/etc/pki/fedmsg/
+    mode=0640
+    owner={{item['owner']}}
+    group={{item['group']}}
+  with_items:
+  - "{{ fedmsg_certs }}"
+  when: fedmsg_certs != []
+  tags:
+  - config
+
+# Three tasks for handling our custom selinux module
+- name: ensure a directory exists for our custom selinux module
+  file: dest=/usr/local/share/fedmsg state=directory
+
+- name: copy over our custom selinux module
+  copy: src=selinux/fedmsg.pp dest=/usr/local/share/fedmsg/fedmsg.pp
+  register: selinux_module
+
+- name: install our custom selinux module
+  command: semodule -i /usr/local/share/fedmsg/fedmsg.pp
+  when: selinux_module|changed

roles/anitya/fedmsg/templates/base.py.j2

@@ -0,0 +1,83 @@
+config = dict(
+    # Set this to dev if you're hacking on fedmsg or an app locally.
+    # Set to stg or prod if running in the Fedora Infrastructure.
+    {% if env == 'staging' %}
+    environment="stg",
+    {% else %}
+    environment="prod",
+    {% endif %}
+
+    {% if not ansible_hostname.startswith('busgateway') %}
+    # These options provide a place for hub processes to write out their last
+    # processed message.  This let's them read it in at startup and figure out
+    # what kind of backlog they have to deal with.
+    {% if env == 'staging' %}
+    # But we have it turned off in staging.
+    #status_directory="/var/run/fedmsg/status",
+    {% else %}
+    status_directory="/var/run/fedmsg/status",
+    {% endif %}
+
+    # This is the URL of a datagrepper instance that we can query for backlog.
+    {% if env == 'staging' %}
+    # But we have it turned off in staging.
+    #datagrepper_url="https://apps.stg.fedoraproject.org/datagrepper/raw",
+    {% else %}
+    datagrepper_url="https://apps.fedoraproject.org/datagrepper/raw",
+    {% endif %}
+    {% endif %}
+
+    # This used to be set to 1 for safety, but it turns out it was
+    # excessive.  It is the number of seconds that fedmsg should sleep
+    # after it has initialized, but before it begins to try and send any
+    # messages.  If set to a non-zero value, this will slow down one-off
+    # fedmsg scripts like the git post-receive hook and pkgdb2branch.
+    # If we are experiencing message-loss problems, one of the first things
+    # to try should be to turn this number up to a non-zero value.  '1' should
+    # be more than sufficient.
+    post_init_sleep=0.4,
+
+    # This is the number of milliseconds to wait before timing out on
+    # connections.. notably to the fedmsg-relay in the event that it has
+    # crashed.
+    zmq_linger=2000,
+
+    # Default is 0
+    high_water_mark=0,
+    io_threads=1,
+
+    # We almost always want the fedmsg-hub to be sending messages with zmq as
+    # opposed to amqp or stomp.  The only exception will be the bugzilla
+    # amqp<->zmq bridge service.
+    zmq_enabled=True,
+
+    # When subscribing to messages, we want to allow splats ('*') so we tell the
+    # hub to not be strict when comparing messages topics to subscription
+    # topics.
+    zmq_strict=False,
+
+    # See the following
+    #   - http://tldp.org/HOWTO/TCP-Keepalive-HOWTO/overview.html
+    #   - http://api.zeromq.org/3-2:zmq-setsockopt
+    zmq_tcp_keepalive=1,
+    zmq_tcp_keepalive_cnt=3,
+    zmq_tcp_keepalive_idle=60,
+    zmq_tcp_keepalive_intvl=5,
+)
+
+# This option adds an IPC socket by which we can monitor hub health.
+try:
+    import os
+    import psutil
+
+    pid = os.getpid()
+    proc = [p for p in psutil.process_iter() if p.pid == pid][0]
+
+    config['moksha.monitoring.socket'] = \
+        'ipc:///var/run/fedmsg/monitoring-%s.socket' % proc.name
+    config['moksha.monitoring.socket.mode'] = '770'
+except (OSError, ImportError):
+    # We run into issues when trying to import psutil from mod_wsgi on rhel7
+    # but this feature is of no concern in that context, so just fail quietly.
+    # https://github.com/jmflinuxtx/kerneltest-harness/pull/17#issuecomment-48007837
+    pass

roles/anitya/fedmsg/templates/logging.py.j2

@@ -0,0 +1,156 @@
+# Setup fedmsg logging.
+
+# All of these modules are just used by the ContextInjector below.
+import inspect
+import logging
+import os
+import socket
+import traceback
+
+psutil = None
+try:
+    import psutil
+except (OSError, ImportError):
+    # We run into issues when trying to import psutil from inside mod_wsgi on
+    # rhel7.  If we hit that here, then just fail quietly.
+    # https://github.com/jmflinuxtx/kerneltest-harness/pull/17#issuecomment-48007837
+    pass
+
+
+class ContextInjector(logging.Filter):
+    """ Logging filter that adds context to log records.
+
+    Filters are typically used to "filter" log records.  They declare a filter
+    method that can return True or False.  Only records with 'True' will
+    actually be logged.
+
+    Here, we somewhat abuse the concept of a filter.  We always return true,
+    but we use the opportunity to hang important contextual information on the
+    log record to later be used by the logging Formatter.  We don't normally
+    want to see all this stuff in normal log records, but we *do* want to see
+    it when we are emailed error messages.  Seeing an error, but not knowing
+    which host it comes from, is not that useful.
+
+    http://docs.python.org/2/howto/logging-cookbook.html#filters-contextual
+    """
+
+    def filter(self, record):
+        current_process = ContextInjector.get_current_process()
+        current_hostname = socket.gethostname()
+
+        record.host = current_hostname
+        record.proc = current_process
+        record.pid = current_process.pid
+        record.proc_name = current_process.name
+        record.command_line = " ".join(current_process.cmdline)
+        record.callstack = self.format_callstack()
+        return True
+
+    @staticmethod
+    def format_callstack():
+        for i, frame in enumerate(f[0] for f in inspect.stack()):
+            if not '__name__' in frame.f_globals:
+                continue
+            modname = frame.f_globals['__name__'].split('.')[0]
+            if modname != "logging":
+                break
+
+        def _format_frame(frame):
+            return '  File "%s", line %i in %s\n    %s' % (frame)
+
+        stack = traceback.extract_stack()
+        stack = stack[:-i]
+        return "\n".join([_format_frame(frame) for frame in stack])
+
+    @staticmethod
+    def get_current_process():
+        mypid = os.getpid()
+
+        if not psutil:
+            raise OSError("Could not import psutil for %r" % mypid)
+
+        for proc in psutil.process_iter():
+            if proc.pid == mypid:
+                return proc
+
+        # This should be impossible.
+        raise ValueError("Could not find process %r" % mypid)
+
+    @classmethod
+    def __json__(cls):
+        """ We need to be jsonifiable for "fedmsg-config" """
+        return {'name': 'ContextInjector'}
+
+
+hefty_format = """Message
+-------
+[%(asctime)s][%(name)10s %(levelname)7s]
+%(message)s
+
+Process Details
+---------------
+host:     %(host)s
+PID:      %(pid)s
+name:     %(proc_name)s
+command:  %(command_line)s
+
+Callstack that lead to the logging statement
+--------------------------------------------
+%(callstack)s
+"""
+
+
+# See the following for constraints on this format http://bit.ly/Xn1WDn
+config = dict(
+    logging=dict(
+        version=1,
+        formatters=dict(
+            bare={
+                "datefmt": "%Y-%m-%d %H:%M:%S",
+                "format": "[%(asctime)s][%(name)10s %(levelname)7s] %(message)s"
+            },
+            hefty={
+                "datefmt": "%Y-%m-%d %H:%M:%S",
+                "format": hefty_format,
+            },
+        ),
+        filters=dict(
+            context={
+                # This "()" syntax in the stdlib doesn't seem to be documented
+                # anywhere.  I had to read
+                # /usr/lib64/python2.7/logging/config.py to figure it out.
+                "()": ContextInjector,
+            },
+        ),
+        handlers=dict(
+            console={
+                "class": "logging.StreamHandler",
+                "formatter": "bare",
+                "level": "INFO",
+                "stream": "ext://sys.stdout",
+            },
+            mailer={
+                "class": "logging.handlers.SMTPHandler",
+                "formatter": "hefty",
+                "filters": ["context"],
+                "level": "ERROR",
+                "mailhost": "bastion.phx2.fedoraproject.org",
+                "fromaddr": "fedmsg@fedoraproject.org",
+                "toaddrs": ["sysadmin-datanommer-members@fedoraproject.org"],
+                "subject": "fedmsg error log",
+            },
+        ),
+        loggers=dict(
+            fedmsg={
+                "level": "INFO",
+                "propagate": False,
+                "handlers": ["console", "mailer"],
+            },
+            moksha={
+                "level": "INFO",
+                "propagate": False,
+                "handlers": ["console", "mailer"],
+            },
+        ),
+    ),
+)

roles/anitya/fedmsg/templates/relay.py.j2

@@ -0,0 +1,16 @@
+config = dict(
+    endpoints={
+        # This is the output side of the relay to which all other
+        # services can listen.
+        "relay_outbound": [
+            # Messages emerge here
+            "tcp://release-monitoring.org:9940',
+        ],
+    },
+
+    # wsgi scripts on the frontend talk back here
+    # so do cronjobs on anitya-backend01.  they get a firewall rule.
+    relay_inbound=[
+        "tcp://anitya-frontend01.fedoraproject.org:9941",
+    ],
+)

roles/anitya/fedmsg/templates/ssl.py.j2

@@ -0,0 +1,192 @@
+
+{% if env == 'staging' %}
+suffix = "stg.phx2.fedoraproject.org"
+topic_prefix = "org.fedoraproject.stg."
+{% else %}
+suffix = "phx2.fedoraproject.org"
+topic_prefix = "org.fedoraproject.prod."
+{% endif %}
+
+vpn_suffix = "vpn.fedoraproject.org"
+
+config = dict(
+    sign_messages=True,
+    validate_signatures=True,
+    ssldir="/etc/pki/fedmsg",
+
+    {% if env == 'staging' %}
+    # In staging, we use the internal IP of proxy01 without ssl.
+    crl_location="http://10.5.126.88/fedmsg/crl.pem",
+    {% else %}
+    crl_location="https://fedoraproject.org/fedmsg/crl.pem",
+    {% endif %}
+    crl_cache="/var/run/fedmsg/crl.pem",
+    crl_cache_expiry=86400,  # Daily
+
+    certnames=dict(
+    [
+        ("bugzilla.bugzilla2fedmsg01", "bugzilla2fedmsg-bugzilla2fedmsg01.%s" % suffix)
+    ] + [
+        ("shell.bugzilla2fedmsg01", "shell-bugzilla2fedmsg01.%s" % suffix)
+    ] + [
+        ("github2fedmsg.github2fedmsg0%i" % i, "github2fedmsg-github2fedmsg0%i.%s" % (i, suffix))
+        for i in range(1, 3)
+    ] + [
+        ("shell.github2fedmsg0%i" % i, "shell-github2fedmsg0%i.%s" % (i, suffix))
+        for i in range(1, 3)
+    ] + [
+        ("fedimg.fedimg01", "fedimg-fedimg01.%s" % suffix),
+        ("shell.fedimg01", "shell-fedimg01.%s" % suffix),
+    ] + [
+        ("kerneltest.kerneltest0%i" % i, "kerneltest-kerneltest0%i.%s" % (i, suffix))
+        for i in range(1, 3)
+    ] + [
+        ("shell.kerneltest0%i" % i, "shell-kerneltest0%i.%s" % (i, suffix))
+        for i in range(1, 3)
+    ] + [
+        ("shell.notifs-web0%i" % i, "shell-notifs-web0%i.%s" % (i, suffix))
+        for i in range(1, 3)
+    ] + [
+        ("fmn.notifs-web0%i" % i, "fmn-notifs-web0%i.%s" % (i, suffix))
+        for i in range(1, 3)
+    ] + [
+        ("fmn.notifs-backend01", "fmn-notifs-backend01.%s" % suffix),
+    ] + [
+        ("shell.pkgdb0%i" % i, "shell-pkgdb0%i.%s" % (i, suffix))
+        for i in range(1, 3)
+    ] + [
+        ("pkgdb2.pkgdb0%i" % i, "pkgdb-pkgdb0%i.%s" % (i, suffix))
+        for i in range(1, 3)
+    ] + [
+        ("mediawiki.wiki0%i" % i, "mediawiki-wiki0%i.%s" % (i, suffix))
+        for i in range(1, 3)
+    ] + [
+        ("shell.fas0%i" % i, "shell-fas0%i.%s" % (i, suffix))
+        for i in range(1, 4)
+    ] + [
+        ("fas.fas0%i" % i, "fas-fas0%i.%s" % (i, suffix))
+        for i in range(1, 4)
+    ] + [
+        ("shell.packages0%i" % i, "shell-packages0%i.%s" % (i, suffix))
+        for i in range(3, 5)
+    ] + [
+        ("fedoratagger.tagger0%i" % i, "fedoratagger-tagger0%i.%s" % (i, suffix))
+        for i in range(1, 3)
+    ] + [
+        ("shell.tagger0%i" % i, "shell-tagger0%i.%s" % (i, suffix))
+        for i in range(1, 3)
+    ] + [
+        ("shell.pkgs0%i" % i, "shell-pkgs0%i.%s" % (i, suffix))
+        for i in range(1, 2)
+    ] + [
+        ("scm.pkgs0%i" % i, "scm-pkgs0%i.%s" % (i, suffix))
+        for i in range(1, 2)
+    ] + [
+        ("lookaside.pkgs0%i" % i, "lookaside-pkgs0%i.%s" % (i, suffix))
+        for i in range(1, 2)
+    ] + [
+        ("shell.relepel01", "shell-relepel01.%s" % suffix),
+        ("shell.branched-composer", "shell-branched-composer.%s" % suffix),
+        ("shell.rawhide-composer", "shell-rawhide-composer.%s" % suffix),
+        ("shell.releng03", "shell-releng03.%s" % suffix),
+        ("shell.releng04", "shell-releng04.%s" % suffix),
+        ("bodhi.relepel01", "bodhi-relepel01.%s" % suffix),
+        ("bodhi.branched-composer", "bodhi-branched-composer.%s" % suffix),
+        ("bodhi.rawhide-composer", "bodhi-rawhide-composer.%s" % suffix),
+        ("bodhi.releng03", "bodhi-releng03.%s" % suffix),
+        ("bodhi.releng04", "bodhi-releng04.%s" % suffix),
+    ] + [
+        ("ftpsync.relepel01", "ftpsync-relepel01.%s" % suffix),
+        ("ftpsync.releng04", "ftpsync-releng04.%s" % suffix),
+    ] + [
+        ("shell.busgateway01", "shell-busgateway01.%s" % suffix),
+    ] + [
+        ("shell.value01", "shell-value01.%s" % suffix),
+        ("supybot.value01", "supybot-value01.%s" % suffix),
+    ] + [
+        ("koji.koji04", "koji-koji04.%s" % suffix),
+        ("koji.koji01", "koji-koji01.%s" % suffix),
+        ("koji.koji03", "koji-koji03.%s" % suffix),
+        ("shell.koji04", "shell-koji04.%s" % suffix),
+        ("shell.koji01", "shell-koji01.%s" % suffix),
+        ("shell.koji03", "shell-koji03.%s" % suffix),
+    ] + [
+        ("nagios.noc01", "nagios-noc01.%s" % suffix),
+        ("shell.noc01", "shell-noc01.%s" % suffix),
+    ] + [
+        ("git.hosted03", "git-hosted03.%s" % vpn_suffix),
+        ("git.hosted04", "git-hosted04.%s" % vpn_suffix),
+        ("trac.hosted03", "trac-hosted03.%s" % vpn_suffix),
+        ("trac.hosted04", "trac-hosted04.%s" % vpn_suffix),
+        ("shell.hosted03", "shell-hosted03.%s" % vpn_suffix),
+        ("shell.hosted04", "shell-hosted04.%s" % vpn_suffix),
+    ] + [
+        ("shell.lockbox01", "shell-lockbox01.%s" % suffix),
+        ("announce.lockbox01", "announce-lockbox01.%s" % suffix),
+    ] + [
+        # These first two entries are here to placate a bug in
+        # python-askbot-fedmsg-0.0.4.  They can be removed once
+        # python-askbot-fedmsg-0.0.5 hits town.
+        ("askbot.ask01.phx2.fedoraproject.org", "askbot-ask01.%s" % suffix),
+        ("askbot.ask01.stg.phx2.fedoraproject.org", "askbot-ask01.%s" % suffix),
+
+        ("askbot.ask01", "askbot-ask01.%s" % suffix),
+        ("shell.ask01", "shell-ask01.%s" % suffix),
+
+        ("askbot.ask02", "askbot-ask02.%s" % suffix),
+        ("shell.ask02", "shell-ask02.%s" % suffix),
+
+        ("fedbadges.badges-backend01", "fedbadges-badges-backend01.%s" % suffix),
+        ("shell.badges-backend01", "shell-badges-backend01.%s" % suffix),
+
+        ("summershum.summershum01", "summershum-summershum01.%s" % suffix),
+        ("shell.summershum01", "shell-summershum01.%s" % suffix),
+
+        ("tahrir.badges-web01", "tahrir-badges-web01.%s" % suffix),
+        ("shell.badges-web01", "shell-badges-web01.%s" % suffix),
+        ("tahrir.badges-web02", "tahrir-badges-web02.%s" % suffix),
+        ("shell.badges-web02", "shell-badges-web02.%s" % suffix),
+
+        ("shell.nuancier01", "shell-nuancier01.%s" % suffix),
+        ("shell.nuancier02", "shell-nuancier02.%s" % suffix),
+        ("nuancier.nuancier01", "nuancier-nuancier01.%s" % suffix),
+        ("nuancier.nuancier02", "nuancier-nuancier02.%s" % suffix),
+
+        ("shell.fedocal01", "shell-fedocal01.%s" % suffix),
+        ("shell.fedocal02", "shell-fedocal02.%s" % suffix),
+        ("fedocal.fedocal01", "fedocal-fedocal01.%s" % suffix),
+        ("fedocal.fedocal02", "fedocal-fedocal02.%s" % suffix),
+
+        ("shell.mailman01", "shell-mailman01.%s" % suffix),
+        ("mailman.mailman01", "mailman-mailman01.%s" % suffix),
+
+        ("shell.bodhi01", "shell-bodhi01.%s" % suffix),
+        ("shell.bodhi02", "shell-bodhi02.%s" % suffix),
+        ("bodhi.bodhi01", "bodhi-bodhi01.%s" % suffix),
+        ("bodhi.bodhi02", "bodhi-bodhi02.%s" % suffix),
+
+        ("shell.elections01", "shell-elections01.%s" % suffix),
+        ("shell.elections02", "shell-elections02.%s" % suffix),
+        ("fedora_elections.elections01", "fedora_elections-elections01.%s" % suffix),
+        ("fedora_elections.elections02", "fedora_elections-elections02.%s" % suffix),
+
+        ("shell.anitya-frontend01", "shell-anitya-frontend01.fedoraproject.org"),
+        ("anitya.anitya-frontend01", "anitya-anitya-frontend01.fedoraproject.org"),
+        ("shell.anitya-backend01", "shell-anitya-backend01.fedoraproject.org"),
+        ("anitya.anitya-backend01", "anitya-anitya-backend01.fedoraproject.org"),
+
+        # This is for the copr backend, which is a little different.  The
+        # "cert-prefix" is just "copr", and is hardcoded in
+        # backend/dispatcher.py.  The hostname is also a little different,
+        # since it is an openstack node.  This might be a little fragile.  :/
+        # See https://github.com/fedora-infra/fedmsg/issues/199 for the plan.
+        ("copr.dhcp-client03", "copr-copr-be.cloud.fedoraproject.org"),
+        ("copr.copr-be-i-00000407", "copr-copr-be.cloud.fedoraproject.org"),
+        ("copr.copr-be", "copr-copr-be.cloud.fedoraproject.org"),
+
+        # Jenkins, also being a cloud node, is weird.  Like copr.
+        ("shell.jenkins-master-unknown", "shell-jenkins.cloud.fedoraproject.org"),
+        ("jenkins.jenkins-master-unknown", "jenkins-jenkins.cloud.fedoraproject.org"),
+    ]),
+)
+