initial commit of the releng tasks

files/hosts/releng-hosts

@@ -0,0 +1,8 @@
+127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
+::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
+10.5.126.23    infrastructure.fedoraproject.org
+10.5.125.63    koji.fedoraproject.org
+10.5.125.36    kojipkgs.fedoraproject.org
+10.5.124.138   arm.koji.fedoraproject.org armpkgs.fedoraproject.org
+10.5.125.44    pkgs.fedoraproject.org  pkgs
+10.5.126.52    mirrors.fedoraproject.org admin.fedoraproject.org

files/iptables/releng

@@ -0,0 +1,211 @@
+#
+# NOTE: Chaging this template updates iptables on all hosts. 
+# Please check with sysadmin-main before pushing out an update here. 
+#
+*nat
+:PREROUTING ACCEPT []
+:POSTROUTING ACCEPT []
+:OUTPUT ACCEPT []
+COMMIT
+*raw
+:PREROUTING ACCEPT []
+:OUTPUT ACCEPT []
+COMMIT
+*filter
+:INPUT DROP []
+:FORWARD ACCEPT []
+:OUTPUT ACCEPT []
+
+# loopback allowed
+-A INPUT -i lo -j ACCEPT
+
+# Accept ping and traceroute (needs icmp)
+-A INPUT -p icmp -j ACCEPT
+
+# Established connections allowed
+-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
+
+# Overwrite any global rules
+
+# allow func through from the overlord (puppet1)
+-A INPUT -p tcp -m tcp -s 209.132.181.6 --dport 51234 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.126.23 --dport 51234 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.127.51 --dport 51234 -j ACCEPT
+
+# Staging separation.  Do not allow stg server to hit _any_ production hosts
+#    exceptions being for infrastructure.fp.o (for packages) and admin.fp.o
+#    for accounts
+
+
+# Temporary measure for ro access to nfs1
+# source app1.stg
+-A INPUT -p tcp -m tcp -s 10.5.126.81 --dport 48621:48624 -j ACCEPT
+-A INPUT -p udp -m udp -s 10.5.126.81 --dport 48621:48624 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.126.81 --dport 2049 -j ACCEPT
+-A INPUT -p udp -m udp -s 10.5.126.81 --dport 2049 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.126.81 --dport 111 -j ACCEPT
+-A INPUT -p udp -m udp -s 10.5.126.81 --dport 111 -j ACCEPT
+
+# source app2.stg
+-A INPUT -p tcp -m tcp -s 10.5.126.82 --dport 48621:48624 -j ACCEPT
+-A INPUT -p udp -m udp -s 10.5.126.82 --dport 48621:48624 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.126.82 --dport 2049 -j ACCEPT
+-A INPUT -p udp -m udp -s 10.5.126.82 --dport 2049 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.126.82 --dport 111 -j ACCEPT
+-A INPUT -p udp -m udp -s 10.5.126.82 --dport 111 -j ACCEPT
+
+# source koji1.stg
+-A INPUT -p tcp -m tcp -s 10.5.126.87 --dport 48621:48624 -j ACCEPT
+-A INPUT -p udp -m udp -s 10.5.126.87 --dport 48621:48624 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.126.87 --dport 2049 -j ACCEPT
+-A INPUT -p udp -m udp -s 10.5.126.87 --dport 2049 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.126.87 --dport 111 -j ACCEPT
+-A INPUT -p udp -m udp -s 10.5.126.87 --dport 111 -j ACCEPT
+
+# source releng1.stg
+-A INPUT -p tcp -m tcp -s 10.5.126.90 --dport 48621:48624 -j ACCEPT
+-A INPUT -p udp -m udp -s 10.5.126.90 --dport 48621:48624 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.126.90 --dport 2049 -j ACCEPT
+-A INPUT -p udp -m udp -s 10.5.126.90 --dport 2049 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.126.90 --dport 111 -j ACCEPT
+-A INPUT -p udp -m udp -s 10.5.126.90 --dport 111 -j ACCEPT
+
+# infrastucture.fp.o
+# proxy1
+-A INPUT -p tcp -m tcp -d 10.5.126.52 --dport 80 -j ACCEPT
+# proxy2.stg
+-A INPUT -p tcp -m tcp -d 10.5.126.89 --dport 80 -j ACCEPT
+
+# kojipkgs
+-A INPUT -p tcp -m tcp -d 10.5.125.36 --dport 80 -j ACCEPT
+
+# admin.fp.o
+# puppet1
+-A INPUT -p tcp -m tcp -d 10.5.126.23 --dport 8140 -j ACCEPT
+-A INPUT -p tcp -m tcp -d 10.5.126.23 --dport 873 -j ACCEPT
+-A INPUT -p tcp -m tcp -d 10.5.126.23 --dport 80 -j ACCEPT
+#-A INPUT -p tcp -m tcp -d 10.5.126.23 --dport 51234:51235 -j ACCEPT
+
+# DNS
+-A INPUT -p udp -m udp -d 10.5.126.21 --dport 53 -j ACCEPT
+-A INPUT -p udp -m udp -d 10.5.126.22 --dport 53 -j ACCEPT
+
+# bastion
+-A INPUT -p tcp -m tcp -d 10.5.126.12 --dport 25 -j ACCEPT
+
+# Func and staging bits
+-A INPUT -s 10.5.126.81 -p tcp -m tcp --dport 51234:51235 -j ACCEPT
+-A INPUT -s 10.5.126.82 -p tcp -m tcp --dport 51234:51235 -j ACCEPT
+# proxy1.stg
+-A INPUT -s 10.5.126.88 -p tcp -m tcp --dport 51234:51235 -j ACCEPT
+# db1.stg
+-A INPUT -s 10.5.126.84 -p tcp -m tcp --dport 51234:51235 -j ACCEPT
+-A INPUT -s 10.5.126.87 -p tcp -m tcp --dport 51234:51235 -j ACCEPT
+-A INPUT -s 10.5.126.90 -p tcp -m tcp --dport 51234:51235 -j ACCEPT
+-A INPUT -s 10.5.126.91 -p tcp -m tcp --dport 51234:51235 -j ACCEPT
+-A INPUT -s 10.5.126.92 -p tcp -m tcp --dport 51234:51235 -j ACCEPT
+# cvs.stg
+-A INPUT -s 10.5.126.83 -p tcp -m tcp --dport 51234:51235 -j ACCEPT
+
+# Allow staging to talk to log02.
+-A INPUT -p tcp -m tcp -d 10.5.126.29 --dport 514 -j ACCEPT
+
+# Ban staging on non-staging hosts only.
+
+
+
+# SSH
+# ssh block against uni in .cz where problem(s) have been cited
+# added by skvidal on jan 24 2011 - as per request from spot
+-A INPUT -p tcp -m tcp -s 147.251.0.0/16 --dport 22 -j REJECT
+# matches last rule for a service we want blocked
+-A INPUT -p tcp -m tcp -s 192.168.100.0/24 --dport 22 -j REJECT --reject-with tcp-reset
+-A INPUT -p tcp -m tcp  --dport 22 -j ACCEPT 
+
+# Allow all netapp traffic
+-A INPUT -p udp -m udp -s 10.5.88.11 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.88.11 -j ACCEPT
+-A INPUT -p udp -m udp -s 10.5.88.20 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.88.20 -j ACCEPT
+-A INPUT -p udp -m udp -s 10.5.88.21 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.88.21 -j ACCEPT
+
+# Allow other PHX-local NFS servers traffic
+# secondary1 server
+-A INPUT -p udp -m udp -s 10.5.126.0/24 -d 10.5.126.27 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.126.0/24 -d 10.5.126.27 -j ACCEPT
+-A INPUT -p udp -m udp -s 10.5.127.0/24 -d 10.5.126.27 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.127.0/24 -d 10.5.126.27 -j ACCEPT
+# secondary1 mounters
+-A INPUT -p udp -m udp -s 10.5.126.27 -j ACCEPT
+
+# NRPE (nagios monitoring)
+-A INPUT -p tcp -m tcp -s 10.5.126.41 --dport 5666 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 192.168.1.10 --dport 5666 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 192.168.1.20 --dport 5666 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 209.132.181.102 --dport 5666 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 209.132.181.102 --dport 5666 -j ACCEPT
+
+# SNMP allows from our monitoring systems
+-A INPUT -p udp -m udp -s 10.5.126.41 --dport 161 -j ACCEPT
+-A INPUT -p udp -m udp -s 10.5.126.10 --dport 161 -j ACCEPT
+-A INPUT -p udp -m udp -s 10.5.126.11 --dport 161 -j ACCEPT
+-A INPUT -p udp -m udp -s 10.5.126.12 --dport 161 -j ACCEPT
+-A INPUT -p udp -m udp -s 10.5.126.23 --dport 161 -j ACCEPT
+-A INPUT -p udp -m udp -s 209.132.181.102 --dport 161 -j ACCEPT
+-A INPUT -p udp -m udp -s 209.132.181.102 --dport 161 -j ACCEPT
+-A INPUT -p udp -m udp -s 192.168.0.0/24 --dport 161 -j ACCEPT
+-A INPUT -p udp -m udp -s 192.168.1.0/24 --dport 161 -j ACCEPT
+
+# NTP servers (if any)
+#-A INPUT -p udp -m udp -s ips-allowed-here --dport 123 -j ACCEPT
+
+# Bacula Backups backup03
+-A INPUT -p tcp -m tcp -s 10.5.126.161 --dport 9102 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 192.168.1.64 --dport 9102 -j ACCEPT
+
+
+# allow fedmsg ports through - this happens after the staging ban so
+# we should be safe from evil (or what not)
+# fedmsg - (tagger, bodhi, and fas) WSGI process ports
+-A INPUT -p tcp -m tcp -s 10.5.124.0/24 --dport 3000:3007 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.125.0/24 --dport 3000:3007 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.126.0/24 --dport 3000:3007 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 192.168.1.0/24 --dport 3000:3007 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 192.168.0.0/24 --dport 3000:3007 -j ACCEPT
+# fedmsg - busmon hub consumer
+-A INPUT -p tcp -m tcp -s 10.5.124.0/24 --dport 3008 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.125.0/24 --dport 3008 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.126.0/24 --dport 3008 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 192.168.1.0/24 --dport 3008 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 192.168.0.0/24 --dport 3008 -j ACCEPT
+# fedmsg - fedmsg-relay
+-A INPUT -p tcp -m tcp -s 10.5.124.0/24 --dport 3998:3999 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.125.0/24 --dport 3998:3999 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.126.0/24 --dport 3998:3999 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 192.168.1.0/24 --dport 3998:3999 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 192.168.0.0/24 --dport 3998:3999 -j ACCEPT
+# fedmsg - hub websocket server
+-A INPUT -p tcp -m tcp -s 10.5.124.0/24 --dport 9919 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.125.0/24 --dport 9919 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 10.5.126.0/24 --dport 9919 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 192.168.1.0/24 --dport 9919 -j ACCEPT
+-A INPUT -p tcp -m tcp -s 192.168.0.0/24 --dport 9919 -j ACCEPT
+
+# Custom Services
+
+# Services TCP
+-A INPUT -p tcp -m tcp  --dport 22 -j ACCEPT
+-A INPUT -p tcp -m tcp  --dport 80 -j ACCEPT
+-A INPUT -p tcp -m tcp  --dport 443 -j ACCEPT
+-A INPUT -p tcp -m tcp  --dport 5666 -j ACCEPT
+
+# Services UDP
+
+# more services we use - ports for random services and TG listeners.
+
+# Extra protection for 192.168.100.x vpn hosts. 
+-A INPUT -s 192.168.100.0/24 -j REJECT --reject-with icmp-host-prohibited
+
+-A INPUT -j REJECT --reject-with icmp-host-prohibited
+COMMIT

files/releng/branched

@@ -0,0 +1,3 @@
+# branched compose
+MAILTO=rel-eng@lists.fedoraproject.org
+15 9 * * * masher TMPDIR=`mktemp -d /tmp/branched.XXXXXX` && cd $TMPDIR && git clone -n git://git.fedorahosted.org/releng && cd releng && git checkout -b stable rawhide-stable && LANG=en_US.UTF-8 ./scripts/buildbranched $(date "+\%Y\%m\%d") && sudo -u ftpsync /usr/local/bin/update-fullfilelist fedora

files/releng/fedmsg/base.py

@@ -0,0 +1,20 @@
+
+config = dict(
+    # Set this to dev if you're hacking on fedmsg or an app locally.
+    # Set to stg or prod if running in the Fedora Infrastructure.
+    environment="prod",
+
+    # Default is 0
+    high_water_mark=0,
+    io_threads=1,
+
+    # We almost always want the fedmsg-hub to be sending messages with zmq as
+    # opposed to amqp or stomp.  The only exception will be the bugzilla
+    # amqp<->zmq bridge service.
+    zmq_enabled=True,
+
+    # When subscribing to messages, we want to allow splats ('*') so we tell the
+    # hub to not be strict when comparing messages topics to subscription
+    # topics.
+    zmq_strict=False,
+)

files/releng/fedmsg/endpoints.py

@@ -0,0 +1,90 @@
+suffix = 'phx2.fedoraproject.org'
+non_phx_suffix = 'fedoraproject.org'
+
+config = dict(
+    # This is a dict of possible addresses from which fedmsg can send
+    # messages.  fedmsg.init(...) requires that a 'name' argument be passed
+    # to it which corresponds with one of the keys in this dict.
+    endpoints = {
+        # This is the output side of the relay to which all other
+        # services can listen.
+        "relay_outbound": [
+            "tcp://app01.%s:3999" % suffix,
+        ],
+
+        # For other, more 'normal' services, fedmsg will try to guess the
+        # name of it's calling module to determine which endpoint definition
+        # to use.  This can be overridden by explicitly providing the name in
+        # the initial call to fedmsg.init(...).
+        "bodhi.app01": [
+            "tcp://app01.%s:300%i" % (suffix, i)
+            for i in range(8)
+        ],
+        "bodhi.app02": [
+            "tcp://app02.%s:300%i" % (suffix, i)
+            for i in range(8)
+        ],
+        "bodhi.releng01": [
+            "tcp://releng01.%s:3000" % suffix
+        ],
+        "bodhi.app03": [
+            "tcp://app03.%s:300%i" % (suffix, i)
+            for i in range(8)
+        ],
+        "bodhi.app04": [
+            "tcp://app04.%s:300%i" % (suffix, i)
+            for i in range(8)
+        ],
+        "bodhi.app05": [
+            "tcp://app05.%s:300%i" % (non_phx_suffix, i)
+            for i in range(8)
+        ],
+        "bodhi.app06": [
+            "tcp://app06.%s:300%i" % (non_phx_suffix, i)
+            for i in range(8)
+        ],
+        "bodhi.app07": [
+            "tcp://app07.%s:300%i" % (suffix, i)
+            for i in range(8)
+        ],
+        "bodhi.app08": [
+            "tcp://app08.%s:300%i" % (non_phx_suffix, i)
+            for i in range(8)
+        ],
+        "bodhi.releng04": [
+            "tcp://releng04.%s:3000" % suffix
+        ],
+        "bodhi.relepel01": [
+            "tcp://relepel01.%s:3000" % suffix
+        ],
+        "fas.fas01": [
+            "tcp://fas01.%s:300%i" % (suffix, i)
+            for i in range(8)
+        ],
+        "fedoratagger.packages01": [
+            "tcp://packages01.%s:300%i" % (suffix, i)
+            for i in range(8)
+        ],
+        "fedoratagger.packages02": [
+            "tcp://packages02.%s:300%i" % (suffix, i)
+            for i in range(8)
+        ],
+        "busmon.app01": [
+            "tcp://app01.%s:3008" % suffix,
+        ],
+        "busmon.app02": [
+            "tcp://app02.%s:3008" % suffix,
+        ],
+        "supybot.value03": [
+            "tcp://value03.%s:3000" % suffix,
+        ],
+    },
+
+    # This is the address of an active->passive relay.  It is used for the
+    # fedmsg-logger command which requires another service with a stable
+    # listening address for it to send messages to.
+    # It is also used by the git-hook, for the same reason.
+    # It is also used by the mediawiki php plugin which, due to the oddities of
+    # php, can't maintain a single passive-bind endpoint of it's own.
+    relay_inbound="tcp://app01.%s:3998" % suffix,
+)

files/releng/fedmsg/ircbot.py

@@ -0,0 +1,18 @@
+config = dict(
+    irc=[
+        dict(
+            network='irc.freenode.net',
+            port=6667,
+            nickname='fedmsg-bot',
+            channel='fedora-fedmsg',
+            make_pretty=True,
+            make_terse=True,
+            # Don't show busmon or the heartbeat... gross.
+            # any httpd topics would also be a huge source of spam.
+            filters=dict(
+                topic=['busmon', 'httpd'],
+                body=['lub-dub'],
+            ),
+        ),
+    ],
+)

files/releng/fedmsg/ssl.py

@@ -0,0 +1,64 @@
+
+suffix = "phx2.fedoraproject.org"
+app_hosts = [
+    "app01.phx2.fedoraproject.org",
+    "app02.phx2.fedoraproject.org",
+    "app03.phx2.fedoraproject.org",
+    "app04.phx2.fedoraproject.org",
+    "app05.fedoraproject.org",
+    "app06.fedoraproject.org",
+    "app07.phx2.fedoraproject.org",
+    "app08.fedoraproject.org",
+]
+
+config = dict(
+    sign_messages=True,
+    validate_signatures=False,
+    ssldir="/etc/pki/fedmsg",
+
+    crl_location="https://fedoraproject.org/fedmsg/crl.pem",
+    crl_cache="/var/run/fedmsg/crl.pem",
+    crl_cache_expiry=86400,  # Daily
+
+    certnames=dict(
+    [
+        ("shell.app0%i" % i, "shell-%s" % app_hosts[i-1])
+        for i in range(1, len(app_hosts) + 1)
+    ] + [
+        ("bodhi.app0%i" % i, "bodhi-%s" % app_hosts[i-1])
+        for i in range(1, len(app_hosts) + 1)
+    ] + [
+        ("mediawiki.app0%i" % i, "mediawiki-%s" % app_hosts[i-1])
+        for i in range(1, len(app_hosts) + 1)
+    ] + [
+        ("shell.fas0%i" % i, "shell-fas0%i.%s" % (i, suffix))
+        for i in range(1, 4)
+    ] + [
+        ("fas.fas0%i" % i, "fas-fas0%i.%s" % (i, suffix))
+        for i in range(1, 4)
+    ] + [
+        ("shell.packages0%i" % i, "shell-packages0%i.%s" % (i, suffix))
+        for i in range(1, 3)
+    ] + [
+        ("fedoratagger.packages0%i" % i, "fedoratagger-packages0%i.%s" % (i, suffix))
+        for i in range(1, 3)
+    ] + [
+        ("shell.pkgs0%i" % i, "shell-pkgs0%i.%s" % (i, suffix))
+        for i in range(1, 2)
+    ] + [
+        ("scm.pkgs0%i" % i, "scm-pkgs0%i.%s" % (i, suffix))
+        for i in range(1, 2)
+    ] + [
+        ("shell.relepel01", "shell-relepel01.%s" % suffix),
+        ("shell.releng04", "shell-releng04.%s" % suffix),
+        ("shell.releng01", "shell-releng01.%s" % suffix),
+        ("bodhi.relepel01", "bodhi-relepel01.%s" % suffix),
+        ("bodhi.releng04", "bodhi-releng04.%s" % suffix),
+        ("bodhi.releng01", "bodhi-releng01.%s" % suffix),
+    ] + [
+        ("shell.value01", "shell-value01.%s" % suffix),
+        ("shell.value03", "shell-value03.%s" % suffix),
+        ("supybot.value03", "supybot-value03.%s" % suffix),
+    ])
+)
+

files/releng/fedora-branched-compose-i386.cfg

@@ -0,0 +1,40 @@
+config_opts['root'] = 'fedora-branched-compose-i386'
+config_opts['target_arch'] = 'i386'
+config_opts['chroot_setup_cmd'] = 'install @buildsys-build vim-enhanced joe pungi'
+config_opts['dist'] = 'fc16'  # only useful for --resultdir variable subst
+config_opts['plugin_conf']['root_cache_enable'] = False
+config_opts['internal_dev_setup'] = False
+config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/srv/pungi', '/srv/pungi' ))
+config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/dev', '/dev' ))
+config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/mnt/koji', '/mnt/koji' ))
+config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/pub/fedora', '/pub/fedora' ))
+
+
+config_opts['yum.conf'] = """
+[main]
+cachedir=/var/cache/yum
+debuglevel=2
+reposdir=/dev/null
+logfile=/var/log/yum.log
+retries=20
+obsoletes=1
+gpgcheck=0
+assumeyes=1
+
+# repos
+
+[fedora]
+name=fedora
+baseurl=http://kojipkgs.fedoraproject.org/mash/branched/i386/os
+enabled=1
+cost=5000
+
+[static]
+name=static
+baseurl=http://kojipkgs.fedoraproject.org/repos/f19-build/latest/i386
+enabled=1
+#cost=2000
+"""
+
+
+

files/releng/fedora-branched-compose-x86_64.cfg

@@ -0,0 +1,40 @@
+config_opts['root'] = 'fedora-branched-compose-x86_64'
+config_opts['target_arch'] = 'x86_64'
+config_opts['chroot_setup_cmd'] = 'install @buildsys-build vim-enhanced joe pungi'
+config_opts['dist'] = 'fc16'  # only useful for --resultdir variable subst
+config_opts['plugin_conf']['root_cache_enable'] = False
+config_opts['internal_dev_setup'] = False
+config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/srv/pungi', '/srv/pungi' ))
+config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/dev', '/dev' ))
+config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/mnt/koji', '/mnt/koji' ))
+config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/pub/fedora', '/pub/fedora' ))
+
+
+config_opts['yum.conf'] = """
+[main]
+cachedir=/var/cache/yum
+debuglevel=2
+reposdir=/dev/null
+logfile=/var/log/yum.log
+retries=20
+obsoletes=1
+gpgcheck=0
+assumeyes=1
+
+# repos
+
+[fedora]
+name=fedora
+baseurl=http://kojipkgs.fedoraproject.org/mash/branched/x86_64/os
+enabled=1
+cost=5000
+
+[static]
+name=static
+baseurl=http://kojipkgs.fedoraproject.org/repos/f19-build/latest/x86_64
+enabled=1
+#cost=2000
+"""
+
+
+

files/releng/fedora-devel-pungi-i386.cfg

@@ -0,0 +1,27 @@
+config_opts['root'] = 'fedora-development-pungi-i386'
+config_opts['target_arch'] = 'i386'
+config_opts['chroot_setup_cmd'] = 'groupinstall buildsys-build'
+config_opts['dist'] = 'fc9'
+config_opts['plugin_conf']['root_cache_enable'] = False
+config_opts['internal_dev_setup'] = False
+config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/dev', '/dev' ))
+config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/dev/pts', '/dev/pts' ))
+
+
+config_opts['yum.conf'] = """
+[main]
+cachedir=/var/cache/yum
+debuglevel=1
+reposdir=/dev/null
+logfile=/var/log/yum.log
+retries=20
+obsoletes=1
+gpgcheck=0
+assumeyes=1
+
+# repos
+
+[fedora]
+name=fedora
+baseurl=http://kojipkgs.fedoraproject.org/mash/branched/i386/os
+"""

files/releng/fedora-devel-pungi-ppc.cfg

@@ -0,0 +1,32 @@
+config_opts['root'] = 'fedora-development-pungi-ppc'
+config_opts['target_arch'] = 'ppc'
+config_opts['chroot_setup_cmd'] = 'groupinstall buildsys-build'
+config_opts['dist'] = 'fc9'
+config_opts['plugin_conf']['root_cache_enable'] = False
+config_opts['internal_dev_setup'] = False
+config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/dev', '/dev' ))
+config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/dev/pts', '/dev/pts' ))
+
+
+config_opts['yum.conf'] = """
+[main]
+cachedir=/var/cache/yum
+debuglevel=1
+reposdir=/dev/null
+logfile=/var/log/yum.log
+retries=20
+obsoletes=1
+gpgcheck=0
+assumeyes=1
+
+# repos
+
+[static]
+name=fedora
+baseurl=http://kojipkgs.fedoraproject.org/repos/dist-rawhide/latest/ppc/
+cost=2
+
+[fedora]
+name=fedora
+baseurl=http://kojipkgs.fedoraproject.org/mash/rawhide/ppc/os
+"""

files/releng/fedora-devel-pungi-x86_64.cfg

@@ -0,0 +1,30 @@
+config_opts['root'] = 'fedora-development-pungi-x86_64'
+config_opts['target_arch'] = 'x86_64'
+config_opts['chroot_setup_cmd'] = 'groupinstall buildsys-build'
+config_opts['dist'] = 'fc9'
+config_opts['plugin_conf']['root_cache_enable'] = False
+config_opts['internal_dev_setup'] = False
+config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/dev', '/dev' ))
+config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/dev/pts', '/dev/pts' ))
+
+
+config_opts['yum.conf'] = """
+[main]
+cachedir=/var/cache/yum
+debuglevel=1
+reposdir=/dev/null
+logfile=/var/log/yum.log
+retries=20
+obsoletes=1
+gpgcheck=0
+assumeyes=1
+# grub/syslinux on x86_64 need glibc-devel.i386 which pulls in glibc.i386, need to exclude all
+# .i?86 packages except these.
+exclude=[!g]*.i*86 g[!l]*.i?86  gl[!i]*.i?86 gli[!b]*.i?86 glib[!c]*.i?86 
+
+# repos
+
+[fedora]
+name=fedora
+baseurl=http://kojipkgs.fedoraproject.org/mash/branched/x86_64/os
+"""

files/releng/fedora-rawhide-compose-i386.cfg

@@ -0,0 +1,36 @@
+config_opts['root'] = 'fedora-rawhide-compose-i386'
+config_opts['target_arch'] = 'i386'
+config_opts['chroot_setup_cmd'] = 'groupinstall buildsys-build'
+config_opts['dist'] = 'fc10'  # only useful for --resultdir variable subst
+config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/mnt/koji', '/mnt/koji' ))
+config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/pub/fedora', '/pub/fedora' ))
+config_opts['plugin_conf']['root_cache_enable'] = False
+
+config_opts['yum.conf'] = """
+[main]
+cachedir=/var/cache/yum
+debuglevel=2
+reposdir=/dev/null
+logfile=/var/log/yum.log
+retries=20
+obsoletes=1
+gpgcheck=0
+assumeyes=1
+
+# repos
+
+[fedora]
+name=fedora
+baseurl=http://kojipkgs.fedoraproject.org/mash/rawhide/i386/os
+enabled=1
+cost=5000
+
+[static]
+name=static
+baseurl=http://kojipkgs.fedoraproject.org/repos/rawhide/latest/i386
+enabled=1
+#cost=2000
+"""
+
+
+

files/releng/mash/masher.config

@@ -0,0 +1,2 @@
+Host *
+    StrictHostKeyChecking no

files/releng/mash/masher.id_rsa.pub

@@ -0,0 +1 @@
+from="10.5.125.64,10.5.125.66,10.5.125.67,10.5.125.68" ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAgEA8qg9fzb6pbl15hUdIB8vcUIIK4LcoY30iwL/eJ/8PJRtWc0MJloAfhjGGmmgf7sJl5Lt5VGoZqBw5x1thUfLpxN6FasEQnAnjsBCJM3dfF3OybOtdMAX7+2f29V4xJ+r1djdJnBM7FeXQ6zDFLAmGjFF4flUNmKa0PPmHjJqXWPoLvQq3mLWTCkscanWEA2WL+ceQ/0Bac3ZuHJQ7yPY+lD3p/hHTnQlito+rJVItGzLTkSxmiZaS3UwUSkxPBB3Tms8e7coALaY3AM9K91mnFi0oFFW+pCPdccjBUfHuzT6Khx0SrbKVUI57riUZs2+FDLiBSST7OT8nDlIgBwMWiwfjgURfvp5HOh46hc3gb0+DCxScXlocRMs6a47rC/EACN7HB4HjbrvBbfQts3GegU67Ia7d3vgDNRknp1sR/RjhYeTPxFAPqXJN5/hFyuHyFYTjavw0BZpSvZvQHgAVvUIZsHwEshvlwFkMU5qnqgnR6oMJxlZHqYw63K3iKeUsKsFJepSV5xb+r9pTtHL/bmsTuaEaPhUOiW4kgq8E0RQ5K20FERFsGNCEgfMoAOnLYYQX0vaYIemnGHTA0Y4bqgo39opVBj7dKipVp3uENmxeYTBSQJ9pRJhlmQxxfk0AupQbs77+K9dSLvMo0E10gHK3fa8r1FigAA4T/IDtLs= masher@releng2.fedora.phx.redhat.com

files/releng/update-fullfilelist

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# currently runs on releng2.fedora.phx.redhat.com
+
+MOD=$1
+[ -z "$MOD" ] && {
+	echo "usage: $0 <module>"
+	exit 1
+}
+
+TMPFILE=$(mktemp -p /tmp/)
+pushd /pub/$MOD > /dev/null
+find * -print > $TMPFILE
+if diff $TMPFILE fullfilelist > /dev/null; then
+	rm -f $TMPFILE
+else
+	mv $TMPFILE fullfilelist
+fi
+chmod 0644 fullfilelist
+popd > /dev/null

files/resolv.conf/phx2

@@ -0,0 +1,6 @@
+search phx2.fedoraproject.org vpn.fedoraproject.org fedoraproject.org
+nameserver 10.5.126.21
+nameserver 10.5.126.22
+options rotate timeout:1
+
+

files/rsyslog/rsyslog.conf.releng

@@ -0,0 +1,73 @@
+#rsyslog v3 config file
+
+# if you experience problems, check
+# http://www.rsyslog.com/troubleshoot for assistance
+
+#### MODULES ####
+
+$ModLoad imuxsock.so	# provides support for local system logging (e.g. via logger command)
+$ModLoad imklog.so	# provides kernel logging support (previously done by rklogd)
+#$ModLoad immark.so	# provides --MARK-- message capability
+
+# Provides UDP syslog reception
+#$ModLoad imudp.so
+#$UDPServerRun 514
+
+# Provides TCP syslog reception
+#$ModLoad imtcp.so  
+#$InputTCPServerRun 514
+
+
+#### GLOBAL DIRECTIVES ####
+
+# Use default timestamp format
+$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
+
+#### RULES ####
+
+# Log all kernel messages to the console.
+# Logging much else clutters up the screen.
+#kern.*                                                 /dev/console
+
+# Log anything (except mail) of level info or higher.
+# Don't log private authentication messages!
+*.info;local6.none;mail.none;authpriv.none;cron.none                /var/log/messages
+
+# The authpriv file has restricted access.
+authpriv.*                                              /var/log/secure
+
+# Log all the mail messages in one place.
+mail.*                                                  -/var/log/maillog
+
+
+# Log cron stuff
+cron.*                                                  /var/log/cron
+
+# Everybody gets emergency messages
+*.emerg                                                 *
+
+# Save news errors of level crit and higher in a special file.
+uucp,news.crit                                          /var/log/spooler
+
+# Save boot messages also to boot.log
+local7.*                                                /var/log/boot.log
+
+# ### begin forwarding rule ###
+# The statement between the begin ... end define a SINGLE forwarding
+# rule. They belong together, do NOT split them. If you create multiple
+# forwarding rules, duplicate the whole block!
+# Remote Logging (we use TCP for reliable delivery)
+#
+# An on-disk queue is created for this action. If the remote host is
+# down, messages are spooled to disk and sent when it is up again.
+$WorkDirectory /var/spool/rsyslog # where to place spool files
+$ActionQueueFileName fwdRule1     # unique name prefix for spool files
+$ActionQueueMaxDiskSpace 512m     # 512M space limit (use as much as possible)
+$ActionQueueSaveOnShutdown on     # save messages to disk on shutdown
+$ActionQueueType LinkedList       # run asynchronously
+$ActionResumeRetryCount -1        # infinite retries if host is down
+# remote host is: name/ip:port, e.g. 192.168.0.1:514, port optional
+#*.* @@remote-host:514
+# ### end of the forwarding rule ###
+cron.*;kern.*;authpriv.*;local7.*;*.info;local6.none              @@log02:514
+

files/ssh/sshd_config.releng

@@ -0,0 +1,138 @@
+#	$OpenBSD: sshd_config,v 1.80 2008/07/02 02:24:18 djm Exp $
+
+# This is the sshd server system-wide configuration file.  See
+# sshd_config(5) for more information.
+
+# This sshd was compiled with PATH=/usr/local/bin:/bin:/usr/bin
+
+# The strategy used for options in the default sshd_config shipped with
+# OpenSSH is to specify options with their default value where
+# possible, but leave them commented.  Uncommented options change a
+# default value.
+
+#Port 22
+#AddressFamily any
+#ListenAddress 0.0.0.0
+#ListenAddress ::
+
+# Disable legacy (protocol version 1) support in the server for new
+# installations. In future the default will change to require explicit
+# activation of protocol 1
+Protocol 2
+
+# HostKey for protocol version 1
+#HostKey /etc/ssh/ssh_host_key
+# HostKeys for protocol version 2
+#HostKey /etc/ssh/ssh_host_rsa_key
+#HostKey /etc/ssh/ssh_host_dsa_key
+
+# Lifetime and size of ephemeral version 1 server key
+#KeyRegenerationInterval 1h
+#ServerKeyBits 1024
+
+# Logging
+# obsoletes QuietMode and FascistLogging
+#SyslogFacility AUTH
+SyslogFacility AUTHPRIV
+LogLevel VERBOSE
+
+# Authentication:
+
+#LoginGraceTime 2m
+#PermitRootLogin yes
+StrictModes no
+#MaxAuthTries 6
+#MaxSessions 10
+
+#RSAAuthentication yes
+#PubkeyAuthentication yes
+#AuthorizedKeysFile	.ssh/authorized_keys
+#AuthorizedKeysCommand none
+#AuthorizedKeysCommandRunAs nobody
+
+# For this to work you will also need host keys in /etc/ssh/ssh_known_hosts
+#RhostsRSAAuthentication no
+# similar for protocol version 2
+#HostbasedAuthentication no
+# Change to yes if you don't trust ~/.ssh/known_hosts for
+# RhostsRSAAuthentication and HostbasedAuthentication
+#IgnoreUserKnownHosts no
+# Don't read the user's ~/.rhosts and ~/.shosts files
+#IgnoreRhosts yes
+
+# To disable tunneled clear text passwords, change to no here!
+#PasswordAuthentication yes
+#PermitEmptyPasswords no
+PasswordAuthentication yes
+
+# Change to no to disable s/key passwords
+#ChallengeResponseAuthentication yes
+ChallengeResponseAuthentication no
+
+# Kerberos options
+#KerberosAuthentication no
+#KerberosOrLocalPasswd yes
+#KerberosTicketCleanup yes
+#KerberosGetAFSToken no
+#KerberosUseKuserok yes
+
+# GSSAPI options
+#GSSAPIAuthentication no
+GSSAPIAuthentication yes
+#GSSAPICleanupCredentials yes
+GSSAPICleanupCredentials yes
+#GSSAPIStrictAcceptorCheck yes
+#GSSAPIKeyExchange no
+
+# Set this to 'yes' to enable PAM authentication, account processing, 
+# and session processing. If this is enabled, PAM authentication will 
+# be allowed through the ChallengeResponseAuthentication and
+# PasswordAuthentication.  Depending on your PAM configuration,
+# PAM authentication via ChallengeResponseAuthentication may bypass
+# the setting of "PermitRootLogin without-password".
+# If you just want the PAM account and session checks to run without
+# PAM authentication, then enable this but set PasswordAuthentication
+# and ChallengeResponseAuthentication to 'no'.
+#UsePAM no
+UsePAM yes
+
+# Accept locale-related environment variables
+AcceptEnv LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES
+AcceptEnv LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT
+AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE
+AcceptEnv XMODIFIERS
+
+#AllowAgentForwarding yes
+#AllowTcpForwarding yes
+#GatewayPorts no
+#X11Forwarding no
+X11Forwarding yes
+#X11DisplayOffset 10
+#X11UseLocalhost yes
+#PrintMotd yes
+#PrintLastLog yes
+#TCPKeepAlive yes
+#UseLogin no
+#UsePrivilegeSeparation yes
+#PermitUserEnvironment no
+#Compression delayed
+#ClientAliveInterval 0
+#ClientAliveCountMax 3
+#ShowPatchLevel no
+#UseDNS yes
+#PidFile /var/run/sshd.pid
+#MaxStartups 10
+#PermitTunnel no
+#ChrootDirectory none
+
+# no default banner path
+#Banner none
+
+# override default of no subsystems
+Subsystem	sftp	/usr/libexec/openssh/sftp-server
+
+# Example of overriding settings on a per-user basis
+#Match User anoncvs
+#	X11Forwarding no
+#	AllowTcpForwarding no
+#	ForceCommand cvs server

playbooks/groups/releng.yml

@@ -0,0 +1,28 @@
+# create a new releng system
+# NOTE: should be used with --limit most of the time
+# NOTE: make sure there is room/space for this instance on the buildvmhost
+# NOTE: most of these vars come from group_vars/releng or from hostvars
+
+- name: make releng systems
+  hosts: releng
+  user: root
+  gather_facts: False
+
+  vars_files: 
+   - /srv/web/infra/ansible/vars/global.yml
+   - ${private}/vars.yml
+   - ${vars}/${ansible_distribution}.yml
+
+  tasks:
+  - include: $tasks/virt_instance_create.yml
+  - include: $tasks/koji/releng_config.yml
+
+  handlers:
+  - include: $handlers/restart_services.yml
+
+  - name: restart kojid
+    action: service name=kojid state=restarted
+  
+
+
+

tasks/koji/base_builder.yml

@@ -54,7 +54,7 @@
   - restart rsyslog
 
 - name: /etc/postfix/main.cf
-  action: copy src=$files/postfix/main.cf.kojibuilder dest=/etc/postfix/main.cf
+  action: copy src=$files/postfix/main.cf.norelay dest=/etc/postfix/main.cf
   notify:
   - restart postfix
 

tasks/koji/releng_config.yml

@@ -0,0 +1,205 @@
+- name: set root passwd
+  action: user name=root password=$builder_rootpw state=present
+
+- name: add mock user as 425
+  action: user name=mock uid=425 state=present home=/var/lib/mock createhome=yes system=yes
+- name: make mock homedir 2775
+  action: file state=directory path=/var/lib/mock mode=2775 owner=mock group=mock
+- name: add mock ssh dir
+  action: file state=directory path=/var/lib/mock/.ssh mode=700 owner=mock group=mock
+- name: add mock ssh keys
+  action: copy src=$files/kojibuilder/mock_auth_keys dest=/var/lib/mock/.ssh/authorized_keys mode=644 owner=mock group=mock
+
+- name: add ftpsync group
+  action: group name=ftpsync gid=263 system=yes state=present
+
+- name: add ftpsync user
+  action: user name=ftpsync uid=263 group=ftpsync system=yes home=/var/tmp/ftpsync createhome=yes system=yes state=present
+
+- name: add the ftpsync update-fullfilelist script
+  action: copy src=$files/releng/update-fullfilelist dest=/usr/local/bin/update-fullfilelist owner=ftpsync group=ftpsync mode=555
+
+- name: add masher group
+  action: group name=masher gid=751 system=yes state=present
+# masher user 751
+- name: add masher user as 751 - and group
+  action: user name=masher uid=751 group=masher home=/home/masher groups=mock,ftpsync
+# masher ssh keys and config
+- name: add masher ssh dir
+  action: file state=directory path=/home/masher/.ssh mode=700 owner=masher group=masher
+- name: add masher ssh keys
+  action: copy src=$files/releng/mash/masher.$item dest=/home/masher/.ssh/$item mode=600 owner=masher group=masher
+  with_items:
+    - id_rsa.pub
+    - config
+
+- name: add masher ssh priv key
+  action: copy src=$private/mash/masher.id_rsa dest=/home/masher/.ssh/id_rsa mode=600 owner=masher group=masher
+
+
+# rawhide group
+- name: rawhide group
+  action: group name=rawhide gid=265
+# rawhide user 265
+- name: add rawhide user
+  action: user name=rawhide uid=265 group=rawhide home=/tmp comment="rawhide compose account"
+ 
+
+- name: iptables
+  action: copy src=$files/iptables/releng dest=/etc/sysconfig/iptables mode=600
+  notify:
+  - restart iptables
+  tags:
+  - configs
+
+- name: sshd_config
+  action: copy src=$files/sshd_config.releng dest=/etc/ssh/sshd_config mode=600
+  notify:
+  - restart sshd
+  tags:
+  - sshd_config
+  - configs
+
+- name: /etc/resolv.conf
+  action: copy src=$files/resolv.conf/phx2 dest=/etc/resolv.conf
+  tags:
+  - configs
+
+- name: /etc/hosts
+  action: copy src=$files/hosts/releng-hostsdest=/etc/hosts
+  tags:
+  - configs
+
+
+- name: rsyslog.conf
+  action: copy src=$files/rsyslog.conf.releng dest=/etc/rsyslog.conf mode=644
+  tags:
+  - configs
+
+- name: /etc/postfix/main.cf
+  action: copy src=$files/postfix/main.cf.norelay dest=/etc/postfix/main.cf
+  tags:
+  - configs
+
+- name: make a bunch of dirs
+  action: file state=directory path=$item
+  with_items:
+    - /pub
+    - /mnt/koji
+    - /pub/fedora
+    - /pub/epel
+    - /epel
+    - /var/spool/rsyslog
+
+- name: add builder infra yum repo
+  action: copy src=$files/kojibuilder/builder-infrastructure.repo dest=/etc/yum.repos.d/builder-infrastructure.repo
+  tags:
+  - configs
+
+- name: add releng yum repo
+  action: copy src=$files/releng/releng.repo dest=/etc/yum.repos.d/releng.repo
+  tags:
+  - configs
+
+- name: clean up packages we do not need
+  action: yum state=removed pkg=$item
+  with_items:
+    - \*firmware\*
+
+- name: add pkgs
+  action: yum  state=installed pkg=$item
+  with_items:
+    - yum-utils
+    - koji
+    - strace
+    - mock
+    - nfs-utils
+    - git
+    - mash
+    - intltool
+    - mutt
+    - koji
+    - createrepo
+
+- name: /etc/koji/koji.conf
+  action: copy src=$files/kojibuilder/koji.conf dest=/etc/koji.conf
+
+
+# mock configs
+- name: put extra special  mock configs in
+  action: copy src=$files/releng/$item dest=/etc/mock/$item mode=644
+  with_items:
+    - fedora-branched-compose-i386.cfg
+    - fedora-rawhide-compose-i386.cfg
+    - fedora-branched-compose-x86_64.cfg
+
+- name: nfs mount - /mnt/koji
+  action: mount name=/mnt/koji src=nfs01.phx2.fedoraproject.org:/ fstype=nfs4 opts=rw,soft,intr passno=0 dump=0 state=present
+- name: route to netapp network
+  action: copy src=$files/kojibuilder/route-eth1 dest=/etc/sysconfig/network-scripts/route-eth1
+
+- name: nfsmount - /pub/fedora
+  action: mount name=/pub/fedora src=vtap-fedora-nfs01.storage.phx2.redhat.com:/vol/fedora_ftp/fedora.redhat.com/pub/fedora fstype=nfs opts=rw,hard,bg,intr,noatime,nodev,nosuid,nfsvers=3 passno=0 dump=0 state=present  
+
+
+# idmapd and make sure it's set to run
+- name: idmapd.conf
+  action: copy src=$files/kojibuilder/idmapd.conf dest=/etc/idmapd.conf
+  tags:
+  - configs
+- name: rpcidmapd on
+  action: service name=rpcidmapd  enabled=true
+
+# put cron job in for branched compose
+- name: branched compose cron
+  action: copy src=$files/releng/branched dest=/etc/cron.d/branched
+
+- name: sudoers defaults
+  action: copy src=$private/files/sudo/releng-sudoers dest=/etc/sudoers mode=0440
+  tags:
+  - configs
+- name: sudoers for ftpsync
+  action: copy src=$private/files/sudo/ftpsync-sudo dest=/etc/sudoers.d/ftpsync mode=0440
+  tags:
+  - configs
+
+# fedmsg message buss stuff. 
+- name: fedmsg install
+  action: yum  state=installed pkg=$item
+  with_items:
+    - fedmsg
+    # needed for now, until moksha 0.8.8-4 pulls it in.
+    - python-bunch
+  tags:
+  - fedmsg
+- name: fedmsg.d dir
+  action: file state=directory path=/etc/fedmsg.d mode=755 owner=root group=root
+  tags:
+  - fedmsg
+- name: fedmsg cert dir
+  action: file state=directory path=/etc/pki/fedmsg mode=755 owner=root group=root
+  tags:
+  - fedmsg
+- name: fedmsg files
+  action: copy src=$files/releng/fedmsg/$item dest=/etc/fedmsg.d/$item mode=644
+  with_items:
+    - ircbot.py
+    - ssl.py
+    - endpoints.py
+    - base.py
+  tags:
+  - fedmsg
+- name: fedmsg ca and crt
+  action: copy src=$private/files/fedmsg-certs/$item dest=/etc/pki/fedmsg/$item mode=644
+  with_items:
+    - ca.crt
+    - bodhi-releng01.phx2.fedoraproject.org.crt
+  tags:
+  - fedmsg
+- name: fedmsg key
+  action: copy src=$private/files/fedmsg-certs/$item dest=/etc/pki/fedmsg/$item mode=640 group=masher
+  with_items:
+    - bodhi-releng01.phx2.fedoraproject.org.key
+  tags:
+  - fedmsg
+