Add cron jobs for bodhi-backend

roles/bodhi/backend/files/fedora-epel-push

@@ -0,0 +1,111 @@
+#!/bin/sh
+
+SOURCE=/mnt/koji/mash/updates
+DEST=/pub/epel/
+
+OPTIONS="-rlptDvHh --stats --delay-updates $RSYNC_OPTS"
+
+for rel in 5 6; do
+
+    OUTPUT1=$(rsync $OPTIONS --exclude "repodata/*" --exclude "headers/*" \
+            $SOURCE/el$rel-epel/ $DEST/$rel/)
+    OUTPUT2=$(rsync $OPTIONS --delete --delete-delay \
+            $SOURCE/el$rel-epel/ $DEST/$rel/)
+
+    # Grep out some signals from the stats
+    bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ')
+    deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l)
+
+    # If anything changed, then publish a fedmsg message as bodhi.updates.sync
+    if [ "$bytes" != "0" -o "$deleted" != "0" ]; then
+      echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"epel\", \"release\": \"$rel\"}" | fedmsg-logger \
+          --cert-prefix ftpsync \
+          --modname bodhi \
+          --topic updates.epel.sync \
+          --json-input &> /dev/null
+    fi
+
+    OUTPUT1=$(rsync $OPTIONS --exclude "repodata/*" --exclude "headers/*" \
+            $SOURCE/el$rel-epel-testing/ $DEST/testing/$rel/)
+    OUTPUT2=$(rsync $OPTIONS --delete --delete-delay \
+            $SOURCE/el$rel-epel-testing/ $DEST/testing/$rel/)
+
+    # Grep out some signals from the stats
+    bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ')
+    deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l)
+
+    # If anything changed, then publish a fedmsg message as bodhi.updates.sync
+    if [ "$bytes" != "0" -o "$deleted" != "0" ]; then
+      echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"epel-testing\", \"release\": \"$rel\"}" | fedmsg-logger \
+          --cert-prefix ftpsync \
+          --modname bodhi \
+          --topic updates.epel.sync \
+          --json-input &> /dev/null
+    fi
+done
+
+for rel in 7; do
+
+    OUTPUT1=$(rsync $OPTIONS --exclude "repodata/*" \
+            $SOURCE/epel$rel/ $DEST/$rel/)
+    OUTPUT2=$(rsync $OPTIONS --delete --delete-delay \
+            $SOURCE/epel$rel/ $DEST/$rel/)
+
+    # Grep out some signals from the stats
+    bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ')
+    deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l)
+
+    # If anything changed, then publish a fedmsg message as bodhi.updates.sync
+    if [ "$bytes" != "0" -o "$deleted" != "0" ]; then
+      echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"epel\", \"release\": \"$rel\"}" | fedmsg-logger \
+          --cert-prefix ftpsync \
+          --modname bodhi \
+          --topic updates.epel.sync \
+          --json-input &> /dev/null
+    fi
+
+    OUTPUT1=$(rsync $OPTIONS --exclude "repodata/*" \
+            $SOURCE/epel$rel-testing/ $DEST/testing/$rel/)
+    OUTPUT2=$(rsync $OPTIONS --delete --delete-delay \
+            $SOURCE/epel$rel-testing/ $DEST/testing/$rel/)
+
+    # Grep out some signals from the stats
+    bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ')
+    deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l)
+
+    # If anything changed, then publish a fedmsg message as bodhi.updates.sync
+    if [ "$bytes" != "0" -o "$deleted" != "0" ]; then
+      echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"epel-testing\", \"release\": \"$rel\"}" | fedmsg-logger \
+          --cert-prefix ftpsync \
+          --modname bodhi \
+          --topic updates.epel.sync \
+          --json-input &> /dev/null
+    fi
+done
+
+for rel in 5 6 7; do
+    if [ ${rel} -eq 7 ]; then
+	TARGET_DIR=${DEST}/${rel}/x86_64/e
+    else
+	TARGET_DIR=${DEST}/${rel}/x86_64
+    fi
+
+    if [ -f ${TARGET_DIR}/epel-release*rpm ]; then
+	# We have a file to match. [This may sort wrong at -9 -> -10]
+	CANDIDATE=$( ls ${TARGET_DIR}/epel-release-*rpm | sort | tail -n 1)
+	TARGET=${DEST}/epel-release-latest-${rel}.noarch.rpm
+	# Does our symbolic link exist?
+	if [ -L ${TARGET} ]; then
+	    # check to see if the link matches the candidate
+	    TEST=$( readlink ${TARGET} )
+	    if [ ${TEST} != ${CANDIDATE} ]; then
+		ln -sf $(echo ${CANDIDATE}|sed -e "s|$DEST|./|g" -e 's|//|/|g') ${TARGET}
+	    fi
+	else
+	    # first time for everything.
+            ln -sf $(echo ${CANDIDATE}|sed -e "s|$DEST|./|g" -e 's|//|/|g') ${TARGET}
+	fi
+    else
+	echo "No target file for epel-release ${rel} to link against."
+    fi
+done

roles/bodhi/backend/files/fedora-updates-push

@@ -0,0 +1,72 @@
+#!/bin/sh
+
+SOURCE=/mnt/koji/mash/updates
+DEST=/pub/fedora/linux/updates/
+ATOMICSOURCE=/mnt/koji/mash/atomic/
+ATOMICDEST=/pub/fedora/linux/atomic/
+
+OPTIONS="-rlptDvHh --stats --delay-updates $RSYNC_OPTS"
+
+for rel in 20 21 22; do
+
+    OUTPUT1=$(rsync $OPTIONS --exclude "repodata/*" \
+            $SOURCE/f$rel-updates/ $DEST/$rel/ --link-dest $DEST/testing/$rel/)
+    OUTPUT2=$(rsync $OPTIONS --delete --delete-delay --exclude=Live --exclude=Images \
+            $SOURCE/f$rel-updates/ $DEST/$rel/)
+
+    # Grep out some signals from the stats
+    bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ')
+    deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l)
+
+    # If anything changed, then publish a fedmsg message as bodhi.updates.sync
+    if [ "$bytes" != "0" -o "$deleted" != "0" ]; then
+      echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"updates\", \"release\": \"$rel\"}" | fedmsg-logger \
+          --cert-prefix ftpsync \
+          --modname bodhi \
+          --topic updates.fedora.sync \
+          --json-input &> /dev/null
+    fi
+
+done
+for rel in 20 21 22; do
+
+    OUTPUT1=$(rsync $OPTIONS --exclude "repodata/*" \
+            $SOURCE/f$rel-updates-testing/ $DEST/testing/$rel/)
+    OUTPUT2=$(rsync $OPTIONS --delete --delete-delay --exclude=Live --exclude=Images \
+            $SOURCE/f$rel-updates-testing/ $DEST/testing/$rel/)
+
+    # Grep out some signals from the stats
+    bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ')
+    deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l)
+
+    # If anything changed, then publish a fedmsg message as bodhi.updates.sync
+    if [ "$bytes" != "0" -o "$deleted" != "0" ]; then
+      echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"updates-testing\", \"release\": \"$rel\"}" | fedmsg-logger \
+          --cert-prefix ftpsync \
+          --modname bodhi \
+          --topic updates.fedora.sync \
+          --json-input &> /dev/null
+    fi
+
+done
+for rel in 21 22; do
+
+    OUTPUT1=$(rsync $OPTIONS --ignore-existing \
+            $ATOMICSOURCE/$rel/objects/ $ATOMICDEST/$rel/objects/)
+    OUTPUT2=$(rsync $OPTIONS --delete --delete-delay --exclude=objects/ \
+            $ATOMICSOURCE/$rel/ $ATOMICDEST/$rel/)
+
+    # Grep out some signals from the stats
+    bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ')
+    deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l)
+
+    # If anything changed, then publish a fedmsg message as bodhi.updates.sync
+    if [ "$bytes" != "0" -o "$deleted" != "0" ]; then
+      echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"atomic\", \"release\": \"$rel\"}" | fedmsg-logger \
+          --cert-prefix ftpsync \
+          --modname bodhi \
+          --topic updates.fedora.sync \
+          --json-input &> /dev/null
+    fi
+done
+

roles/bodhi/backend/tasks/main.yml

@@ -134,3 +134,81 @@
   - restart httpd
   tags:
   - config
+
+#
+# cron job that syncs packages to koji
+#
+- name: put owner-sync-pkgdb in place
+  template: src=owner-sync-pkgdb.j2 dest=/usr/local/bin/owner-sync-pkgdb mode=0755
+  tags:
+  - config
+
+- name: sync packages from pkgdb2 to koji (el5)
+  cron: name="owner-sync-el5" minute="7,17,27,37,47,57" user="root" 
+        job="/usr/local/bin/owner-sync-pkgdb dist-5E-epel"
+        cron_file=update-koji-owner-EL-5
+  when: inventory_hostname.startswith('bodhi-backend01') and env == "production"
+
+- name: sync packages from pkgdb2 to koji (el6)
+  cron: name="owner-sync-el5" minute="7,17,27,37,47,57" user="root" 
+        job="/usr/local/bin/owner-sync-pkgdb dist-6E-epel"
+        cron_file=update-koji-owner-EL-6
+  when: inventory_hostname.startswith('bodhi-backend01') and env == "production"
+  
+- name: sync packages from pkgdb2 to koji (epel7)
+  cron: name="owner-sync-el5" minute="7,17,27,37,47,57" user="root" 
+        job="/usr/local/bin/owner-sync-pkgdb epel7"
+        cron_file=update-koji-owner-epel7
+  when: inventory_hostname.startswith('bodhi-backend01') and env == "production"
+  
+- name: sync packages from pkgdb2 to koji (f20)
+  cron: name="owner-sync-el5" minute="7,17,27,37,47,57" user="root" 
+        job="/usr/local/bin/owner-sync-pkgdb f20"
+        cron_file=update-koji-owner-f20
+  when: inventory_hostname.startswith('bodhi-backend01')
+
+#
+# cron job that syncs updates to master mirror
+#
+
+- name: put fedora-updates-push in place
+  copy: src=fedora-updates-push dest=/usr/local/bin/fedora-updates-push mode=0755
+  tags:
+  - config
+  when: inventory_hostname.startswith('bodhi-backend01') and env == "production"
+
+- name: put fedora-epel-push in place
+  copy: src=fedora-epel-push dest=/usr/local/bin/fedora-epel-push mode=0755
+  tags:
+  - config
+  when: inventory_hostname.startswith('bodhi-backend02') and env == "production"
+
+- name: put update-fullfilelist in place
+  copy: src=update-fullfilelist dest=/usr/local/bin/update-fullfilelist mode=0755
+  tags:
+  - config
+  when: inventory_hostname.startswith('bodhi-backend01') and env == "production"
+
+- name: Updates sync cron job. 
+  cron: name="updates-sync" minute="15,45" user="ftpsync"
+        job="/usr/local/bin/lock-wrapper fedora-updates-push '/usr/local/bin/fedora-updates-push && /usr/local/bin/update-fullfilelist fedora"
+        cron_file=updates-sync
+  when: inventory_hostname.startswith('bodhi-backend01') and env == "production"
+  tags:
+  - config
+
+- name: epel Updates sync cron job. 
+  cron: name="epel-updates-sync" minute="15,45" user="ftpsync"
+        job="/usr/local/bin/lock-wrapper fedora-epel-push '/usr/local/bin/fedora-epel-push && /usr/local/bin/update-fullfilelist epel"
+        cron_file=updates-sync
+  when: inventory_hostname.startswith('bodhi-backend02') and env == "production"
+  tags:
+  - config
+
+- name: directory sizes update cron job. 
+  cron: name="directory-sizes-update" minute="30" hour="19" user="ftpsync"
+        job="/usr/bin/find /srv/pub/alt/ /srv/pub/archive/ /srv/pub/fedora-secondary/ /srv/pub/fedora/ /srv/pub/epel/ -type d ! -path '/srv/pub/fedora/.snapshot*' ! -path '/srv/pub/epel/.snapshot*' ! -path '/srv/pub/alt/.snapshot*' ! -path '/srv/pub/archive/.snapshot*' ! -path '/srv/pub/fedora-secondary/.snapshot*' ! -path '/srv/pub/alt/stage*' ! -path '/srv/pub/alt/tmp' ! -path '/srv/pub/alt/screenshots/f21/source' | grep -v snapshot | /usr/bin/xargs -n 1 /usr/bin/du --exclude=.snapshot --exclude=stage -sh > /tmp/DIRECTORY_SIZES.txt 2> /dev/null; cp /tmp/DIRECTORY_SIZES.txt /srv/pub/"
+        cron_file=directory-sizes-update
+  when: inventory_hostname.startswith('bodhi-backend02') and env == "production"
+  tags:
+  - config

roles/bodhi/backend/templates/owner-sync-pkgdb.j2

@@ -0,0 +1,197 @@
+#!/usr/bin/python2
+
+# cronjobs are run on releng01.stg
+# Looks like:
+# /usr/local/bin/owner-sync-pkgdb f19 
+# /usr/local/bin/owner-sync-pkgdb dist-5E-epel 
+# /usr/local/bin/owner-sync-pkgdb dist-6E-epel
+# /usr/local/bin/owner-sync-pkgdb epel7 
+
+import sys
+import os
+import ConfigParser
+from urlparse import urljoin
+
+import requests
+
+DEBUG=False
+VERIFY=True
+{% if env == 'staging' %}
+BASEURL = os.environ.get('PACKAGEDBURL') or 'https://admin.stg.fedoraproject.org/pkgdb/'
+{% else %}
+BASEURL = os.environ.get('PACKAGEDBURL') or 'https://admin.fedoraproject.org/pkgdb/'
+{% endif %}
+if not BASEURL.endswith('/'):
+    BASEURL = BASEURL + '/'
+
+# Why do we have this?  Seems insecure....
+sys.path.append('.')
+
+try:
+    import koji
+except:
+    import brew as koji
+
+extraArchList = {'kernel': ('i586', 'i686', 'noarch'),
+                'kernel-xen-2.6': ('i586', 'i686', 'noarch'),
+                'glibc': ('i686',),
+                'openssl': ('i686',),
+                'em8300-kmod': ('i586', 'i686'),
+                'sysprof-kmod': ('i586', 'i686'),
+                }
+
+def usage():
+    print "Usage: owner-sync <tag>"
+    print "    <tag>: tag to synchronize owners for"
+    sys.exit(1)
+
+def get_options():
+    # shamelessly stolen from koji CLI
+    opts = {
+        'server': 'http://koji.fedoraproject.org/kojihub',
+        'weburl': 'http://koji.fedoraproject.org/koji',
+        'cert': '/etc/pki/pkgdb/pkgdb.pem',
+        'ca': '/etc/pki/pkgdb/fedora-server-ca.cert',
+        'serverca': '/etc/pki/pkgdb/fedora-server-ca.cert'
+    }
+    for configFile in ('/etc/koji.conf', os.path.expanduser('~/.koji/config')):
+        if os.access(configFile, os.F_OK):
+            f = open(configFile)
+            config = ConfigParser.ConfigParser()
+            config.readfp(f)
+            f.close()
+            if config.has_section('koji'):
+                for name, value in config.items('koji'):
+                    if opts.has_key(name):
+                        opts[name] = value
+    for entry in opts.keys():
+        if entry == 'server' or entry == 'weburl':
+            pass
+        opts[entry] = os.path.expanduser(opts[entry])
+    return opts
+
+if __name__ == '__main__':
+    try:
+        tag=sys.argv[1]
+    except:
+        print "ERROR: no tag specified!\n"
+        usage()
+
+    if tag.endswith('epel') or tag.startswith('epel'):
+        if tag.startswith('epel'):
+            version = tag.split('epel')[1]
+        else:
+            version = tag.split('-')[1][:-1]
+
+        data = requests.get(urljoin(BASEURL, 'api/collections'), verify=VERIFY).json()
+        branch_names = set()
+        for collection in (c for c in data['collections'] if c['status'] != 'EOL'):
+            ### TODO: check with pingou that this is now returning the same
+            # format as the collection names in api/vcs
+            # By moving the data from gitbranchname into branchname, I think
+            # that the data will now match
+            branch_names.add(collection['branchname'])
+
+        if tag.startswith('epel'):
+            # Ex: epel7 => epel7
+            reponame = tag
+        else:
+            # Ex: dist-6E-epel => el6
+            reponame = 'el%s' % version
+        if reponame not in branch_names:
+            print 'tag %s => repo %s: does not seem to be a non-EOL branch' % (tag, reponame)
+            sys.exit(1)
+
+        # EPEL needs a separate entry in koji for each epel version
+
+        data = requests.get(urljoin(BASEURL, 'api/vcs?format=json'), verify=VERIFY).json()
+        acls = data['packageAcls']
+        pkgs = {}
+        for pkg_name in acls:
+            try:
+                owners = acls[pkg_name][reponame]
+            except KeyError:
+                # Package is not branched for this release
+                continue
+            if len(owners['commit']['people']):
+                # Arbitrarily take the first committer listed as the owner in
+                # koji
+                pkgs[pkg_name] = owners['commit']['people'][0]
+            else:
+                pkgs[pkg_name] = 'orphan'
+        pkgList = pkgs.keys()
+        BuildEPEL = True
+        arches = ["primary"]
+    else:
+        # Fedora only needs one entry per package for all Fedora releases
+        # Use the owner from bugzilla for simplicity
+        data = requests.get(urljoin(BASEURL, 'api/bugzilla?format=json'), verify=VERIFY).json()
+        acls = data['bugzillaAcls']
+        pkgList = acls['Fedora'].keys()
+        pkgs = {}
+        for pkg in acls['Fedora']:
+            owner = acls['Fedora'][pkg]['owner']
+            owner = owner.replace('group::', '').replace('@', '')
+            pkgs[pkg] = owner
+
+        #pkgs = dict(((p, acls['Fedora'][p]['owner']) for p in acls['Fedora']))
+        BuildEPEL = False
+{% if env == 'staging' %}
+        arches = ["primary"]
+{% else %}
+        arches = ["primary", "arm", "ppc", "s390"]
+{% endif %}
+    pkgList.sort()
+
+    options = get_options()
+
+    for arch in arches:
+        if arch == "primary":
+{% if env == 'staging' %}
+            session = koji.ClientSession("http://koji.stg.fedoraproject.org/kojihub")
+{% else %}
+            session = koji.ClientSession("http://koji.fedoraproject.org/kojihub")
+{% endif %}
+        else:
+            session = koji.ClientSession("http://%s.koji.fedoraproject.org/kojihub" % arch)
+        try:
+            session.ssl_login(options['cert'], options['ca'], options['serverca'])
+        except:
+            print "Unable to sync to %s hub" % arch
+            continue
+        kojitag = session.getTag(tag)
+        if kojitag is None:
+            print "ERROR: tag %s does not exist!\n" % (tag)
+            usage()
+
+        kojipkgs = {}
+        kojiusers = [user['name'] for user in session.listUsers()]
+
+        for p in session.listPackages(tagID=tag, inherited = True):
+            kojipkgs[p['package_name']] = p
+
+        for pkg in pkgList:
+            owner = pkgs[pkg]
+            if DEBUG:
+                print '[DEBUG] Package: %s, Owner: %s' % (pkg, owner)
+
+            if not owner in kojiusers:
+                # add the user first
+                if DEBUG:
+                    print "Adding user %s" % owner
+                else:
+                    session.createUser(owner)
+                    kojiusers.append(owner)
+            if not kojipkgs.has_key(pkg):
+                if DEBUG:
+                    print "Adding package %s for %s with owner %s" % (pkg, tag, owner)
+                else:
+                    extraArches = None
+                    if pkg in extraArchList:
+                        extraArches = extraArchList[pkg]
+                    session.packageListAdd(tag, pkg, owner = owner, extra_arches=extraArches)
+            elif kojipkgs[pkg]['owner_name'] != owner:
+                if DEBUG:
+                    print "Setting owner for %s in %s to %s" % (pkg, tag, owner)
+                else:
+                    session.packageListSetOwner(tag, pkg, owner, force = True)