From a1c718de54fa7b3bc66fdd3238361cfbc64d5b20 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 1 Jun 2015 20:32:06 +0000 Subject: [PATCH] Add cron jobs for bodhi-backend --- roles/bodhi/backend/files/fedora-epel-push | 111 ++++++++++ roles/bodhi/backend/files/fedora-updates-push | 72 +++++++ roles/bodhi/backend/tasks/main.yml | 78 +++++++ .../backend/templates/owner-sync-pkgdb.j2 | 197 ++++++++++++++++++ 4 files changed, 458 insertions(+) create mode 100755 roles/bodhi/backend/files/fedora-epel-push create mode 100755 roles/bodhi/backend/files/fedora-updates-push create mode 100755 roles/bodhi/backend/templates/owner-sync-pkgdb.j2 diff --git a/roles/bodhi/backend/files/fedora-epel-push b/roles/bodhi/backend/files/fedora-epel-push new file mode 100755 index 0000000000..169b2107a7 --- /dev/null +++ b/roles/bodhi/backend/files/fedora-epel-push @@ -0,0 +1,111 @@ +#!/bin/sh + +SOURCE=/mnt/koji/mash/updates +DEST=/pub/epel/ + +OPTIONS="-rlptDvHh --stats --delay-updates $RSYNC_OPTS" + +for rel in 5 6; do + + OUTPUT1=$(rsync $OPTIONS --exclude "repodata/*" --exclude "headers/*" \ + $SOURCE/el$rel-epel/ $DEST/$rel/) + OUTPUT2=$(rsync $OPTIONS --delete --delete-delay \ + $SOURCE/el$rel-epel/ $DEST/$rel/) + + # Grep out some signals from the stats + bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ') + deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l) + + # If anything changed, then publish a fedmsg message as bodhi.updates.sync + if [ "$bytes" != "0" -o "$deleted" != "0" ]; then + echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"epel\", \"release\": \"$rel\"}" | fedmsg-logger \ + --cert-prefix ftpsync \ + --modname bodhi \ + --topic updates.epel.sync \ + --json-input &> /dev/null + fi + + OUTPUT1=$(rsync $OPTIONS --exclude "repodata/*" --exclude "headers/*" \ + $SOURCE/el$rel-epel-testing/ $DEST/testing/$rel/) + OUTPUT2=$(rsync $OPTIONS --delete --delete-delay \ + $SOURCE/el$rel-epel-testing/ $DEST/testing/$rel/) + + # Grep out some signals from the stats + bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ') + deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l) + + # If anything changed, then publish a fedmsg message as bodhi.updates.sync + if [ "$bytes" != "0" -o "$deleted" != "0" ]; then + echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"epel-testing\", \"release\": \"$rel\"}" | fedmsg-logger \ + --cert-prefix ftpsync \ + --modname bodhi \ + --topic updates.epel.sync \ + --json-input &> /dev/null + fi +done + +for rel in 7; do + + OUTPUT1=$(rsync $OPTIONS --exclude "repodata/*" \ + $SOURCE/epel$rel/ $DEST/$rel/) + OUTPUT2=$(rsync $OPTIONS --delete --delete-delay \ + $SOURCE/epel$rel/ $DEST/$rel/) + + # Grep out some signals from the stats + bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ') + deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l) + + # If anything changed, then publish a fedmsg message as bodhi.updates.sync + if [ "$bytes" != "0" -o "$deleted" != "0" ]; then + echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"epel\", \"release\": \"$rel\"}" | fedmsg-logger \ + --cert-prefix ftpsync \ + --modname bodhi \ + --topic updates.epel.sync \ + --json-input &> /dev/null + fi + + OUTPUT1=$(rsync $OPTIONS --exclude "repodata/*" \ + $SOURCE/epel$rel-testing/ $DEST/testing/$rel/) + OUTPUT2=$(rsync $OPTIONS --delete --delete-delay \ + $SOURCE/epel$rel-testing/ $DEST/testing/$rel/) + + # Grep out some signals from the stats + bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ') + deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l) + + # If anything changed, then publish a fedmsg message as bodhi.updates.sync + if [ "$bytes" != "0" -o "$deleted" != "0" ]; then + echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"epel-testing\", \"release\": \"$rel\"}" | fedmsg-logger \ + --cert-prefix ftpsync \ + --modname bodhi \ + --topic updates.epel.sync \ + --json-input &> /dev/null + fi +done + +for rel in 5 6 7; do + if [ ${rel} -eq 7 ]; then + TARGET_DIR=${DEST}/${rel}/x86_64/e + else + TARGET_DIR=${DEST}/${rel}/x86_64 + fi + + if [ -f ${TARGET_DIR}/epel-release*rpm ]; then + # We have a file to match. [This may sort wrong at -9 -> -10] + CANDIDATE=$( ls ${TARGET_DIR}/epel-release-*rpm | sort | tail -n 1) + TARGET=${DEST}/epel-release-latest-${rel}.noarch.rpm + # Does our symbolic link exist? + if [ -L ${TARGET} ]; then + # check to see if the link matches the candidate + TEST=$( readlink ${TARGET} ) + if [ ${TEST} != ${CANDIDATE} ]; then + ln -sf $(echo ${CANDIDATE}|sed -e "s|$DEST|./|g" -e 's|//|/|g') ${TARGET} + fi + else + # first time for everything. + ln -sf $(echo ${CANDIDATE}|sed -e "s|$DEST|./|g" -e 's|//|/|g') ${TARGET} + fi + else + echo "No target file for epel-release ${rel} to link against." + fi +done diff --git a/roles/bodhi/backend/files/fedora-updates-push b/roles/bodhi/backend/files/fedora-updates-push new file mode 100755 index 0000000000..1313337ac0 --- /dev/null +++ b/roles/bodhi/backend/files/fedora-updates-push @@ -0,0 +1,72 @@ +#!/bin/sh + +SOURCE=/mnt/koji/mash/updates +DEST=/pub/fedora/linux/updates/ +ATOMICSOURCE=/mnt/koji/mash/atomic/ +ATOMICDEST=/pub/fedora/linux/atomic/ + +OPTIONS="-rlptDvHh --stats --delay-updates $RSYNC_OPTS" + +for rel in 20 21 22; do + + OUTPUT1=$(rsync $OPTIONS --exclude "repodata/*" \ + $SOURCE/f$rel-updates/ $DEST/$rel/ --link-dest $DEST/testing/$rel/) + OUTPUT2=$(rsync $OPTIONS --delete --delete-delay --exclude=Live --exclude=Images \ + $SOURCE/f$rel-updates/ $DEST/$rel/) + + # Grep out some signals from the stats + bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ') + deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l) + + # If anything changed, then publish a fedmsg message as bodhi.updates.sync + if [ "$bytes" != "0" -o "$deleted" != "0" ]; then + echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"updates\", \"release\": \"$rel\"}" | fedmsg-logger \ + --cert-prefix ftpsync \ + --modname bodhi \ + --topic updates.fedora.sync \ + --json-input &> /dev/null + fi + +done +for rel in 20 21 22; do + + OUTPUT1=$(rsync $OPTIONS --exclude "repodata/*" \ + $SOURCE/f$rel-updates-testing/ $DEST/testing/$rel/) + OUTPUT2=$(rsync $OPTIONS --delete --delete-delay --exclude=Live --exclude=Images \ + $SOURCE/f$rel-updates-testing/ $DEST/testing/$rel/) + + # Grep out some signals from the stats + bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ') + deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l) + + # If anything changed, then publish a fedmsg message as bodhi.updates.sync + if [ "$bytes" != "0" -o "$deleted" != "0" ]; then + echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"updates-testing\", \"release\": \"$rel\"}" | fedmsg-logger \ + --cert-prefix ftpsync \ + --modname bodhi \ + --topic updates.fedora.sync \ + --json-input &> /dev/null + fi + +done +for rel in 21 22; do + + OUTPUT1=$(rsync $OPTIONS --ignore-existing \ + $ATOMICSOURCE/$rel/objects/ $ATOMICDEST/$rel/objects/) + OUTPUT2=$(rsync $OPTIONS --delete --delete-delay --exclude=objects/ \ + $ATOMICSOURCE/$rel/ $ATOMICDEST/$rel/) + + # Grep out some signals from the stats + bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ') + deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l) + + # If anything changed, then publish a fedmsg message as bodhi.updates.sync + if [ "$bytes" != "0" -o "$deleted" != "0" ]; then + echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"atomic\", \"release\": \"$rel\"}" | fedmsg-logger \ + --cert-prefix ftpsync \ + --modname bodhi \ + --topic updates.fedora.sync \ + --json-input &> /dev/null + fi +done + diff --git a/roles/bodhi/backend/tasks/main.yml b/roles/bodhi/backend/tasks/main.yml index b67479c1f5..1ed066b3dc 100644 --- a/roles/bodhi/backend/tasks/main.yml +++ b/roles/bodhi/backend/tasks/main.yml @@ -134,3 +134,81 @@ - restart httpd tags: - config + +# +# cron job that syncs packages to koji +# +- name: put owner-sync-pkgdb in place + template: src=owner-sync-pkgdb.j2 dest=/usr/local/bin/owner-sync-pkgdb mode=0755 + tags: + - config + +- name: sync packages from pkgdb2 to koji (el5) + cron: name="owner-sync-el5" minute="7,17,27,37,47,57" user="root" + job="/usr/local/bin/owner-sync-pkgdb dist-5E-epel" + cron_file=update-koji-owner-EL-5 + when: inventory_hostname.startswith('bodhi-backend01') and env == "production" + +- name: sync packages from pkgdb2 to koji (el6) + cron: name="owner-sync-el5" minute="7,17,27,37,47,57" user="root" + job="/usr/local/bin/owner-sync-pkgdb dist-6E-epel" + cron_file=update-koji-owner-EL-6 + when: inventory_hostname.startswith('bodhi-backend01') and env == "production" + +- name: sync packages from pkgdb2 to koji (epel7) + cron: name="owner-sync-el5" minute="7,17,27,37,47,57" user="root" + job="/usr/local/bin/owner-sync-pkgdb epel7" + cron_file=update-koji-owner-epel7 + when: inventory_hostname.startswith('bodhi-backend01') and env == "production" + +- name: sync packages from pkgdb2 to koji (f20) + cron: name="owner-sync-el5" minute="7,17,27,37,47,57" user="root" + job="/usr/local/bin/owner-sync-pkgdb f20" + cron_file=update-koji-owner-f20 + when: inventory_hostname.startswith('bodhi-backend01') + +# +# cron job that syncs updates to master mirror +# + +- name: put fedora-updates-push in place + copy: src=fedora-updates-push dest=/usr/local/bin/fedora-updates-push mode=0755 + tags: + - config + when: inventory_hostname.startswith('bodhi-backend01') and env == "production" + +- name: put fedora-epel-push in place + copy: src=fedora-epel-push dest=/usr/local/bin/fedora-epel-push mode=0755 + tags: + - config + when: inventory_hostname.startswith('bodhi-backend02') and env == "production" + +- name: put update-fullfilelist in place + copy: src=update-fullfilelist dest=/usr/local/bin/update-fullfilelist mode=0755 + tags: + - config + when: inventory_hostname.startswith('bodhi-backend01') and env == "production" + +- name: Updates sync cron job. + cron: name="updates-sync" minute="15,45" user="ftpsync" + job="/usr/local/bin/lock-wrapper fedora-updates-push '/usr/local/bin/fedora-updates-push && /usr/local/bin/update-fullfilelist fedora" + cron_file=updates-sync + when: inventory_hostname.startswith('bodhi-backend01') and env == "production" + tags: + - config + +- name: epel Updates sync cron job. + cron: name="epel-updates-sync" minute="15,45" user="ftpsync" + job="/usr/local/bin/lock-wrapper fedora-epel-push '/usr/local/bin/fedora-epel-push && /usr/local/bin/update-fullfilelist epel" + cron_file=updates-sync + when: inventory_hostname.startswith('bodhi-backend02') and env == "production" + tags: + - config + +- name: directory sizes update cron job. + cron: name="directory-sizes-update" minute="30" hour="19" user="ftpsync" + job="/usr/bin/find /srv/pub/alt/ /srv/pub/archive/ /srv/pub/fedora-secondary/ /srv/pub/fedora/ /srv/pub/epel/ -type d ! -path '/srv/pub/fedora/.snapshot*' ! -path '/srv/pub/epel/.snapshot*' ! -path '/srv/pub/alt/.snapshot*' ! -path '/srv/pub/archive/.snapshot*' ! -path '/srv/pub/fedora-secondary/.snapshot*' ! -path '/srv/pub/alt/stage*' ! -path '/srv/pub/alt/tmp' ! -path '/srv/pub/alt/screenshots/f21/source' | grep -v snapshot | /usr/bin/xargs -n 1 /usr/bin/du --exclude=.snapshot --exclude=stage -sh > /tmp/DIRECTORY_SIZES.txt 2> /dev/null; cp /tmp/DIRECTORY_SIZES.txt /srv/pub/" + cron_file=directory-sizes-update + when: inventory_hostname.startswith('bodhi-backend02') and env == "production" + tags: + - config diff --git a/roles/bodhi/backend/templates/owner-sync-pkgdb.j2 b/roles/bodhi/backend/templates/owner-sync-pkgdb.j2 new file mode 100755 index 0000000000..5d27c55db8 --- /dev/null +++ b/roles/bodhi/backend/templates/owner-sync-pkgdb.j2 @@ -0,0 +1,197 @@ +#!/usr/bin/python2 + +# cronjobs are run on releng01.stg +# Looks like: +# /usr/local/bin/owner-sync-pkgdb f19 +# /usr/local/bin/owner-sync-pkgdb dist-5E-epel +# /usr/local/bin/owner-sync-pkgdb dist-6E-epel +# /usr/local/bin/owner-sync-pkgdb epel7 + +import sys +import os +import ConfigParser +from urlparse import urljoin + +import requests + +DEBUG=False +VERIFY=True +{% if env == 'staging' %} +BASEURL = os.environ.get('PACKAGEDBURL') or 'https://admin.stg.fedoraproject.org/pkgdb/' +{% else %} +BASEURL = os.environ.get('PACKAGEDBURL') or 'https://admin.fedoraproject.org/pkgdb/' +{% endif %} +if not BASEURL.endswith('/'): + BASEURL = BASEURL + '/' + +# Why do we have this? Seems insecure.... +sys.path.append('.') + +try: + import koji +except: + import brew as koji + +extraArchList = {'kernel': ('i586', 'i686', 'noarch'), + 'kernel-xen-2.6': ('i586', 'i686', 'noarch'), + 'glibc': ('i686',), + 'openssl': ('i686',), + 'em8300-kmod': ('i586', 'i686'), + 'sysprof-kmod': ('i586', 'i686'), + } + +def usage(): + print "Usage: owner-sync " + print " : tag to synchronize owners for" + sys.exit(1) + +def get_options(): + # shamelessly stolen from koji CLI + opts = { + 'server': 'http://koji.fedoraproject.org/kojihub', + 'weburl': 'http://koji.fedoraproject.org/koji', + 'cert': '/etc/pki/pkgdb/pkgdb.pem', + 'ca': '/etc/pki/pkgdb/fedora-server-ca.cert', + 'serverca': '/etc/pki/pkgdb/fedora-server-ca.cert' + } + for configFile in ('/etc/koji.conf', os.path.expanduser('~/.koji/config')): + if os.access(configFile, os.F_OK): + f = open(configFile) + config = ConfigParser.ConfigParser() + config.readfp(f) + f.close() + if config.has_section('koji'): + for name, value in config.items('koji'): + if opts.has_key(name): + opts[name] = value + for entry in opts.keys(): + if entry == 'server' or entry == 'weburl': + pass + opts[entry] = os.path.expanduser(opts[entry]) + return opts + +if __name__ == '__main__': + try: + tag=sys.argv[1] + except: + print "ERROR: no tag specified!\n" + usage() + + if tag.endswith('epel') or tag.startswith('epel'): + if tag.startswith('epel'): + version = tag.split('epel')[1] + else: + version = tag.split('-')[1][:-1] + + data = requests.get(urljoin(BASEURL, 'api/collections'), verify=VERIFY).json() + branch_names = set() + for collection in (c for c in data['collections'] if c['status'] != 'EOL'): + ### TODO: check with pingou that this is now returning the same + # format as the collection names in api/vcs + # By moving the data from gitbranchname into branchname, I think + # that the data will now match + branch_names.add(collection['branchname']) + + if tag.startswith('epel'): + # Ex: epel7 => epel7 + reponame = tag + else: + # Ex: dist-6E-epel => el6 + reponame = 'el%s' % version + if reponame not in branch_names: + print 'tag %s => repo %s: does not seem to be a non-EOL branch' % (tag, reponame) + sys.exit(1) + + # EPEL needs a separate entry in koji for each epel version + + data = requests.get(urljoin(BASEURL, 'api/vcs?format=json'), verify=VERIFY).json() + acls = data['packageAcls'] + pkgs = {} + for pkg_name in acls: + try: + owners = acls[pkg_name][reponame] + except KeyError: + # Package is not branched for this release + continue + if len(owners['commit']['people']): + # Arbitrarily take the first committer listed as the owner in + # koji + pkgs[pkg_name] = owners['commit']['people'][0] + else: + pkgs[pkg_name] = 'orphan' + pkgList = pkgs.keys() + BuildEPEL = True + arches = ["primary"] + else: + # Fedora only needs one entry per package for all Fedora releases + # Use the owner from bugzilla for simplicity + data = requests.get(urljoin(BASEURL, 'api/bugzilla?format=json'), verify=VERIFY).json() + acls = data['bugzillaAcls'] + pkgList = acls['Fedora'].keys() + pkgs = {} + for pkg in acls['Fedora']: + owner = acls['Fedora'][pkg]['owner'] + owner = owner.replace('group::', '').replace('@', '') + pkgs[pkg] = owner + + #pkgs = dict(((p, acls['Fedora'][p]['owner']) for p in acls['Fedora'])) + BuildEPEL = False +{% if env == 'staging' %} + arches = ["primary"] +{% else %} + arches = ["primary", "arm", "ppc", "s390"] +{% endif %} + pkgList.sort() + + options = get_options() + + for arch in arches: + if arch == "primary": +{% if env == 'staging' %} + session = koji.ClientSession("http://koji.stg.fedoraproject.org/kojihub") +{% else %} + session = koji.ClientSession("http://koji.fedoraproject.org/kojihub") +{% endif %} + else: + session = koji.ClientSession("http://%s.koji.fedoraproject.org/kojihub" % arch) + try: + session.ssl_login(options['cert'], options['ca'], options['serverca']) + except: + print "Unable to sync to %s hub" % arch + continue + kojitag = session.getTag(tag) + if kojitag is None: + print "ERROR: tag %s does not exist!\n" % (tag) + usage() + + kojipkgs = {} + kojiusers = [user['name'] for user in session.listUsers()] + + for p in session.listPackages(tagID=tag, inherited = True): + kojipkgs[p['package_name']] = p + + for pkg in pkgList: + owner = pkgs[pkg] + if DEBUG: + print '[DEBUG] Package: %s, Owner: %s' % (pkg, owner) + + if not owner in kojiusers: + # add the user first + if DEBUG: + print "Adding user %s" % owner + else: + session.createUser(owner) + kojiusers.append(owner) + if not kojipkgs.has_key(pkg): + if DEBUG: + print "Adding package %s for %s with owner %s" % (pkg, tag, owner) + else: + extraArches = None + if pkg in extraArchList: + extraArches = extraArchList[pkg] + session.packageListAdd(tag, pkg, owner = owner, extra_arches=extraArches) + elif kojipkgs[pkg]['owner_name'] != owner: + if DEBUG: + print "Setting owner for %s in %s to %s" % (pkg, tag, owner) + else: + session.packageListSetOwner(tag, pkg, owner, force = True)