Add cron jobs for bodhi-backend

This commit is contained in:
Kevin Fenzi 2015-06-01 20:32:06 +00:00
parent 1123b0dc80
commit a1c718de54
4 changed files with 458 additions and 0 deletions

View file

@ -0,0 +1,111 @@
#!/bin/sh
SOURCE=/mnt/koji/mash/updates
DEST=/pub/epel/
OPTIONS="-rlptDvHh --stats --delay-updates $RSYNC_OPTS"
for rel in 5 6; do
OUTPUT1=$(rsync $OPTIONS --exclude "repodata/*" --exclude "headers/*" \
$SOURCE/el$rel-epel/ $DEST/$rel/)
OUTPUT2=$(rsync $OPTIONS --delete --delete-delay \
$SOURCE/el$rel-epel/ $DEST/$rel/)
# Grep out some signals from the stats
bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ')
deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l)
# If anything changed, then publish a fedmsg message as bodhi.updates.sync
if [ "$bytes" != "0" -o "$deleted" != "0" ]; then
echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"epel\", \"release\": \"$rel\"}" | fedmsg-logger \
--cert-prefix ftpsync \
--modname bodhi \
--topic updates.epel.sync \
--json-input &> /dev/null
fi
OUTPUT1=$(rsync $OPTIONS --exclude "repodata/*" --exclude "headers/*" \
$SOURCE/el$rel-epel-testing/ $DEST/testing/$rel/)
OUTPUT2=$(rsync $OPTIONS --delete --delete-delay \
$SOURCE/el$rel-epel-testing/ $DEST/testing/$rel/)
# Grep out some signals from the stats
bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ')
deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l)
# If anything changed, then publish a fedmsg message as bodhi.updates.sync
if [ "$bytes" != "0" -o "$deleted" != "0" ]; then
echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"epel-testing\", \"release\": \"$rel\"}" | fedmsg-logger \
--cert-prefix ftpsync \
--modname bodhi \
--topic updates.epel.sync \
--json-input &> /dev/null
fi
done
for rel in 7; do
OUTPUT1=$(rsync $OPTIONS --exclude "repodata/*" \
$SOURCE/epel$rel/ $DEST/$rel/)
OUTPUT2=$(rsync $OPTIONS --delete --delete-delay \
$SOURCE/epel$rel/ $DEST/$rel/)
# Grep out some signals from the stats
bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ')
deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l)
# If anything changed, then publish a fedmsg message as bodhi.updates.sync
if [ "$bytes" != "0" -o "$deleted" != "0" ]; then
echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"epel\", \"release\": \"$rel\"}" | fedmsg-logger \
--cert-prefix ftpsync \
--modname bodhi \
--topic updates.epel.sync \
--json-input &> /dev/null
fi
OUTPUT1=$(rsync $OPTIONS --exclude "repodata/*" \
$SOURCE/epel$rel-testing/ $DEST/testing/$rel/)
OUTPUT2=$(rsync $OPTIONS --delete --delete-delay \
$SOURCE/epel$rel-testing/ $DEST/testing/$rel/)
# Grep out some signals from the stats
bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ')
deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l)
# If anything changed, then publish a fedmsg message as bodhi.updates.sync
if [ "$bytes" != "0" -o "$deleted" != "0" ]; then
echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"epel-testing\", \"release\": \"$rel\"}" | fedmsg-logger \
--cert-prefix ftpsync \
--modname bodhi \
--topic updates.epel.sync \
--json-input &> /dev/null
fi
done
for rel in 5 6 7; do
if [ ${rel} -eq 7 ]; then
TARGET_DIR=${DEST}/${rel}/x86_64/e
else
TARGET_DIR=${DEST}/${rel}/x86_64
fi
if [ -f ${TARGET_DIR}/epel-release*rpm ]; then
# We have a file to match. [This may sort wrong at -9 -> -10]
CANDIDATE=$( ls ${TARGET_DIR}/epel-release-*rpm | sort | tail -n 1)
TARGET=${DEST}/epel-release-latest-${rel}.noarch.rpm
# Does our symbolic link exist?
if [ -L ${TARGET} ]; then
# check to see if the link matches the candidate
TEST=$( readlink ${TARGET} )
if [ ${TEST} != ${CANDIDATE} ]; then
ln -sf $(echo ${CANDIDATE}|sed -e "s|$DEST|./|g" -e 's|//|/|g') ${TARGET}
fi
else
# first time for everything.
ln -sf $(echo ${CANDIDATE}|sed -e "s|$DEST|./|g" -e 's|//|/|g') ${TARGET}
fi
else
echo "No target file for epel-release ${rel} to link against."
fi
done

View file

@ -0,0 +1,72 @@
#!/bin/sh
SOURCE=/mnt/koji/mash/updates
DEST=/pub/fedora/linux/updates/
ATOMICSOURCE=/mnt/koji/mash/atomic/
ATOMICDEST=/pub/fedora/linux/atomic/
OPTIONS="-rlptDvHh --stats --delay-updates $RSYNC_OPTS"
for rel in 20 21 22; do
OUTPUT1=$(rsync $OPTIONS --exclude "repodata/*" \
$SOURCE/f$rel-updates/ $DEST/$rel/ --link-dest $DEST/testing/$rel/)
OUTPUT2=$(rsync $OPTIONS --delete --delete-delay --exclude=Live --exclude=Images \
$SOURCE/f$rel-updates/ $DEST/$rel/)
# Grep out some signals from the stats
bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ')
deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l)
# If anything changed, then publish a fedmsg message as bodhi.updates.sync
if [ "$bytes" != "0" -o "$deleted" != "0" ]; then
echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"updates\", \"release\": \"$rel\"}" | fedmsg-logger \
--cert-prefix ftpsync \
--modname bodhi \
--topic updates.fedora.sync \
--json-input &> /dev/null
fi
done
for rel in 20 21 22; do
OUTPUT1=$(rsync $OPTIONS --exclude "repodata/*" \
$SOURCE/f$rel-updates-testing/ $DEST/testing/$rel/)
OUTPUT2=$(rsync $OPTIONS --delete --delete-delay --exclude=Live --exclude=Images \
$SOURCE/f$rel-updates-testing/ $DEST/testing/$rel/)
# Grep out some signals from the stats
bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ')
deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l)
# If anything changed, then publish a fedmsg message as bodhi.updates.sync
if [ "$bytes" != "0" -o "$deleted" != "0" ]; then
echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"updates-testing\", \"release\": \"$rel\"}" | fedmsg-logger \
--cert-prefix ftpsync \
--modname bodhi \
--topic updates.fedora.sync \
--json-input &> /dev/null
fi
done
for rel in 21 22; do
OUTPUT1=$(rsync $OPTIONS --ignore-existing \
$ATOMICSOURCE/$rel/objects/ $ATOMICDEST/$rel/objects/)
OUTPUT2=$(rsync $OPTIONS --delete --delete-delay --exclude=objects/ \
$ATOMICSOURCE/$rel/ $ATOMICDEST/$rel/)
# Grep out some signals from the stats
bytes=$(echo "$OUTPUT1" | grep "Literal data" | awk ' { print $3 } ')
deleted=$(echo "$OUTPUT2" | grep "deleting " | wc -l)
# If anything changed, then publish a fedmsg message as bodhi.updates.sync
if [ "$bytes" != "0" -o "$deleted" != "0" ]; then
echo "{\"bytes\": \"$bytes\", \"deleted\": \"$deleted\", \"repo\": \"atomic\", \"release\": \"$rel\"}" | fedmsg-logger \
--cert-prefix ftpsync \
--modname bodhi \
--topic updates.fedora.sync \
--json-input &> /dev/null
fi
done

View file

@ -134,3 +134,81 @@
- restart httpd
tags:
- config
#
# cron job that syncs packages to koji
#
- name: put owner-sync-pkgdb in place
template: src=owner-sync-pkgdb.j2 dest=/usr/local/bin/owner-sync-pkgdb mode=0755
tags:
- config
- name: sync packages from pkgdb2 to koji (el5)
cron: name="owner-sync-el5" minute="7,17,27,37,47,57" user="root"
job="/usr/local/bin/owner-sync-pkgdb dist-5E-epel"
cron_file=update-koji-owner-EL-5
when: inventory_hostname.startswith('bodhi-backend01') and env == "production"
- name: sync packages from pkgdb2 to koji (el6)
cron: name="owner-sync-el5" minute="7,17,27,37,47,57" user="root"
job="/usr/local/bin/owner-sync-pkgdb dist-6E-epel"
cron_file=update-koji-owner-EL-6
when: inventory_hostname.startswith('bodhi-backend01') and env == "production"
- name: sync packages from pkgdb2 to koji (epel7)
cron: name="owner-sync-el5" minute="7,17,27,37,47,57" user="root"
job="/usr/local/bin/owner-sync-pkgdb epel7"
cron_file=update-koji-owner-epel7
when: inventory_hostname.startswith('bodhi-backend01') and env == "production"
- name: sync packages from pkgdb2 to koji (f20)
cron: name="owner-sync-el5" minute="7,17,27,37,47,57" user="root"
job="/usr/local/bin/owner-sync-pkgdb f20"
cron_file=update-koji-owner-f20
when: inventory_hostname.startswith('bodhi-backend01')
#
# cron job that syncs updates to master mirror
#
- name: put fedora-updates-push in place
copy: src=fedora-updates-push dest=/usr/local/bin/fedora-updates-push mode=0755
tags:
- config
when: inventory_hostname.startswith('bodhi-backend01') and env == "production"
- name: put fedora-epel-push in place
copy: src=fedora-epel-push dest=/usr/local/bin/fedora-epel-push mode=0755
tags:
- config
when: inventory_hostname.startswith('bodhi-backend02') and env == "production"
- name: put update-fullfilelist in place
copy: src=update-fullfilelist dest=/usr/local/bin/update-fullfilelist mode=0755
tags:
- config
when: inventory_hostname.startswith('bodhi-backend01') and env == "production"
- name: Updates sync cron job.
cron: name="updates-sync" minute="15,45" user="ftpsync"
job="/usr/local/bin/lock-wrapper fedora-updates-push '/usr/local/bin/fedora-updates-push && /usr/local/bin/update-fullfilelist fedora"
cron_file=updates-sync
when: inventory_hostname.startswith('bodhi-backend01') and env == "production"
tags:
- config
- name: epel Updates sync cron job.
cron: name="epel-updates-sync" minute="15,45" user="ftpsync"
job="/usr/local/bin/lock-wrapper fedora-epel-push '/usr/local/bin/fedora-epel-push && /usr/local/bin/update-fullfilelist epel"
cron_file=updates-sync
when: inventory_hostname.startswith('bodhi-backend02') and env == "production"
tags:
- config
- name: directory sizes update cron job.
cron: name="directory-sizes-update" minute="30" hour="19" user="ftpsync"
job="/usr/bin/find /srv/pub/alt/ /srv/pub/archive/ /srv/pub/fedora-secondary/ /srv/pub/fedora/ /srv/pub/epel/ -type d ! -path '/srv/pub/fedora/.snapshot*' ! -path '/srv/pub/epel/.snapshot*' ! -path '/srv/pub/alt/.snapshot*' ! -path '/srv/pub/archive/.snapshot*' ! -path '/srv/pub/fedora-secondary/.snapshot*' ! -path '/srv/pub/alt/stage*' ! -path '/srv/pub/alt/tmp' ! -path '/srv/pub/alt/screenshots/f21/source' | grep -v snapshot | /usr/bin/xargs -n 1 /usr/bin/du --exclude=.snapshot --exclude=stage -sh > /tmp/DIRECTORY_SIZES.txt 2> /dev/null; cp /tmp/DIRECTORY_SIZES.txt /srv/pub/"
cron_file=directory-sizes-update
when: inventory_hostname.startswith('bodhi-backend02') and env == "production"
tags:
- config

View file

@ -0,0 +1,197 @@
#!/usr/bin/python2
# cronjobs are run on releng01.stg
# Looks like:
# /usr/local/bin/owner-sync-pkgdb f19
# /usr/local/bin/owner-sync-pkgdb dist-5E-epel
# /usr/local/bin/owner-sync-pkgdb dist-6E-epel
# /usr/local/bin/owner-sync-pkgdb epel7
import sys
import os
import ConfigParser
from urlparse import urljoin
import requests
DEBUG=False
VERIFY=True
{% if env == 'staging' %}
BASEURL = os.environ.get('PACKAGEDBURL') or 'https://admin.stg.fedoraproject.org/pkgdb/'
{% else %}
BASEURL = os.environ.get('PACKAGEDBURL') or 'https://admin.fedoraproject.org/pkgdb/'
{% endif %}
if not BASEURL.endswith('/'):
BASEURL = BASEURL + '/'
# Why do we have this? Seems insecure....
sys.path.append('.')
try:
import koji
except:
import brew as koji
extraArchList = {'kernel': ('i586', 'i686', 'noarch'),
'kernel-xen-2.6': ('i586', 'i686', 'noarch'),
'glibc': ('i686',),
'openssl': ('i686',),
'em8300-kmod': ('i586', 'i686'),
'sysprof-kmod': ('i586', 'i686'),
}
def usage():
print "Usage: owner-sync <tag>"
print " <tag>: tag to synchronize owners for"
sys.exit(1)
def get_options():
# shamelessly stolen from koji CLI
opts = {
'server': 'http://koji.fedoraproject.org/kojihub',
'weburl': 'http://koji.fedoraproject.org/koji',
'cert': '/etc/pki/pkgdb/pkgdb.pem',
'ca': '/etc/pki/pkgdb/fedora-server-ca.cert',
'serverca': '/etc/pki/pkgdb/fedora-server-ca.cert'
}
for configFile in ('/etc/koji.conf', os.path.expanduser('~/.koji/config')):
if os.access(configFile, os.F_OK):
f = open(configFile)
config = ConfigParser.ConfigParser()
config.readfp(f)
f.close()
if config.has_section('koji'):
for name, value in config.items('koji'):
if opts.has_key(name):
opts[name] = value
for entry in opts.keys():
if entry == 'server' or entry == 'weburl':
pass
opts[entry] = os.path.expanduser(opts[entry])
return opts
if __name__ == '__main__':
try:
tag=sys.argv[1]
except:
print "ERROR: no tag specified!\n"
usage()
if tag.endswith('epel') or tag.startswith('epel'):
if tag.startswith('epel'):
version = tag.split('epel')[1]
else:
version = tag.split('-')[1][:-1]
data = requests.get(urljoin(BASEURL, 'api/collections'), verify=VERIFY).json()
branch_names = set()
for collection in (c for c in data['collections'] if c['status'] != 'EOL'):
### TODO: check with pingou that this is now returning the same
# format as the collection names in api/vcs
# By moving the data from gitbranchname into branchname, I think
# that the data will now match
branch_names.add(collection['branchname'])
if tag.startswith('epel'):
# Ex: epel7 => epel7
reponame = tag
else:
# Ex: dist-6E-epel => el6
reponame = 'el%s' % version
if reponame not in branch_names:
print 'tag %s => repo %s: does not seem to be a non-EOL branch' % (tag, reponame)
sys.exit(1)
# EPEL needs a separate entry in koji for each epel version
data = requests.get(urljoin(BASEURL, 'api/vcs?format=json'), verify=VERIFY).json()
acls = data['packageAcls']
pkgs = {}
for pkg_name in acls:
try:
owners = acls[pkg_name][reponame]
except KeyError:
# Package is not branched for this release
continue
if len(owners['commit']['people']):
# Arbitrarily take the first committer listed as the owner in
# koji
pkgs[pkg_name] = owners['commit']['people'][0]
else:
pkgs[pkg_name] = 'orphan'
pkgList = pkgs.keys()
BuildEPEL = True
arches = ["primary"]
else:
# Fedora only needs one entry per package for all Fedora releases
# Use the owner from bugzilla for simplicity
data = requests.get(urljoin(BASEURL, 'api/bugzilla?format=json'), verify=VERIFY).json()
acls = data['bugzillaAcls']
pkgList = acls['Fedora'].keys()
pkgs = {}
for pkg in acls['Fedora']:
owner = acls['Fedora'][pkg]['owner']
owner = owner.replace('group::', '').replace('@', '')
pkgs[pkg] = owner
#pkgs = dict(((p, acls['Fedora'][p]['owner']) for p in acls['Fedora']))
BuildEPEL = False
{% if env == 'staging' %}
arches = ["primary"]
{% else %}
arches = ["primary", "arm", "ppc", "s390"]
{% endif %}
pkgList.sort()
options = get_options()
for arch in arches:
if arch == "primary":
{% if env == 'staging' %}
session = koji.ClientSession("http://koji.stg.fedoraproject.org/kojihub")
{% else %}
session = koji.ClientSession("http://koji.fedoraproject.org/kojihub")
{% endif %}
else:
session = koji.ClientSession("http://%s.koji.fedoraproject.org/kojihub" % arch)
try:
session.ssl_login(options['cert'], options['ca'], options['serverca'])
except:
print "Unable to sync to %s hub" % arch
continue
kojitag = session.getTag(tag)
if kojitag is None:
print "ERROR: tag %s does not exist!\n" % (tag)
usage()
kojipkgs = {}
kojiusers = [user['name'] for user in session.listUsers()]
for p in session.listPackages(tagID=tag, inherited = True):
kojipkgs[p['package_name']] = p
for pkg in pkgList:
owner = pkgs[pkg]
if DEBUG:
print '[DEBUG] Package: %s, Owner: %s' % (pkg, owner)
if not owner in kojiusers:
# add the user first
if DEBUG:
print "Adding user %s" % owner
else:
session.createUser(owner)
kojiusers.append(owner)
if not kojipkgs.has_key(pkg):
if DEBUG:
print "Adding package %s for %s with owner %s" % (pkg, tag, owner)
else:
extraArches = None
if pkg in extraArchList:
extraArches = extraArchList[pkg]
session.packageListAdd(tag, pkg, owner = owner, extra_arches=extraArches)
elif kojipkgs[pkg]['owner_name'] != owner:
if DEBUG:
print "Setting owner for %s in %s to %s" % (pkg, tag, owner)
else:
session.packageListSetOwner(tag, pkg, owner, force = True)