From 8ca8220f83aa2f0feec04c977af70fca2f7831e0 Mon Sep 17 00:00:00 2001 From: Till Maas Date: Sun, 8 Jan 2017 16:20:16 +0100 Subject: [PATCH] Revert "Remove koji-gc hotfix" This reverts commit 3c24ea35b323f810abf7d4c16fb9936063de2629. --- roles/koji_hub/files/koji-gc.py-hotfix | 927 +++++++++++++++++++++++++ roles/koji_hub/tasks/main.yml | 9 + 2 files changed, 936 insertions(+) create mode 100755 roles/koji_hub/files/koji-gc.py-hotfix diff --git a/roles/koji_hub/files/koji-gc.py-hotfix b/roles/koji_hub/files/koji-gc.py-hotfix new file mode 100755 index 0000000000..4116f3b184 --- /dev/null +++ b/roles/koji_hub/files/koji-gc.py-hotfix @@ -0,0 +1,927 @@ +#!/usr/bin/python + +# koji-gc: a garbage collection tool for Koji +# Copyright (c) 2007-2014 Red Hat, Inc. +# +# Authors: +# Mike McLean + +try: + import krbV +except ImportError: # pragma: no cover + pass +import koji +from koji.util import LazyDict, LazyValue +import koji.policy +import ConfigParser +from email.MIMEText import MIMEText +import fnmatch +import optparse +import os +import pprint +import smtplib +import socket # for socket.error +import sys +import time +import xmlrpclib # for ProtocolError and Fault + + +OptionParser = optparse.OptionParser +if optparse.__version__ == "1.4.1+": + def _op_error(self, msg): + self.print_usage(sys.stderr) + msg = "%s: error: %s\n" % (self._get_prog_name(), msg) + if msg: + sys.stderr.write(msg) + sys.exit(2) + OptionParser.error = _op_error + + +def _(args): + """Stub function for translation""" + return args + +def get_options(): + """process options from command line and config file""" + + usage = _("%prog [options]") + parser = OptionParser(usage=usage) + parser.add_option("-c", "--config-file", metavar="FILE", + help=_("use alternate configuration file")) + parser.add_option("--keytab", help=_("specify a Kerberos keytab to use")) + parser.add_option("--principal", help=_("specify a Kerberos principal to use")) + parser.add_option("--krbservice", default="host", + help=_("the service name of the principal being used by the hub")) + parser.add_option("--runas", metavar="USER", + help=_("run as the specified user (requires special privileges)")) + parser.add_option("--user", help=_("specify user")) + parser.add_option("--password", help=_("specify password")) + parser.add_option("--noauth", action="store_true", default=False, + help=_("do not authenticate")) + parser.add_option("--network-hack", action="store_true", default=False, + help=optparse.SUPPRESS_HELP) # no longer used + parser.add_option("--cert", default='/etc/koji-gc/client.crt', + help=_("Client SSL certificate file for authentication")) + parser.add_option("--ca", default='', + help=_("ignored")) # FIXME: remove in next major release + parser.add_option("--serverca", default='/etc/koji-gc/serverca.crt', + help=_("CA cert file that issued the hub certificate")) + parser.add_option("-n", "--test", action="store_true", default=False, + help=_("test mode")) + parser.add_option("-d", "--debug", action="store_true", default=False, + help=_("show debug output")) + parser.add_option("--debug-xmlrpc", action="store_true", default=False, + help=_("show xmlrpc debug output")) + parser.add_option("--smtp-host", metavar="HOST", + help=_("specify smtp server for notifications")) + parser.add_option("--no-mail", action='store_false', default=True, dest="mail", + help=_("don't send notifications")) + parser.add_option("--send-mail", action='store_true', dest="mail", + help=_("send notifications")) + parser.add_option("--email-domain", default="fedoraproject.org", + help=_("Email domain appended to Koji username for notifications")) + parser.add_option("--from-addr", default="Koji Build System ", + help=_("From address for notifications")) + parser.add_option("--action", help=_("action(s) to take")) + parser.add_option("--delay", metavar="INTERVAL", default = '5 days', + help="time before eligible builds are placed in trashcan") + parser.add_option("--grace-period", default='4 weeks', metavar="INTERVAL", + help="time that builds are held in trashcan") + parser.add_option("--skip-main", action="store_true", default=False, + help=_("don't actually run main")) + parser.add_option("--unprotected-keys", metavar="KEYS", + help=_("allow builds signed with these keys to be deleted")) + parser.add_option("--tag-filter", "--tag", metavar="PATTERN", action="append", + help=_("Process only tags matching PATTERN when pruning")) + parser.add_option("--ignore-tags", metavar="PATTERN", action="append", + help=_("Ignore tags matching PATTERN when pruning")) + parser.add_option("--pkg-filter", "--pkg", "--package", + metavar="PATTERN", action='append', + help=_("Process only packages matching PATTERN")) + parser.add_option("--bypass-locks", metavar="PATTERN", action="append", + help=_("Bypass locks for tags matching PATTERN")) + parser.add_option("--purge", action="store_true", default=False, + help=_("When pruning, attempt to delete the builds that are untagged")) + parser.add_option("--trashcan-tag", default='trashcan', metavar="TAG", + help=_("specify an alternate trashcan tag")) + parser.add_option("--weburl", default="http://localhost/koji", metavar="URL", + help=_("url of koji web server (for use in notifications)")) + parser.add_option("-s", "--server", help=_("url of koji XMLRPC server")) + #parse once to get the config file + (options, args) = parser.parse_args() + + defaults = parser.get_default_values() + config = ConfigParser.ConfigParser() + cf = getattr(options, 'config_file', None) + if cf: + if not os.access(cf, os.F_OK): + parser.error(_("No such file: %s") % cf) + assert False # pragma: no cover + else: + cf = '/etc/koji-gc/koji-gc.conf' + if not os.access(cf, os.F_OK): + cf = None + if not cf: + print "no config file" + config = None + else: + config.read(cf) + #allow config file to update defaults for certain options + cfgmap = [ + ['keytab', None, 'string'], + ['principal', None, 'string'], + ['krbservice', None, 'string'], + ['krb_rdns', None, 'boolean'], + ['runas', None, 'string'], + ['user', None, 'string'], + ['password', None, 'string'], + ['noauth', None, 'boolean'], + ['cert', None, 'string'], + ['ca', None, 'string'], # FIXME: remove in next major release + ['serverca', None, 'string'], + ['server', None, 'string'], + ['weburl', None, 'string'], + ['smtp_host', None, 'string'], + ['from_addr', None, 'string'], + ['email_domain', None, 'string'], + ['mail', None, 'boolean'], + ['delay', None, 'string'], + ['unprotected_keys', None, 'string'], + ['grace_period', None, 'string'], + ['trashcan_tag', None, 'string'], + ['use_old_ssl', None, 'boolean'], + ['no_ssl_verify', None, 'boolean'], + ['timeout', None, 'integer'], + ] + for name, alias, type in cfgmap: + if alias is None: + alias = ('main', name) + if config.has_option(*alias): + if options.debug: + print "Using option %s from config file" % (alias,) + if type == 'integer': + setattr(defaults, name, config.getint(*alias)) + elif type == 'boolean': + setattr(defaults, name, config.getboolean(*alias)) + else: + setattr(defaults, name, config.get(*alias)) + #parse again with defaults + (options, args) = parser.parse_args(values=defaults) + options.config = config + + #figure out actions + actions = ('prune', 'trash', 'delete', 'salvage') + if options.action: + options.action = options.action.lower().replace(',',' ').split() + for x in options.action: + if x not in actions: + parser.error(_("Invalid action: %s") % x) + else: + options.action = ('delete', 'prune', 'trash') + + #split patterns for unprotected keys + if options.unprotected_keys: + options.unprotected_key_patterns = options.unprotected_keys.replace(',',' ').split() + else: + options.unprotected_key_patterns = [] + + #parse key aliases + options.key_aliases = {} + try: + if config and config.has_option('main', 'key_aliases'): + for line in config.get('main','key_aliases').splitlines(): + parts = line.split() + if len(parts) < 2: + continue + options.key_aliases[parts[0].upper()] = parts[1] + except ValueError, e: + print e + parser.error(_("Invalid key alias data in config: %s") % config.get('main','key_aliases')) + + #parse time intervals + for key in ('delay', 'grace_period'): + try: + value = getattr(options, key) + value = parse_duration(value) + setattr(options, key, value) + if options.debug: + print "%s: %s seconds" % (key, value) + except ValueError: + parser.error(_("Invalid time interval: %s") % value) + + return options, args + +def check_tag(name): + """Check tag name against options and determine if we should process it + + The ignore option takes priority here. + Returns True if we should process the tag, False otherwise + """ + if options.ignore_tags: + for pattern in options.ignore_tags: + if fnmatch.fnmatch(name, options.tag_filter): + return False + if options.tag_filter: + for pattern in options.tag_filter: + if fnmatch.fnmatch(name, pattern): + return True + #doesn't match any pattern in filter + return False + else: + #not ignored and no filter specified + return True + +def check_package(name): + """Check package name against options and determine if we should process it + + Returns True if we should process the package, False otherwise + """ + if options.pkg_filter: + for pattern in options.pkg_filter: + if fnmatch.fnmatch(name, pattern): + return True + #doesn't match any pattern in filter + return False + else: + #no filter specified + return True + +time_units = { + 'second' : 1, + 'minute' : 60, + 'hour' : 3600, + 'day' : 86400, + 'week' : 604800, +} +time_unit_aliases = [ + #[unit, alias, alias, ...] + ['week', 'weeks', 'wk', 'wks'], + ['hour', 'hours', 'hr', 'hrs'], + ['day', 'days'], + ['minute', 'minutes', 'min', 'mins'], + ['second', 'seconds', 'sec', 'secs', 's'], +] +def parse_duration(str): + """Parse time duration from string, returns duration in seconds""" + ret = 0 + n = None + unit = None + def parse_num(s): + try: + return int(s) + except ValueError: + pass + try: + return float(s) + except ValueError: + pass + return None + for x in str.split(): + if n is None: + n = parse_num(x) + if n is not None: + continue + #perhaps the unit is appended w/o a space + for names in time_unit_aliases: + for name in names: + if x.endswith(name): + n = parse_num(x[:-len(name)]) + if n is None: + continue + unit = names[0] + # combined at end + break + if unit: + break + else: + raise ValueError, "Invalid time interval: %s" % str + if unit is None: + x = x.lower() + for names in time_unit_aliases: + for name in names: + if x == name: + unit = names[0] + break + if unit: + break + else: + raise ValueError, "Invalid time interval: %s" % str + ret += n * time_units[unit] + n = None + unit = None + return ret + +def error(msg=None, code=1): + if msg: + sys.stderr.write(msg + "\n") + sys.stderr.flush() + sys.exit(code) + +def warn(msg): + sys.stderr.write(msg + "\n") + sys.stderr.flush() + +def ensure_connection(session): + try: + ret = session.getAPIVersion() + except xmlrpclib.ProtocolError: + error(_("Error: Unable to connect to server")) + if ret != koji.API_VERSION: + warn(_("WARNING: The server is at API version %d and the client is at %d" % (ret, koji.API_VERSION))) + +def has_krb_creds(): + if not sys.modules.has_key('krbV'): + return False + try: + ctx = krbV.default_context() + ccache = ctx.default_ccache() + princ = ccache.principal() + return True + except krbV.Krb5Error: + return False + +def activate_session(session): + """Test and login the session is applicable""" + global options + if options.noauth: + #skip authentication + pass + elif os.path.isfile(options.cert): + # authenticate using SSL client cert + session.ssl_login(options.cert, None, options.serverca, proxyuser=options.runas) + elif options.user: + #authenticate using user/password + session.login() + elif has_krb_creds() or (options.keytab and options.principal): + try: + if options.keytab and options.principal: + session.krb_login(principal=options.principal, keytab=options.keytab, proxyuser=options.runas) + else: + session.krb_login(proxyuser=options.runas) + except krbV.Krb5Error, e: + error(_("Kerberos authentication failed: %s (%s)") % (e.args[1], e.args[0])) + except socket.error, e: + warn(_("Could not connect to Kerberos authentication service: '%s'") % e.args[1]) + if not options.noauth and not session.logged_in: + error(_("Error: unable to log in, no authentication methods available")) + ensure_connection(session) + if options.debug: + print "successfully connected to hub" + +def send_warning_notice(owner_name, builds): + if not options.mail: + return + if not builds: + print "Warning: empty build list. No notice sent" + return + head = """\ +The following build(s) are unreferenced and have been marked for +deletion. They will be held in the trashcan tag for a grace period. +At the end of that period they will be deleted permanently. This +garbage collection is a normal part of build system operation. +Please see the following url for more information: + + http://fedoraproject.org/wiki/Koji/GarbageCollection""" + fmt="""\ +Build: %%(name)s-%%(version)s-%%(release)s +%s/buildinfo?buildID=%%(id)i""" % options.weburl + middle = '\n\n'.join([fmt % b for b in builds]) + tail = """\ +If you would like to protect any of these builds from deletion, please +refer to the document linked above for instructions.""" + + msg = MIMEText('\n\n'.join([head, middle, tail])) + if len(builds) == 1: + msg['Subject'] = "1 build marked for deletion" + else: + msg['Subject'] = "%i builds marked for deletion" % len(builds) + msg['From'] = options.from_addr + msg['To'] = "%s@%s" % (owner_name, options.email_domain) #XXX! + msg['X-Koji-Builder'] = owner_name + if options.test: + if options.debug: + print str(msg) + else: + print "Would have sent warning notice to %s" % msg['To'] + else: + if options.debug: + print "Sending warning notice to %s" % msg['To'] + s = smtplib.SMTP(options.smtp_host) + s.sendmail(msg['From'], msg['To'], msg.as_string()) + s.quit() + + +def main(args): + activate_session(session) + for x in options.action: + globals()['handle_' + x]() + + +def handle_trash(): + print "Getting untagged builds..." + untagged = session.untaggedBuilds() + print "...got %i builds" % len(untagged) + min_age = options.delay + trashcan_tag = options.trashcan_tag + #Step 1: place unreferenced builds into trashcan + i = 0 + N = len(untagged) + to_trash = [] + for binfo in untagged: + i += 1 + nvr = "%(name)s-%(version)s-%(release)s" % binfo + if not check_package(binfo['name']): + if options.debug: + print "[%i/%i] Skipping package: %s" % (i, N, nvr) + continue + try: + refs = session.buildReferences(binfo['id'], limit=10) + except xmlrpclib.Fault: + print "[%i/%i] Error checking references for %s. Skipping" % (i, N, nvr) + continue + #XXX - this is more data than we need + # also, this call takes waaaay longer than it should + if refs['tags']: + # must have been tagged just now + print "[%i/%i] Build is tagged [?]: %s" % (i, N, nvr) + continue + if refs['rpms']: + if options.debug: + print "[%i/%i] Build has %i rpm references: %s" % (i, N, len(refs['rpms']), nvr) + #pprint.pprint(refs['rpms']) + continue + if refs['archives']: + if options.debug: + print "[%i/%i] Build has %i archive references: %s" % (i, N, len(refs['archives']), nvr) + #pprint.pprint(refs['archives']) + continue + ts = refs['last_used'] + if ts: + #work around server bug + if isinstance(ts, list): + ts = ts[0] + #XXX - should really check time server side + if options.debug: + print "[%i/%i] Build has been used in a buildroot: %s" % (i, N, nvr) + print "Last_used: %r" % ts + age = time.time() - ts + if age < min_age: + continue + #see how long build has been untagged + history = session.tagHistory(build=binfo['id']) + age = None + binfo2 = None + if not history: + #never tagged, we'll have to use the build create time + binfo2 = session.getBuild(binfo['id']) + ts = binfo2.get('creation_ts') + if ts is None: + # older api with no good way to get a proper timestamp for + # a build, so we have the following hack + task_id = binfo2.get('task_id') + if task_id: + tinfo = session.getTaskInfo(task_id) + if tinfo['completion_ts']: + age = time.time() - tinfo['completion_ts'] + else: + age = time.time() - ts + else: + history = [(h['revoke_event'],h) for h in history] + last = max(history)[1] + if not last['revoke_event']: + #this might happen if the build was tagged just now + print "[%i/%i] Warning: build not untagged: %s" % (i, N, nvr) + continue + age = time.time() - last['revoke_ts'] + if age is not None and age < min_age: + if options.debug: + print "[%i/%i] Build untagged only recently: %s" % (i, N, nvr) + continue + #check build signatures + keys = get_build_sigs(binfo['id'], cache=True) + if keys and options.debug: + print "Build: %s, Keys: %s" % (nvr, keys) + if protected_sig(keys): + print "Skipping build %s. Keys: %s" % (nvr, keys) + continue + + #ok, go ahead add it to the list + if binfo2 is None: + binfo2 = session.getBuild(binfo['id']) + binfo2['nvr'] = nvr + print "[%i/%i] Adding build to trash list: %s" % (i, N, nvr) + to_trash.append(binfo2) + + #process to_trash + #group by owner so we can reduce the number of notices + by_owner = {} + for binfo in to_trash: + by_owner.setdefault(binfo['owner_name'], []).append(binfo) + owners = by_owner.keys() + owners.sort() + for owner_name in owners: + builds = [(b['nvr'], b) for b in by_owner[owner_name]] + builds.sort() + send_warning_notice(owner_name, [x[1] for x in builds]) + for nvr, binfo in builds: + if options.test: + print "Would have moved to trashcan: %s" % nvr + else: + if options.debug: + print "Moving to trashcan: %s" % nvr + #figure out package owner + count = {} + for pkg in session.listPackages(pkgID=binfo['name']): + count.setdefault(pkg['owner_id'], 0) + count[pkg['owner_id']] += 1 + if not count: + print "Warning: no owner for %s, using build owner" % nvr + #best we can do currently + owner = binfo['owner_id'] + else: + owner = max([(n, k) for k, n in count.iteritems()])[1] + session.packageListAdd(trashcan_tag, binfo['name'], owner) + session.tagBuildBypass(trashcan_tag, binfo['id'], force=True) + +def protected_sig(keys): + """Check list of keys and see if any are protected + + returns True if ANY are protected (not on unprotected list) + returns False if ALL are unprotected + """ + for key in keys: + if not key: + continue + if not sigmatch(key, options.unprotected_key_patterns): + #this key is protected + return True + return False + + +def handle_salvage(): + """Reclaim builds from trashcan + + Check builds in trashcan for new tags or references and salvage them + (remove trashcan tag) if appropriate. + + The delete action also does this, but this is for when you want to + run this action only.""" + return handle_delete(just_salvage=True) + +def salvage_build(binfo): + """Removes trashcan tag from a build and prints a message""" + if options.test: + print "Would have untagged from trashcan: %(nvr)s" % binfo + else: + if options.debug: + print "Untagging from trashcan: %(nvr)s" % binfo + session.untagBuildBypass(options.trashcan_tag, binfo['id'], force=True) + +def handle_delete(just_salvage=False): + """Delete builds that have been in the trashcan for long enough + + If just_salvage is True, goes into salvage mode. In salvage mode it only + reclaims referenced builds from the trashcan, it does not perform any + deletes + """ + print "Getting list of builds in trash..." + trashcan_tag = options.trashcan_tag + trash = [(b['nvr'], b) for b in session.listTagged(trashcan_tag)] + trash.sort() + print "...got %i builds" % len(trash) + #XXX - it would be better if there were more appropriate server calls for this + grace_period = options.grace_period + for nvr, binfo in trash: + # see if build has been tagged elsewhere + if not check_package(binfo['name']): + if options.debug: + print "Skipping package: %s" % nvr + continue + tags = [t['name'] for t in session.listTags(build=binfo['id']) if t['name'] != trashcan_tag] + if tags: + print "Build %s tagged elsewhere: %s" % (nvr, tags) + salvage_build(binfo) + continue + #check build signatures + keys = get_build_sigs(binfo['id'], cache=False) + if keys and options.debug: + print "Build: %s, Keys: %s" % (nvr, keys) + if protected_sig(keys): + print "Salvaging signed build %s. Keys: %s" % (nvr, keys) + salvage_build(binfo) + continue + if just_salvage: + # skip the rest when salvaging + continue + # determine how long this build has been in the trashcan + history = session.tagHistory(build=binfo['id'], tag=trashcan_tag) + current = [x for x in history if x['active']] + if not current: + #untagged just now? + print "Warning: history missing for %s" % nvr + pprint.pprint(binfo) + pprint.pprint(history) + continue + assert len(current) == 1 #see db constraint + current = current[0] + age = time.time() - current['create_ts'] + if age < grace_period: + if options.debug: + print "Skipping build %s, age=%i" % (nvr, age) + continue + + # go ahead and delete + if options.test: + print "Would have deleted build from trashcan: %s" % nvr + else: + print "Deleting build: %s" % nvr + session.untagBuildBypass(trashcan_tag, binfo['id']) + try: + session.deleteBuild(binfo['id']) + except (xmlrpclib.Fault, koji.GenericError), e: + print "Warning: deletion failed: %s" % e + #server issue + pass + #TODO - log details for delete failures + + +class TagPruneTest(koji.policy.MatchTest): + name = 'tag' + field = 'tagname' + + +class PackagePruneTest(koji.policy.MatchTest): + name = 'package' + field = 'pkgname' + + +class VolumePruneTest(koji.policy.MatchTest): + name = 'volume' + field = 'volname' + + +class SigPruneTest(koji.policy.BaseSimpleTest): + name = 'sig' + + def run(self, data): + # true if any of the keys match any of the patterns + patterns = self.str.split()[1:] + for key in data['keys']: + if sigmatch(key, patterns): + return True + return False + + +def sigmatch(key, patterns): + """Test whether a key id matches any of the given patterns + + Supports key aliases + """ + if not isinstance(patterns, (tuple, list)): + patterns = (patterns,) + for pat in patterns: + if fnmatch.fnmatch(key, pat): + return True + alias = options.key_aliases.get(key.upper()) + if alias is not None and fnmatch.fnmatch(alias, pat): + return True + return False + + +class OrderPruneTest(koji.policy.CompareTest): + name = 'order' + field = 'order' + allow_float = False + + +class AgePruneTest(koji.policy.BaseSimpleTest): + name = 'age' + cmp_idx = koji.policy.CompareTest.operators + + def __init__(self, str): + """Read the test parameters from string""" + super(AgePruneTest, self).__init__(str) + self.cmp, value = str.split(None, 2)[1:] + self.func = self.cmp_idx.get(self.cmp, None) + if self.func is None: + raise Exception, "Invalid comparison in test: %s" % str + self.span = parse_duration(value) + + def run(self, data): + return self.func(time.time() - data['ts'], self.span) + + +def read_policies(fn=None): + """Read tag gc policies from file + + The expected format as follows + test [params] [&& test [params] ...] :: (keep|untag|skip) + """ + fo = file(fn, 'r') + tests = koji.policy.findSimpleTests(globals()) + ret = koji.policy.SimpleRuleSet(fo, tests) + fo.close() + return ret + +def scan_policies(str): + """Read tag gc policies from a string + + The expected format as follows + test [params] [&& test [params] ...] :: (keep|untag|skip) + """ + tests = koji.policy.findSimpleTests(globals()) + return koji.policy.SimpleRuleSet(str.splitlines(), tests) + +build_sig_cache = {} + +def get_build_sigs(build, cache=False): + if cache and build in build_sig_cache: + return build_sig_cache[build] + rpms = session.listRPMs(buildID=build) + keys = {} + if not rpms: + # for non-rpm builds we have no easy way of checking signatures + ret = build_sig_cache[build] = [] + return ret + else: + #TODO - multicall helps, but it might be good to have a more robust server-side call + session.multicall = True + for rpminfo in rpms: + session.queryRPMSigs(rpm_id=rpminfo['id']) + for rpminfo, [sigs] in zip(rpms, session.multiCall()): + for sig in sigs: + if sig['sigkey']: + keys.setdefault(sig['sigkey'], 1) + ret = build_sig_cache[build] = keys.keys() + return ret + +def handle_prune(): + """Untag old builds according to policy + + If purge is True, will also attempt to delete the pruned builds afterwards + """ + #read policy + if not options.config or not options.config.has_option('prune', 'policy'): + print "Skipping prune step. No policies available." + return + #policies = read_policies(options.policy_file) + policies = scan_policies(options.config.get('prune', 'policy')) + for action in policies.all_actions(): + if action not in ("keep", "untag", "skip"): + raise Exception, "Invalid action: %s" % action + if options.debug: + pprint.pprint(policies.ruleset) + #get tags + tags = session.listTags(queryOpts={'order': 'name'}) + untagged = {} + build_ids = {} + for taginfo in tags: + tagname = taginfo['name'] + if tagname == options.trashcan_tag: + if options.debug: + print "Skipping trashcan tag: %s" % tagname + continue + if not check_tag(tagname): + #if options.debug: + # print "skipping tag due to filter: %s" % tagname + continue + bypass = False + if taginfo['locked']: + if options.bypass_locks: + for pattern in options.bypass_locks: + if fnmatch.fnmatch(tagname, pattern): + bypass = True + break + if bypass: + print "Bypassing lock on tag: %s" % tagname + else: + if options.debug: + print "skipping locked tag: %s" % tagname + continue + if options.debug: + print "Pruning tag: %s" % tagname + #get builds + history = session.tagHistory(tag=tagname, active=True, queryOpts={'order': '-create_ts'}) + if not history: + if options.debug: + print "No history for %s" % tagname + continue + pkghist = {} + for h in history: + if taginfo['maven_include_all'] and h['maven_build_id']: + pkghist.setdefault(h['name'] + '-' + h['version'], []).append(h) + else: + pkghist.setdefault(h['name'], []).append(h) + pkgs = pkghist.keys() + pkgs.sort() + for pkg in pkgs: + if not check_package(pkg): + #if options.debug: + # print "skipping package due to filter: %s" % pkg + continue + if options.debug: + print pkg + hist = pkghist[pkg] + #these are the *active* history entries for tag/pkg + skipped = 0 + for order, entry in enumerate(hist): + # get sig data + nvr = "%(name)s-%(version)s-%(release)s" % entry + data = { + 'tagname' : tagname, + 'pkgname' : pkg, + 'order': order - skipped, + 'ts' : entry['create_ts'], + 'nvr' : nvr, + } + data = LazyDict(data) + data['keys'] = LazyValue(get_build_sigs, (entry['build_id'],), {'cache':True}) + data['volname'] = LazyValue(lambda x: session.getBuild(x).get('volume_name'), + (entry['build_id'],), cache=True) + build_ids[nvr] = entry['build_id'] + action = policies.apply(data) + if action is None: + if options.debug: + print "No policy for %s (%s)" % (nvr, tagname) + if action == 'skip': + skipped += 1 + if options.debug: + print policies.last_rule() + print "%s: %s (%s, %i)" % (action, nvr, tagname, order) + if action == 'untag': + if options.test: + print "Would have untagged %s from %s" % (nvr, tagname) + untagged.setdefault(nvr, {})[tagname] = 1 + else: + print "Untagging build %s from %s" % (nvr, tagname) + try: + session.untagBuildBypass(taginfo['id'], entry['build_id'], force=bypass) + untagged.setdefault(nvr, {})[tagname] = 1 + except (xmlrpclib.Fault, koji.GenericError), e: + print "Warning: untag operation failed: %s" % e + pass + # if action == 'keep' do nothing + if options.purge and untagged: + print "Attempting to purge %i builds" % len(untagged) + for nvr in untagged: + build_id = build_ids[nvr] + tags = [t['name'] for t in session.listTags(build_id)] + if options.test: + #filted out the tags we would have dropped above + tags = [t for t in tags if t not in untagged[nvr]] + if tags: + #still tagged somewhere + print "Skipping %s, still tagged: %s" % (nvr, tags) + continue + #check cached sigs first to save a little time + if build_id in build_sig_cache: + keys = build_sig_cache[build_id] + if protected_sig(keys): + print "Skipping %s, signatures: %s" % (nvr, keys) + continue + #recheck signatures in case build was signed during run + keys = get_build_sigs(build_id, cache=False) + if protected_sig(keys): + print "Skipping %s, signatures: %s" % (nvr, keys) + continue + + if options.test: + print "Would have deleted build: %s" % nvr + else: + print "Deleting untagged build: %s" % nvr + try: + session.deleteBuild(build_id, strict=False) + except (xmlrpclib.Fault, koji.GenericError), e: + print "Warning: deletion failed: %s" % e + #server issue + pass + +if __name__ == "__main__": + + options, args = get_options() + + session_opts = koji.grab_session_options(options) + session = koji.ClientSession(options.server, session_opts) + rv = 0 + try: + if not options.skip_main: + rv = main(args) + if not rv: + rv = 0 + except KeyboardInterrupt: + pass + except SystemExit: + rv = 1 + #except: + # if options.debug: + # raise + # else: + # exctype, value = sys.exc_info()[:2] + # rv = 1 + # print "%s: %s" % (exctype, value) + try: + session.logout() + except: + pass + if not options.skip_main: + sys.exit(rv) diff --git a/roles/koji_hub/tasks/main.yml b/roles/koji_hub/tasks/main.yml index b3f861f92d..de351d91a0 100644 --- a/roles/koji_hub/tasks/main.yml +++ b/roles/koji_hub/tasks/main.yml @@ -20,6 +20,15 @@ - packages - koji_hub +# Fix kerberos support for koji-gc +# https://pagure.io/koji/pull-request/246 +# https://pagure.io/koji/pull-request/248 +- name: hotfix - koji-gc kerberos support + copy: src=koji-gc.py-hotfix dest=/usr/sbin/koji-gc mode=0755 + tags: + - koji_hub + - hotfix + - name: make koji pki directory file: state=directory path=/etc/pki/koji/ owner=root group=root