diff --git a/roles/koji_hub/files/koji-gc.py-hotfix b/roles/koji_hub/files/koji-gc.py-hotfix deleted file mode 100755 index 4116f3b184..0000000000 --- a/roles/koji_hub/files/koji-gc.py-hotfix +++ /dev/null @@ -1,927 +0,0 @@ -#!/usr/bin/python - -# koji-gc: a garbage collection tool for Koji -# Copyright (c) 2007-2014 Red Hat, Inc. -# -# Authors: -# Mike McLean - -try: - import krbV -except ImportError: # pragma: no cover - pass -import koji -from koji.util import LazyDict, LazyValue -import koji.policy -import ConfigParser -from email.MIMEText import MIMEText -import fnmatch -import optparse -import os -import pprint -import smtplib -import socket # for socket.error -import sys -import time -import xmlrpclib # for ProtocolError and Fault - - -OptionParser = optparse.OptionParser -if optparse.__version__ == "1.4.1+": - def _op_error(self, msg): - self.print_usage(sys.stderr) - msg = "%s: error: %s\n" % (self._get_prog_name(), msg) - if msg: - sys.stderr.write(msg) - sys.exit(2) - OptionParser.error = _op_error - - -def _(args): - """Stub function for translation""" - return args - -def get_options(): - """process options from command line and config file""" - - usage = _("%prog [options]") - parser = OptionParser(usage=usage) - parser.add_option("-c", "--config-file", metavar="FILE", - help=_("use alternate configuration file")) - parser.add_option("--keytab", help=_("specify a Kerberos keytab to use")) - parser.add_option("--principal", help=_("specify a Kerberos principal to use")) - parser.add_option("--krbservice", default="host", - help=_("the service name of the principal being used by the hub")) - parser.add_option("--runas", metavar="USER", - help=_("run as the specified user (requires special privileges)")) - parser.add_option("--user", help=_("specify user")) - parser.add_option("--password", help=_("specify password")) - parser.add_option("--noauth", action="store_true", default=False, - help=_("do not authenticate")) - parser.add_option("--network-hack", action="store_true", default=False, - help=optparse.SUPPRESS_HELP) # no longer used - parser.add_option("--cert", default='/etc/koji-gc/client.crt', - help=_("Client SSL certificate file for authentication")) - parser.add_option("--ca", default='', - help=_("ignored")) # FIXME: remove in next major release - parser.add_option("--serverca", default='/etc/koji-gc/serverca.crt', - help=_("CA cert file that issued the hub certificate")) - parser.add_option("-n", "--test", action="store_true", default=False, - help=_("test mode")) - parser.add_option("-d", "--debug", action="store_true", default=False, - help=_("show debug output")) - parser.add_option("--debug-xmlrpc", action="store_true", default=False, - help=_("show xmlrpc debug output")) - parser.add_option("--smtp-host", metavar="HOST", - help=_("specify smtp server for notifications")) - parser.add_option("--no-mail", action='store_false', default=True, dest="mail", - help=_("don't send notifications")) - parser.add_option("--send-mail", action='store_true', dest="mail", - help=_("send notifications")) - parser.add_option("--email-domain", default="fedoraproject.org", - help=_("Email domain appended to Koji username for notifications")) - parser.add_option("--from-addr", default="Koji Build System ", - help=_("From address for notifications")) - parser.add_option("--action", help=_("action(s) to take")) - parser.add_option("--delay", metavar="INTERVAL", default = '5 days', - help="time before eligible builds are placed in trashcan") - parser.add_option("--grace-period", default='4 weeks', metavar="INTERVAL", - help="time that builds are held in trashcan") - parser.add_option("--skip-main", action="store_true", default=False, - help=_("don't actually run main")) - parser.add_option("--unprotected-keys", metavar="KEYS", - help=_("allow builds signed with these keys to be deleted")) - parser.add_option("--tag-filter", "--tag", metavar="PATTERN", action="append", - help=_("Process only tags matching PATTERN when pruning")) - parser.add_option("--ignore-tags", metavar="PATTERN", action="append", - help=_("Ignore tags matching PATTERN when pruning")) - parser.add_option("--pkg-filter", "--pkg", "--package", - metavar="PATTERN", action='append', - help=_("Process only packages matching PATTERN")) - parser.add_option("--bypass-locks", metavar="PATTERN", action="append", - help=_("Bypass locks for tags matching PATTERN")) - parser.add_option("--purge", action="store_true", default=False, - help=_("When pruning, attempt to delete the builds that are untagged")) - parser.add_option("--trashcan-tag", default='trashcan', metavar="TAG", - help=_("specify an alternate trashcan tag")) - parser.add_option("--weburl", default="http://localhost/koji", metavar="URL", - help=_("url of koji web server (for use in notifications)")) - parser.add_option("-s", "--server", help=_("url of koji XMLRPC server")) - #parse once to get the config file - (options, args) = parser.parse_args() - - defaults = parser.get_default_values() - config = ConfigParser.ConfigParser() - cf = getattr(options, 'config_file', None) - if cf: - if not os.access(cf, os.F_OK): - parser.error(_("No such file: %s") % cf) - assert False # pragma: no cover - else: - cf = '/etc/koji-gc/koji-gc.conf' - if not os.access(cf, os.F_OK): - cf = None - if not cf: - print "no config file" - config = None - else: - config.read(cf) - #allow config file to update defaults for certain options - cfgmap = [ - ['keytab', None, 'string'], - ['principal', None, 'string'], - ['krbservice', None, 'string'], - ['krb_rdns', None, 'boolean'], - ['runas', None, 'string'], - ['user', None, 'string'], - ['password', None, 'string'], - ['noauth', None, 'boolean'], - ['cert', None, 'string'], - ['ca', None, 'string'], # FIXME: remove in next major release - ['serverca', None, 'string'], - ['server', None, 'string'], - ['weburl', None, 'string'], - ['smtp_host', None, 'string'], - ['from_addr', None, 'string'], - ['email_domain', None, 'string'], - ['mail', None, 'boolean'], - ['delay', None, 'string'], - ['unprotected_keys', None, 'string'], - ['grace_period', None, 'string'], - ['trashcan_tag', None, 'string'], - ['use_old_ssl', None, 'boolean'], - ['no_ssl_verify', None, 'boolean'], - ['timeout', None, 'integer'], - ] - for name, alias, type in cfgmap: - if alias is None: - alias = ('main', name) - if config.has_option(*alias): - if options.debug: - print "Using option %s from config file" % (alias,) - if type == 'integer': - setattr(defaults, name, config.getint(*alias)) - elif type == 'boolean': - setattr(defaults, name, config.getboolean(*alias)) - else: - setattr(defaults, name, config.get(*alias)) - #parse again with defaults - (options, args) = parser.parse_args(values=defaults) - options.config = config - - #figure out actions - actions = ('prune', 'trash', 'delete', 'salvage') - if options.action: - options.action = options.action.lower().replace(',',' ').split() - for x in options.action: - if x not in actions: - parser.error(_("Invalid action: %s") % x) - else: - options.action = ('delete', 'prune', 'trash') - - #split patterns for unprotected keys - if options.unprotected_keys: - options.unprotected_key_patterns = options.unprotected_keys.replace(',',' ').split() - else: - options.unprotected_key_patterns = [] - - #parse key aliases - options.key_aliases = {} - try: - if config and config.has_option('main', 'key_aliases'): - for line in config.get('main','key_aliases').splitlines(): - parts = line.split() - if len(parts) < 2: - continue - options.key_aliases[parts[0].upper()] = parts[1] - except ValueError, e: - print e - parser.error(_("Invalid key alias data in config: %s") % config.get('main','key_aliases')) - - #parse time intervals - for key in ('delay', 'grace_period'): - try: - value = getattr(options, key) - value = parse_duration(value) - setattr(options, key, value) - if options.debug: - print "%s: %s seconds" % (key, value) - except ValueError: - parser.error(_("Invalid time interval: %s") % value) - - return options, args - -def check_tag(name): - """Check tag name against options and determine if we should process it - - The ignore option takes priority here. - Returns True if we should process the tag, False otherwise - """ - if options.ignore_tags: - for pattern in options.ignore_tags: - if fnmatch.fnmatch(name, options.tag_filter): - return False - if options.tag_filter: - for pattern in options.tag_filter: - if fnmatch.fnmatch(name, pattern): - return True - #doesn't match any pattern in filter - return False - else: - #not ignored and no filter specified - return True - -def check_package(name): - """Check package name against options and determine if we should process it - - Returns True if we should process the package, False otherwise - """ - if options.pkg_filter: - for pattern in options.pkg_filter: - if fnmatch.fnmatch(name, pattern): - return True - #doesn't match any pattern in filter - return False - else: - #no filter specified - return True - -time_units = { - 'second' : 1, - 'minute' : 60, - 'hour' : 3600, - 'day' : 86400, - 'week' : 604800, -} -time_unit_aliases = [ - #[unit, alias, alias, ...] - ['week', 'weeks', 'wk', 'wks'], - ['hour', 'hours', 'hr', 'hrs'], - ['day', 'days'], - ['minute', 'minutes', 'min', 'mins'], - ['second', 'seconds', 'sec', 'secs', 's'], -] -def parse_duration(str): - """Parse time duration from string, returns duration in seconds""" - ret = 0 - n = None - unit = None - def parse_num(s): - try: - return int(s) - except ValueError: - pass - try: - return float(s) - except ValueError: - pass - return None - for x in str.split(): - if n is None: - n = parse_num(x) - if n is not None: - continue - #perhaps the unit is appended w/o a space - for names in time_unit_aliases: - for name in names: - if x.endswith(name): - n = parse_num(x[:-len(name)]) - if n is None: - continue - unit = names[0] - # combined at end - break - if unit: - break - else: - raise ValueError, "Invalid time interval: %s" % str - if unit is None: - x = x.lower() - for names in time_unit_aliases: - for name in names: - if x == name: - unit = names[0] - break - if unit: - break - else: - raise ValueError, "Invalid time interval: %s" % str - ret += n * time_units[unit] - n = None - unit = None - return ret - -def error(msg=None, code=1): - if msg: - sys.stderr.write(msg + "\n") - sys.stderr.flush() - sys.exit(code) - -def warn(msg): - sys.stderr.write(msg + "\n") - sys.stderr.flush() - -def ensure_connection(session): - try: - ret = session.getAPIVersion() - except xmlrpclib.ProtocolError: - error(_("Error: Unable to connect to server")) - if ret != koji.API_VERSION: - warn(_("WARNING: The server is at API version %d and the client is at %d" % (ret, koji.API_VERSION))) - -def has_krb_creds(): - if not sys.modules.has_key('krbV'): - return False - try: - ctx = krbV.default_context() - ccache = ctx.default_ccache() - princ = ccache.principal() - return True - except krbV.Krb5Error: - return False - -def activate_session(session): - """Test and login the session is applicable""" - global options - if options.noauth: - #skip authentication - pass - elif os.path.isfile(options.cert): - # authenticate using SSL client cert - session.ssl_login(options.cert, None, options.serverca, proxyuser=options.runas) - elif options.user: - #authenticate using user/password - session.login() - elif has_krb_creds() or (options.keytab and options.principal): - try: - if options.keytab and options.principal: - session.krb_login(principal=options.principal, keytab=options.keytab, proxyuser=options.runas) - else: - session.krb_login(proxyuser=options.runas) - except krbV.Krb5Error, e: - error(_("Kerberos authentication failed: %s (%s)") % (e.args[1], e.args[0])) - except socket.error, e: - warn(_("Could not connect to Kerberos authentication service: '%s'") % e.args[1]) - if not options.noauth and not session.logged_in: - error(_("Error: unable to log in, no authentication methods available")) - ensure_connection(session) - if options.debug: - print "successfully connected to hub" - -def send_warning_notice(owner_name, builds): - if not options.mail: - return - if not builds: - print "Warning: empty build list. No notice sent" - return - head = """\ -The following build(s) are unreferenced and have been marked for -deletion. They will be held in the trashcan tag for a grace period. -At the end of that period they will be deleted permanently. This -garbage collection is a normal part of build system operation. -Please see the following url for more information: - - http://fedoraproject.org/wiki/Koji/GarbageCollection""" - fmt="""\ -Build: %%(name)s-%%(version)s-%%(release)s -%s/buildinfo?buildID=%%(id)i""" % options.weburl - middle = '\n\n'.join([fmt % b for b in builds]) - tail = """\ -If you would like to protect any of these builds from deletion, please -refer to the document linked above for instructions.""" - - msg = MIMEText('\n\n'.join([head, middle, tail])) - if len(builds) == 1: - msg['Subject'] = "1 build marked for deletion" - else: - msg['Subject'] = "%i builds marked for deletion" % len(builds) - msg['From'] = options.from_addr - msg['To'] = "%s@%s" % (owner_name, options.email_domain) #XXX! - msg['X-Koji-Builder'] = owner_name - if options.test: - if options.debug: - print str(msg) - else: - print "Would have sent warning notice to %s" % msg['To'] - else: - if options.debug: - print "Sending warning notice to %s" % msg['To'] - s = smtplib.SMTP(options.smtp_host) - s.sendmail(msg['From'], msg['To'], msg.as_string()) - s.quit() - - -def main(args): - activate_session(session) - for x in options.action: - globals()['handle_' + x]() - - -def handle_trash(): - print "Getting untagged builds..." - untagged = session.untaggedBuilds() - print "...got %i builds" % len(untagged) - min_age = options.delay - trashcan_tag = options.trashcan_tag - #Step 1: place unreferenced builds into trashcan - i = 0 - N = len(untagged) - to_trash = [] - for binfo in untagged: - i += 1 - nvr = "%(name)s-%(version)s-%(release)s" % binfo - if not check_package(binfo['name']): - if options.debug: - print "[%i/%i] Skipping package: %s" % (i, N, nvr) - continue - try: - refs = session.buildReferences(binfo['id'], limit=10) - except xmlrpclib.Fault: - print "[%i/%i] Error checking references for %s. Skipping" % (i, N, nvr) - continue - #XXX - this is more data than we need - # also, this call takes waaaay longer than it should - if refs['tags']: - # must have been tagged just now - print "[%i/%i] Build is tagged [?]: %s" % (i, N, nvr) - continue - if refs['rpms']: - if options.debug: - print "[%i/%i] Build has %i rpm references: %s" % (i, N, len(refs['rpms']), nvr) - #pprint.pprint(refs['rpms']) - continue - if refs['archives']: - if options.debug: - print "[%i/%i] Build has %i archive references: %s" % (i, N, len(refs['archives']), nvr) - #pprint.pprint(refs['archives']) - continue - ts = refs['last_used'] - if ts: - #work around server bug - if isinstance(ts, list): - ts = ts[0] - #XXX - should really check time server side - if options.debug: - print "[%i/%i] Build has been used in a buildroot: %s" % (i, N, nvr) - print "Last_used: %r" % ts - age = time.time() - ts - if age < min_age: - continue - #see how long build has been untagged - history = session.tagHistory(build=binfo['id']) - age = None - binfo2 = None - if not history: - #never tagged, we'll have to use the build create time - binfo2 = session.getBuild(binfo['id']) - ts = binfo2.get('creation_ts') - if ts is None: - # older api with no good way to get a proper timestamp for - # a build, so we have the following hack - task_id = binfo2.get('task_id') - if task_id: - tinfo = session.getTaskInfo(task_id) - if tinfo['completion_ts']: - age = time.time() - tinfo['completion_ts'] - else: - age = time.time() - ts - else: - history = [(h['revoke_event'],h) for h in history] - last = max(history)[1] - if not last['revoke_event']: - #this might happen if the build was tagged just now - print "[%i/%i] Warning: build not untagged: %s" % (i, N, nvr) - continue - age = time.time() - last['revoke_ts'] - if age is not None and age < min_age: - if options.debug: - print "[%i/%i] Build untagged only recently: %s" % (i, N, nvr) - continue - #check build signatures - keys = get_build_sigs(binfo['id'], cache=True) - if keys and options.debug: - print "Build: %s, Keys: %s" % (nvr, keys) - if protected_sig(keys): - print "Skipping build %s. Keys: %s" % (nvr, keys) - continue - - #ok, go ahead add it to the list - if binfo2 is None: - binfo2 = session.getBuild(binfo['id']) - binfo2['nvr'] = nvr - print "[%i/%i] Adding build to trash list: %s" % (i, N, nvr) - to_trash.append(binfo2) - - #process to_trash - #group by owner so we can reduce the number of notices - by_owner = {} - for binfo in to_trash: - by_owner.setdefault(binfo['owner_name'], []).append(binfo) - owners = by_owner.keys() - owners.sort() - for owner_name in owners: - builds = [(b['nvr'], b) for b in by_owner[owner_name]] - builds.sort() - send_warning_notice(owner_name, [x[1] for x in builds]) - for nvr, binfo in builds: - if options.test: - print "Would have moved to trashcan: %s" % nvr - else: - if options.debug: - print "Moving to trashcan: %s" % nvr - #figure out package owner - count = {} - for pkg in session.listPackages(pkgID=binfo['name']): - count.setdefault(pkg['owner_id'], 0) - count[pkg['owner_id']] += 1 - if not count: - print "Warning: no owner for %s, using build owner" % nvr - #best we can do currently - owner = binfo['owner_id'] - else: - owner = max([(n, k) for k, n in count.iteritems()])[1] - session.packageListAdd(trashcan_tag, binfo['name'], owner) - session.tagBuildBypass(trashcan_tag, binfo['id'], force=True) - -def protected_sig(keys): - """Check list of keys and see if any are protected - - returns True if ANY are protected (not on unprotected list) - returns False if ALL are unprotected - """ - for key in keys: - if not key: - continue - if not sigmatch(key, options.unprotected_key_patterns): - #this key is protected - return True - return False - - -def handle_salvage(): - """Reclaim builds from trashcan - - Check builds in trashcan for new tags or references and salvage them - (remove trashcan tag) if appropriate. - - The delete action also does this, but this is for when you want to - run this action only.""" - return handle_delete(just_salvage=True) - -def salvage_build(binfo): - """Removes trashcan tag from a build and prints a message""" - if options.test: - print "Would have untagged from trashcan: %(nvr)s" % binfo - else: - if options.debug: - print "Untagging from trashcan: %(nvr)s" % binfo - session.untagBuildBypass(options.trashcan_tag, binfo['id'], force=True) - -def handle_delete(just_salvage=False): - """Delete builds that have been in the trashcan for long enough - - If just_salvage is True, goes into salvage mode. In salvage mode it only - reclaims referenced builds from the trashcan, it does not perform any - deletes - """ - print "Getting list of builds in trash..." - trashcan_tag = options.trashcan_tag - trash = [(b['nvr'], b) for b in session.listTagged(trashcan_tag)] - trash.sort() - print "...got %i builds" % len(trash) - #XXX - it would be better if there were more appropriate server calls for this - grace_period = options.grace_period - for nvr, binfo in trash: - # see if build has been tagged elsewhere - if not check_package(binfo['name']): - if options.debug: - print "Skipping package: %s" % nvr - continue - tags = [t['name'] for t in session.listTags(build=binfo['id']) if t['name'] != trashcan_tag] - if tags: - print "Build %s tagged elsewhere: %s" % (nvr, tags) - salvage_build(binfo) - continue - #check build signatures - keys = get_build_sigs(binfo['id'], cache=False) - if keys and options.debug: - print "Build: %s, Keys: %s" % (nvr, keys) - if protected_sig(keys): - print "Salvaging signed build %s. Keys: %s" % (nvr, keys) - salvage_build(binfo) - continue - if just_salvage: - # skip the rest when salvaging - continue - # determine how long this build has been in the trashcan - history = session.tagHistory(build=binfo['id'], tag=trashcan_tag) - current = [x for x in history if x['active']] - if not current: - #untagged just now? - print "Warning: history missing for %s" % nvr - pprint.pprint(binfo) - pprint.pprint(history) - continue - assert len(current) == 1 #see db constraint - current = current[0] - age = time.time() - current['create_ts'] - if age < grace_period: - if options.debug: - print "Skipping build %s, age=%i" % (nvr, age) - continue - - # go ahead and delete - if options.test: - print "Would have deleted build from trashcan: %s" % nvr - else: - print "Deleting build: %s" % nvr - session.untagBuildBypass(trashcan_tag, binfo['id']) - try: - session.deleteBuild(binfo['id']) - except (xmlrpclib.Fault, koji.GenericError), e: - print "Warning: deletion failed: %s" % e - #server issue - pass - #TODO - log details for delete failures - - -class TagPruneTest(koji.policy.MatchTest): - name = 'tag' - field = 'tagname' - - -class PackagePruneTest(koji.policy.MatchTest): - name = 'package' - field = 'pkgname' - - -class VolumePruneTest(koji.policy.MatchTest): - name = 'volume' - field = 'volname' - - -class SigPruneTest(koji.policy.BaseSimpleTest): - name = 'sig' - - def run(self, data): - # true if any of the keys match any of the patterns - patterns = self.str.split()[1:] - for key in data['keys']: - if sigmatch(key, patterns): - return True - return False - - -def sigmatch(key, patterns): - """Test whether a key id matches any of the given patterns - - Supports key aliases - """ - if not isinstance(patterns, (tuple, list)): - patterns = (patterns,) - for pat in patterns: - if fnmatch.fnmatch(key, pat): - return True - alias = options.key_aliases.get(key.upper()) - if alias is not None and fnmatch.fnmatch(alias, pat): - return True - return False - - -class OrderPruneTest(koji.policy.CompareTest): - name = 'order' - field = 'order' - allow_float = False - - -class AgePruneTest(koji.policy.BaseSimpleTest): - name = 'age' - cmp_idx = koji.policy.CompareTest.operators - - def __init__(self, str): - """Read the test parameters from string""" - super(AgePruneTest, self).__init__(str) - self.cmp, value = str.split(None, 2)[1:] - self.func = self.cmp_idx.get(self.cmp, None) - if self.func is None: - raise Exception, "Invalid comparison in test: %s" % str - self.span = parse_duration(value) - - def run(self, data): - return self.func(time.time() - data['ts'], self.span) - - -def read_policies(fn=None): - """Read tag gc policies from file - - The expected format as follows - test [params] [&& test [params] ...] :: (keep|untag|skip) - """ - fo = file(fn, 'r') - tests = koji.policy.findSimpleTests(globals()) - ret = koji.policy.SimpleRuleSet(fo, tests) - fo.close() - return ret - -def scan_policies(str): - """Read tag gc policies from a string - - The expected format as follows - test [params] [&& test [params] ...] :: (keep|untag|skip) - """ - tests = koji.policy.findSimpleTests(globals()) - return koji.policy.SimpleRuleSet(str.splitlines(), tests) - -build_sig_cache = {} - -def get_build_sigs(build, cache=False): - if cache and build in build_sig_cache: - return build_sig_cache[build] - rpms = session.listRPMs(buildID=build) - keys = {} - if not rpms: - # for non-rpm builds we have no easy way of checking signatures - ret = build_sig_cache[build] = [] - return ret - else: - #TODO - multicall helps, but it might be good to have a more robust server-side call - session.multicall = True - for rpminfo in rpms: - session.queryRPMSigs(rpm_id=rpminfo['id']) - for rpminfo, [sigs] in zip(rpms, session.multiCall()): - for sig in sigs: - if sig['sigkey']: - keys.setdefault(sig['sigkey'], 1) - ret = build_sig_cache[build] = keys.keys() - return ret - -def handle_prune(): - """Untag old builds according to policy - - If purge is True, will also attempt to delete the pruned builds afterwards - """ - #read policy - if not options.config or not options.config.has_option('prune', 'policy'): - print "Skipping prune step. No policies available." - return - #policies = read_policies(options.policy_file) - policies = scan_policies(options.config.get('prune', 'policy')) - for action in policies.all_actions(): - if action not in ("keep", "untag", "skip"): - raise Exception, "Invalid action: %s" % action - if options.debug: - pprint.pprint(policies.ruleset) - #get tags - tags = session.listTags(queryOpts={'order': 'name'}) - untagged = {} - build_ids = {} - for taginfo in tags: - tagname = taginfo['name'] - if tagname == options.trashcan_tag: - if options.debug: - print "Skipping trashcan tag: %s" % tagname - continue - if not check_tag(tagname): - #if options.debug: - # print "skipping tag due to filter: %s" % tagname - continue - bypass = False - if taginfo['locked']: - if options.bypass_locks: - for pattern in options.bypass_locks: - if fnmatch.fnmatch(tagname, pattern): - bypass = True - break - if bypass: - print "Bypassing lock on tag: %s" % tagname - else: - if options.debug: - print "skipping locked tag: %s" % tagname - continue - if options.debug: - print "Pruning tag: %s" % tagname - #get builds - history = session.tagHistory(tag=tagname, active=True, queryOpts={'order': '-create_ts'}) - if not history: - if options.debug: - print "No history for %s" % tagname - continue - pkghist = {} - for h in history: - if taginfo['maven_include_all'] and h['maven_build_id']: - pkghist.setdefault(h['name'] + '-' + h['version'], []).append(h) - else: - pkghist.setdefault(h['name'], []).append(h) - pkgs = pkghist.keys() - pkgs.sort() - for pkg in pkgs: - if not check_package(pkg): - #if options.debug: - # print "skipping package due to filter: %s" % pkg - continue - if options.debug: - print pkg - hist = pkghist[pkg] - #these are the *active* history entries for tag/pkg - skipped = 0 - for order, entry in enumerate(hist): - # get sig data - nvr = "%(name)s-%(version)s-%(release)s" % entry - data = { - 'tagname' : tagname, - 'pkgname' : pkg, - 'order': order - skipped, - 'ts' : entry['create_ts'], - 'nvr' : nvr, - } - data = LazyDict(data) - data['keys'] = LazyValue(get_build_sigs, (entry['build_id'],), {'cache':True}) - data['volname'] = LazyValue(lambda x: session.getBuild(x).get('volume_name'), - (entry['build_id'],), cache=True) - build_ids[nvr] = entry['build_id'] - action = policies.apply(data) - if action is None: - if options.debug: - print "No policy for %s (%s)" % (nvr, tagname) - if action == 'skip': - skipped += 1 - if options.debug: - print policies.last_rule() - print "%s: %s (%s, %i)" % (action, nvr, tagname, order) - if action == 'untag': - if options.test: - print "Would have untagged %s from %s" % (nvr, tagname) - untagged.setdefault(nvr, {})[tagname] = 1 - else: - print "Untagging build %s from %s" % (nvr, tagname) - try: - session.untagBuildBypass(taginfo['id'], entry['build_id'], force=bypass) - untagged.setdefault(nvr, {})[tagname] = 1 - except (xmlrpclib.Fault, koji.GenericError), e: - print "Warning: untag operation failed: %s" % e - pass - # if action == 'keep' do nothing - if options.purge and untagged: - print "Attempting to purge %i builds" % len(untagged) - for nvr in untagged: - build_id = build_ids[nvr] - tags = [t['name'] for t in session.listTags(build_id)] - if options.test: - #filted out the tags we would have dropped above - tags = [t for t in tags if t not in untagged[nvr]] - if tags: - #still tagged somewhere - print "Skipping %s, still tagged: %s" % (nvr, tags) - continue - #check cached sigs first to save a little time - if build_id in build_sig_cache: - keys = build_sig_cache[build_id] - if protected_sig(keys): - print "Skipping %s, signatures: %s" % (nvr, keys) - continue - #recheck signatures in case build was signed during run - keys = get_build_sigs(build_id, cache=False) - if protected_sig(keys): - print "Skipping %s, signatures: %s" % (nvr, keys) - continue - - if options.test: - print "Would have deleted build: %s" % nvr - else: - print "Deleting untagged build: %s" % nvr - try: - session.deleteBuild(build_id, strict=False) - except (xmlrpclib.Fault, koji.GenericError), e: - print "Warning: deletion failed: %s" % e - #server issue - pass - -if __name__ == "__main__": - - options, args = get_options() - - session_opts = koji.grab_session_options(options) - session = koji.ClientSession(options.server, session_opts) - rv = 0 - try: - if not options.skip_main: - rv = main(args) - if not rv: - rv = 0 - except KeyboardInterrupt: - pass - except SystemExit: - rv = 1 - #except: - # if options.debug: - # raise - # else: - # exctype, value = sys.exc_info()[:2] - # rv = 1 - # print "%s: %s" % (exctype, value) - try: - session.logout() - except: - pass - if not options.skip_main: - sys.exit(rv) diff --git a/roles/koji_hub/tasks/main.yml b/roles/koji_hub/tasks/main.yml index de351d91a0..b3f861f92d 100644 --- a/roles/koji_hub/tasks/main.yml +++ b/roles/koji_hub/tasks/main.yml @@ -20,15 +20,6 @@ - packages - koji_hub -# Fix kerberos support for koji-gc -# https://pagure.io/koji/pull-request/246 -# https://pagure.io/koji/pull-request/248 -- name: hotfix - koji-gc kerberos support - copy: src=koji-gc.py-hotfix dest=/usr/sbin/koji-gc mode=0755 - tags: - - koji_hub - - hotfix - - name: make koji pki directory file: state=directory path=/etc/pki/koji/ owner=root group=root