remove no longer needed hotfix koji-gc
Signed-off-by: Dennis Gilmore <ausil@fedoraproject.org>
This commit is contained in:
parent
1a20e6dd03
commit
3199c15328
2 changed files with 0 additions and 936 deletions
|
|
@ -1,927 +0,0 @@
|
||||||
#!/usr/bin/python
|
|
||||||
|
|
||||||
# koji-gc: a garbage collection tool for Koji
|
|
||||||
# Copyright (c) 2007-2014 Red Hat, Inc.
|
|
||||||
#
|
|
||||||
# Authors:
|
|
||||||
# Mike McLean <mikem@redhat.com>
|
|
||||||
|
|
||||||
try:
|
|
||||||
import krbV
|
|
||||||
except ImportError: # pragma: no cover
|
|
||||||
pass
|
|
||||||
import koji
|
|
||||||
from koji.util import LazyDict, LazyValue
|
|
||||||
import koji.policy
|
|
||||||
import ConfigParser
|
|
||||||
from email.MIMEText import MIMEText
|
|
||||||
import fnmatch
|
|
||||||
import optparse
|
|
||||||
import os
|
|
||||||
import pprint
|
|
||||||
import smtplib
|
|
||||||
import socket # for socket.error
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
import xmlrpclib # for ProtocolError and Fault
|
|
||||||
|
|
||||||
|
|
||||||
OptionParser = optparse.OptionParser
|
|
||||||
if optparse.__version__ == "1.4.1+":
|
|
||||||
def _op_error(self, msg):
|
|
||||||
self.print_usage(sys.stderr)
|
|
||||||
msg = "%s: error: %s\n" % (self._get_prog_name(), msg)
|
|
||||||
if msg:
|
|
||||||
sys.stderr.write(msg)
|
|
||||||
sys.exit(2)
|
|
||||||
OptionParser.error = _op_error
|
|
||||||
|
|
||||||
|
|
||||||
def _(args):
|
|
||||||
"""Stub function for translation"""
|
|
||||||
return args
|
|
||||||
|
|
||||||
def get_options():
|
|
||||||
"""process options from command line and config file"""
|
|
||||||
|
|
||||||
usage = _("%prog [options]")
|
|
||||||
parser = OptionParser(usage=usage)
|
|
||||||
parser.add_option("-c", "--config-file", metavar="FILE",
|
|
||||||
help=_("use alternate configuration file"))
|
|
||||||
parser.add_option("--keytab", help=_("specify a Kerberos keytab to use"))
|
|
||||||
parser.add_option("--principal", help=_("specify a Kerberos principal to use"))
|
|
||||||
parser.add_option("--krbservice", default="host",
|
|
||||||
help=_("the service name of the principal being used by the hub"))
|
|
||||||
parser.add_option("--runas", metavar="USER",
|
|
||||||
help=_("run as the specified user (requires special privileges)"))
|
|
||||||
parser.add_option("--user", help=_("specify user"))
|
|
||||||
parser.add_option("--password", help=_("specify password"))
|
|
||||||
parser.add_option("--noauth", action="store_true", default=False,
|
|
||||||
help=_("do not authenticate"))
|
|
||||||
parser.add_option("--network-hack", action="store_true", default=False,
|
|
||||||
help=optparse.SUPPRESS_HELP) # no longer used
|
|
||||||
parser.add_option("--cert", default='/etc/koji-gc/client.crt',
|
|
||||||
help=_("Client SSL certificate file for authentication"))
|
|
||||||
parser.add_option("--ca", default='',
|
|
||||||
help=_("ignored")) # FIXME: remove in next major release
|
|
||||||
parser.add_option("--serverca", default='/etc/koji-gc/serverca.crt',
|
|
||||||
help=_("CA cert file that issued the hub certificate"))
|
|
||||||
parser.add_option("-n", "--test", action="store_true", default=False,
|
|
||||||
help=_("test mode"))
|
|
||||||
parser.add_option("-d", "--debug", action="store_true", default=False,
|
|
||||||
help=_("show debug output"))
|
|
||||||
parser.add_option("--debug-xmlrpc", action="store_true", default=False,
|
|
||||||
help=_("show xmlrpc debug output"))
|
|
||||||
parser.add_option("--smtp-host", metavar="HOST",
|
|
||||||
help=_("specify smtp server for notifications"))
|
|
||||||
parser.add_option("--no-mail", action='store_false', default=True, dest="mail",
|
|
||||||
help=_("don't send notifications"))
|
|
||||||
parser.add_option("--send-mail", action='store_true', dest="mail",
|
|
||||||
help=_("send notifications"))
|
|
||||||
parser.add_option("--email-domain", default="fedoraproject.org",
|
|
||||||
help=_("Email domain appended to Koji username for notifications"))
|
|
||||||
parser.add_option("--from-addr", default="Koji Build System <buildsys@example.com>",
|
|
||||||
help=_("From address for notifications"))
|
|
||||||
parser.add_option("--action", help=_("action(s) to take"))
|
|
||||||
parser.add_option("--delay", metavar="INTERVAL", default = '5 days',
|
|
||||||
help="time before eligible builds are placed in trashcan")
|
|
||||||
parser.add_option("--grace-period", default='4 weeks', metavar="INTERVAL",
|
|
||||||
help="time that builds are held in trashcan")
|
|
||||||
parser.add_option("--skip-main", action="store_true", default=False,
|
|
||||||
help=_("don't actually run main"))
|
|
||||||
parser.add_option("--unprotected-keys", metavar="KEYS",
|
|
||||||
help=_("allow builds signed with these keys to be deleted"))
|
|
||||||
parser.add_option("--tag-filter", "--tag", metavar="PATTERN", action="append",
|
|
||||||
help=_("Process only tags matching PATTERN when pruning"))
|
|
||||||
parser.add_option("--ignore-tags", metavar="PATTERN", action="append",
|
|
||||||
help=_("Ignore tags matching PATTERN when pruning"))
|
|
||||||
parser.add_option("--pkg-filter", "--pkg", "--package",
|
|
||||||
metavar="PATTERN", action='append',
|
|
||||||
help=_("Process only packages matching PATTERN"))
|
|
||||||
parser.add_option("--bypass-locks", metavar="PATTERN", action="append",
|
|
||||||
help=_("Bypass locks for tags matching PATTERN"))
|
|
||||||
parser.add_option("--purge", action="store_true", default=False,
|
|
||||||
help=_("When pruning, attempt to delete the builds that are untagged"))
|
|
||||||
parser.add_option("--trashcan-tag", default='trashcan', metavar="TAG",
|
|
||||||
help=_("specify an alternate trashcan tag"))
|
|
||||||
parser.add_option("--weburl", default="http://localhost/koji", metavar="URL",
|
|
||||||
help=_("url of koji web server (for use in notifications)"))
|
|
||||||
parser.add_option("-s", "--server", help=_("url of koji XMLRPC server"))
|
|
||||||
#parse once to get the config file
|
|
||||||
(options, args) = parser.parse_args()
|
|
||||||
|
|
||||||
defaults = parser.get_default_values()
|
|
||||||
config = ConfigParser.ConfigParser()
|
|
||||||
cf = getattr(options, 'config_file', None)
|
|
||||||
if cf:
|
|
||||||
if not os.access(cf, os.F_OK):
|
|
||||||
parser.error(_("No such file: %s") % cf)
|
|
||||||
assert False # pragma: no cover
|
|
||||||
else:
|
|
||||||
cf = '/etc/koji-gc/koji-gc.conf'
|
|
||||||
if not os.access(cf, os.F_OK):
|
|
||||||
cf = None
|
|
||||||
if not cf:
|
|
||||||
print "no config file"
|
|
||||||
config = None
|
|
||||||
else:
|
|
||||||
config.read(cf)
|
|
||||||
#allow config file to update defaults for certain options
|
|
||||||
cfgmap = [
|
|
||||||
['keytab', None, 'string'],
|
|
||||||
['principal', None, 'string'],
|
|
||||||
['krbservice', None, 'string'],
|
|
||||||
['krb_rdns', None, 'boolean'],
|
|
||||||
['runas', None, 'string'],
|
|
||||||
['user', None, 'string'],
|
|
||||||
['password', None, 'string'],
|
|
||||||
['noauth', None, 'boolean'],
|
|
||||||
['cert', None, 'string'],
|
|
||||||
['ca', None, 'string'], # FIXME: remove in next major release
|
|
||||||
['serverca', None, 'string'],
|
|
||||||
['server', None, 'string'],
|
|
||||||
['weburl', None, 'string'],
|
|
||||||
['smtp_host', None, 'string'],
|
|
||||||
['from_addr', None, 'string'],
|
|
||||||
['email_domain', None, 'string'],
|
|
||||||
['mail', None, 'boolean'],
|
|
||||||
['delay', None, 'string'],
|
|
||||||
['unprotected_keys', None, 'string'],
|
|
||||||
['grace_period', None, 'string'],
|
|
||||||
['trashcan_tag', None, 'string'],
|
|
||||||
['use_old_ssl', None, 'boolean'],
|
|
||||||
['no_ssl_verify', None, 'boolean'],
|
|
||||||
['timeout', None, 'integer'],
|
|
||||||
]
|
|
||||||
for name, alias, type in cfgmap:
|
|
||||||
if alias is None:
|
|
||||||
alias = ('main', name)
|
|
||||||
if config.has_option(*alias):
|
|
||||||
if options.debug:
|
|
||||||
print "Using option %s from config file" % (alias,)
|
|
||||||
if type == 'integer':
|
|
||||||
setattr(defaults, name, config.getint(*alias))
|
|
||||||
elif type == 'boolean':
|
|
||||||
setattr(defaults, name, config.getboolean(*alias))
|
|
||||||
else:
|
|
||||||
setattr(defaults, name, config.get(*alias))
|
|
||||||
#parse again with defaults
|
|
||||||
(options, args) = parser.parse_args(values=defaults)
|
|
||||||
options.config = config
|
|
||||||
|
|
||||||
#figure out actions
|
|
||||||
actions = ('prune', 'trash', 'delete', 'salvage')
|
|
||||||
if options.action:
|
|
||||||
options.action = options.action.lower().replace(',',' ').split()
|
|
||||||
for x in options.action:
|
|
||||||
if x not in actions:
|
|
||||||
parser.error(_("Invalid action: %s") % x)
|
|
||||||
else:
|
|
||||||
options.action = ('delete', 'prune', 'trash')
|
|
||||||
|
|
||||||
#split patterns for unprotected keys
|
|
||||||
if options.unprotected_keys:
|
|
||||||
options.unprotected_key_patterns = options.unprotected_keys.replace(',',' ').split()
|
|
||||||
else:
|
|
||||||
options.unprotected_key_patterns = []
|
|
||||||
|
|
||||||
#parse key aliases
|
|
||||||
options.key_aliases = {}
|
|
||||||
try:
|
|
||||||
if config and config.has_option('main', 'key_aliases'):
|
|
||||||
for line in config.get('main','key_aliases').splitlines():
|
|
||||||
parts = line.split()
|
|
||||||
if len(parts) < 2:
|
|
||||||
continue
|
|
||||||
options.key_aliases[parts[0].upper()] = parts[1]
|
|
||||||
except ValueError, e:
|
|
||||||
print e
|
|
||||||
parser.error(_("Invalid key alias data in config: %s") % config.get('main','key_aliases'))
|
|
||||||
|
|
||||||
#parse time intervals
|
|
||||||
for key in ('delay', 'grace_period'):
|
|
||||||
try:
|
|
||||||
value = getattr(options, key)
|
|
||||||
value = parse_duration(value)
|
|
||||||
setattr(options, key, value)
|
|
||||||
if options.debug:
|
|
||||||
print "%s: %s seconds" % (key, value)
|
|
||||||
except ValueError:
|
|
||||||
parser.error(_("Invalid time interval: %s") % value)
|
|
||||||
|
|
||||||
return options, args
|
|
||||||
|
|
||||||
def check_tag(name):
|
|
||||||
"""Check tag name against options and determine if we should process it
|
|
||||||
|
|
||||||
The ignore option takes priority here.
|
|
||||||
Returns True if we should process the tag, False otherwise
|
|
||||||
"""
|
|
||||||
if options.ignore_tags:
|
|
||||||
for pattern in options.ignore_tags:
|
|
||||||
if fnmatch.fnmatch(name, options.tag_filter):
|
|
||||||
return False
|
|
||||||
if options.tag_filter:
|
|
||||||
for pattern in options.tag_filter:
|
|
||||||
if fnmatch.fnmatch(name, pattern):
|
|
||||||
return True
|
|
||||||
#doesn't match any pattern in filter
|
|
||||||
return False
|
|
||||||
else:
|
|
||||||
#not ignored and no filter specified
|
|
||||||
return True
|
|
||||||
|
|
||||||
def check_package(name):
|
|
||||||
"""Check package name against options and determine if we should process it
|
|
||||||
|
|
||||||
Returns True if we should process the package, False otherwise
|
|
||||||
"""
|
|
||||||
if options.pkg_filter:
|
|
||||||
for pattern in options.pkg_filter:
|
|
||||||
if fnmatch.fnmatch(name, pattern):
|
|
||||||
return True
|
|
||||||
#doesn't match any pattern in filter
|
|
||||||
return False
|
|
||||||
else:
|
|
||||||
#no filter specified
|
|
||||||
return True
|
|
||||||
|
|
||||||
time_units = {
|
|
||||||
'second' : 1,
|
|
||||||
'minute' : 60,
|
|
||||||
'hour' : 3600,
|
|
||||||
'day' : 86400,
|
|
||||||
'week' : 604800,
|
|
||||||
}
|
|
||||||
time_unit_aliases = [
|
|
||||||
#[unit, alias, alias, ...]
|
|
||||||
['week', 'weeks', 'wk', 'wks'],
|
|
||||||
['hour', 'hours', 'hr', 'hrs'],
|
|
||||||
['day', 'days'],
|
|
||||||
['minute', 'minutes', 'min', 'mins'],
|
|
||||||
['second', 'seconds', 'sec', 'secs', 's'],
|
|
||||||
]
|
|
||||||
def parse_duration(str):
|
|
||||||
"""Parse time duration from string, returns duration in seconds"""
|
|
||||||
ret = 0
|
|
||||||
n = None
|
|
||||||
unit = None
|
|
||||||
def parse_num(s):
|
|
||||||
try:
|
|
||||||
return int(s)
|
|
||||||
except ValueError:
|
|
||||||
pass
|
|
||||||
try:
|
|
||||||
return float(s)
|
|
||||||
except ValueError:
|
|
||||||
pass
|
|
||||||
return None
|
|
||||||
for x in str.split():
|
|
||||||
if n is None:
|
|
||||||
n = parse_num(x)
|
|
||||||
if n is not None:
|
|
||||||
continue
|
|
||||||
#perhaps the unit is appended w/o a space
|
|
||||||
for names in time_unit_aliases:
|
|
||||||
for name in names:
|
|
||||||
if x.endswith(name):
|
|
||||||
n = parse_num(x[:-len(name)])
|
|
||||||
if n is None:
|
|
||||||
continue
|
|
||||||
unit = names[0]
|
|
||||||
# combined at end
|
|
||||||
break
|
|
||||||
if unit:
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
raise ValueError, "Invalid time interval: %s" % str
|
|
||||||
if unit is None:
|
|
||||||
x = x.lower()
|
|
||||||
for names in time_unit_aliases:
|
|
||||||
for name in names:
|
|
||||||
if x == name:
|
|
||||||
unit = names[0]
|
|
||||||
break
|
|
||||||
if unit:
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
raise ValueError, "Invalid time interval: %s" % str
|
|
||||||
ret += n * time_units[unit]
|
|
||||||
n = None
|
|
||||||
unit = None
|
|
||||||
return ret
|
|
||||||
|
|
||||||
def error(msg=None, code=1):
|
|
||||||
if msg:
|
|
||||||
sys.stderr.write(msg + "\n")
|
|
||||||
sys.stderr.flush()
|
|
||||||
sys.exit(code)
|
|
||||||
|
|
||||||
def warn(msg):
|
|
||||||
sys.stderr.write(msg + "\n")
|
|
||||||
sys.stderr.flush()
|
|
||||||
|
|
||||||
def ensure_connection(session):
|
|
||||||
try:
|
|
||||||
ret = session.getAPIVersion()
|
|
||||||
except xmlrpclib.ProtocolError:
|
|
||||||
error(_("Error: Unable to connect to server"))
|
|
||||||
if ret != koji.API_VERSION:
|
|
||||||
warn(_("WARNING: The server is at API version %d and the client is at %d" % (ret, koji.API_VERSION)))
|
|
||||||
|
|
||||||
def has_krb_creds():
|
|
||||||
if not sys.modules.has_key('krbV'):
|
|
||||||
return False
|
|
||||||
try:
|
|
||||||
ctx = krbV.default_context()
|
|
||||||
ccache = ctx.default_ccache()
|
|
||||||
princ = ccache.principal()
|
|
||||||
return True
|
|
||||||
except krbV.Krb5Error:
|
|
||||||
return False
|
|
||||||
|
|
||||||
def activate_session(session):
|
|
||||||
"""Test and login the session is applicable"""
|
|
||||||
global options
|
|
||||||
if options.noauth:
|
|
||||||
#skip authentication
|
|
||||||
pass
|
|
||||||
elif os.path.isfile(options.cert):
|
|
||||||
# authenticate using SSL client cert
|
|
||||||
session.ssl_login(options.cert, None, options.serverca, proxyuser=options.runas)
|
|
||||||
elif options.user:
|
|
||||||
#authenticate using user/password
|
|
||||||
session.login()
|
|
||||||
elif has_krb_creds() or (options.keytab and options.principal):
|
|
||||||
try:
|
|
||||||
if options.keytab and options.principal:
|
|
||||||
session.krb_login(principal=options.principal, keytab=options.keytab, proxyuser=options.runas)
|
|
||||||
else:
|
|
||||||
session.krb_login(proxyuser=options.runas)
|
|
||||||
except krbV.Krb5Error, e:
|
|
||||||
error(_("Kerberos authentication failed: %s (%s)") % (e.args[1], e.args[0]))
|
|
||||||
except socket.error, e:
|
|
||||||
warn(_("Could not connect to Kerberos authentication service: '%s'") % e.args[1])
|
|
||||||
if not options.noauth and not session.logged_in:
|
|
||||||
error(_("Error: unable to log in, no authentication methods available"))
|
|
||||||
ensure_connection(session)
|
|
||||||
if options.debug:
|
|
||||||
print "successfully connected to hub"
|
|
||||||
|
|
||||||
def send_warning_notice(owner_name, builds):
|
|
||||||
if not options.mail:
|
|
||||||
return
|
|
||||||
if not builds:
|
|
||||||
print "Warning: empty build list. No notice sent"
|
|
||||||
return
|
|
||||||
head = """\
|
|
||||||
The following build(s) are unreferenced and have been marked for
|
|
||||||
deletion. They will be held in the trashcan tag for a grace period.
|
|
||||||
At the end of that period they will be deleted permanently. This
|
|
||||||
garbage collection is a normal part of build system operation.
|
|
||||||
Please see the following url for more information:
|
|
||||||
|
|
||||||
http://fedoraproject.org/wiki/Koji/GarbageCollection"""
|
|
||||||
fmt="""\
|
|
||||||
Build: %%(name)s-%%(version)s-%%(release)s
|
|
||||||
%s/buildinfo?buildID=%%(id)i""" % options.weburl
|
|
||||||
middle = '\n\n'.join([fmt % b for b in builds])
|
|
||||||
tail = """\
|
|
||||||
If you would like to protect any of these builds from deletion, please
|
|
||||||
refer to the document linked above for instructions."""
|
|
||||||
|
|
||||||
msg = MIMEText('\n\n'.join([head, middle, tail]))
|
|
||||||
if len(builds) == 1:
|
|
||||||
msg['Subject'] = "1 build marked for deletion"
|
|
||||||
else:
|
|
||||||
msg['Subject'] = "%i builds marked for deletion" % len(builds)
|
|
||||||
msg['From'] = options.from_addr
|
|
||||||
msg['To'] = "%s@%s" % (owner_name, options.email_domain) #XXX!
|
|
||||||
msg['X-Koji-Builder'] = owner_name
|
|
||||||
if options.test:
|
|
||||||
if options.debug:
|
|
||||||
print str(msg)
|
|
||||||
else:
|
|
||||||
print "Would have sent warning notice to %s" % msg['To']
|
|
||||||
else:
|
|
||||||
if options.debug:
|
|
||||||
print "Sending warning notice to %s" % msg['To']
|
|
||||||
s = smtplib.SMTP(options.smtp_host)
|
|
||||||
s.sendmail(msg['From'], msg['To'], msg.as_string())
|
|
||||||
s.quit()
|
|
||||||
|
|
||||||
|
|
||||||
def main(args):
|
|
||||||
activate_session(session)
|
|
||||||
for x in options.action:
|
|
||||||
globals()['handle_' + x]()
|
|
||||||
|
|
||||||
|
|
||||||
def handle_trash():
|
|
||||||
print "Getting untagged builds..."
|
|
||||||
untagged = session.untaggedBuilds()
|
|
||||||
print "...got %i builds" % len(untagged)
|
|
||||||
min_age = options.delay
|
|
||||||
trashcan_tag = options.trashcan_tag
|
|
||||||
#Step 1: place unreferenced builds into trashcan
|
|
||||||
i = 0
|
|
||||||
N = len(untagged)
|
|
||||||
to_trash = []
|
|
||||||
for binfo in untagged:
|
|
||||||
i += 1
|
|
||||||
nvr = "%(name)s-%(version)s-%(release)s" % binfo
|
|
||||||
if not check_package(binfo['name']):
|
|
||||||
if options.debug:
|
|
||||||
print "[%i/%i] Skipping package: %s" % (i, N, nvr)
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
refs = session.buildReferences(binfo['id'], limit=10)
|
|
||||||
except xmlrpclib.Fault:
|
|
||||||
print "[%i/%i] Error checking references for %s. Skipping" % (i, N, nvr)
|
|
||||||
continue
|
|
||||||
#XXX - this is more data than we need
|
|
||||||
# also, this call takes waaaay longer than it should
|
|
||||||
if refs['tags']:
|
|
||||||
# must have been tagged just now
|
|
||||||
print "[%i/%i] Build is tagged [?]: %s" % (i, N, nvr)
|
|
||||||
continue
|
|
||||||
if refs['rpms']:
|
|
||||||
if options.debug:
|
|
||||||
print "[%i/%i] Build has %i rpm references: %s" % (i, N, len(refs['rpms']), nvr)
|
|
||||||
#pprint.pprint(refs['rpms'])
|
|
||||||
continue
|
|
||||||
if refs['archives']:
|
|
||||||
if options.debug:
|
|
||||||
print "[%i/%i] Build has %i archive references: %s" % (i, N, len(refs['archives']), nvr)
|
|
||||||
#pprint.pprint(refs['archives'])
|
|
||||||
continue
|
|
||||||
ts = refs['last_used']
|
|
||||||
if ts:
|
|
||||||
#work around server bug
|
|
||||||
if isinstance(ts, list):
|
|
||||||
ts = ts[0]
|
|
||||||
#XXX - should really check time server side
|
|
||||||
if options.debug:
|
|
||||||
print "[%i/%i] Build has been used in a buildroot: %s" % (i, N, nvr)
|
|
||||||
print "Last_used: %r" % ts
|
|
||||||
age = time.time() - ts
|
|
||||||
if age < min_age:
|
|
||||||
continue
|
|
||||||
#see how long build has been untagged
|
|
||||||
history = session.tagHistory(build=binfo['id'])
|
|
||||||
age = None
|
|
||||||
binfo2 = None
|
|
||||||
if not history:
|
|
||||||
#never tagged, we'll have to use the build create time
|
|
||||||
binfo2 = session.getBuild(binfo['id'])
|
|
||||||
ts = binfo2.get('creation_ts')
|
|
||||||
if ts is None:
|
|
||||||
# older api with no good way to get a proper timestamp for
|
|
||||||
# a build, so we have the following hack
|
|
||||||
task_id = binfo2.get('task_id')
|
|
||||||
if task_id:
|
|
||||||
tinfo = session.getTaskInfo(task_id)
|
|
||||||
if tinfo['completion_ts']:
|
|
||||||
age = time.time() - tinfo['completion_ts']
|
|
||||||
else:
|
|
||||||
age = time.time() - ts
|
|
||||||
else:
|
|
||||||
history = [(h['revoke_event'],h) for h in history]
|
|
||||||
last = max(history)[1]
|
|
||||||
if not last['revoke_event']:
|
|
||||||
#this might happen if the build was tagged just now
|
|
||||||
print "[%i/%i] Warning: build not untagged: %s" % (i, N, nvr)
|
|
||||||
continue
|
|
||||||
age = time.time() - last['revoke_ts']
|
|
||||||
if age is not None and age < min_age:
|
|
||||||
if options.debug:
|
|
||||||
print "[%i/%i] Build untagged only recently: %s" % (i, N, nvr)
|
|
||||||
continue
|
|
||||||
#check build signatures
|
|
||||||
keys = get_build_sigs(binfo['id'], cache=True)
|
|
||||||
if keys and options.debug:
|
|
||||||
print "Build: %s, Keys: %s" % (nvr, keys)
|
|
||||||
if protected_sig(keys):
|
|
||||||
print "Skipping build %s. Keys: %s" % (nvr, keys)
|
|
||||||
continue
|
|
||||||
|
|
||||||
#ok, go ahead add it to the list
|
|
||||||
if binfo2 is None:
|
|
||||||
binfo2 = session.getBuild(binfo['id'])
|
|
||||||
binfo2['nvr'] = nvr
|
|
||||||
print "[%i/%i] Adding build to trash list: %s" % (i, N, nvr)
|
|
||||||
to_trash.append(binfo2)
|
|
||||||
|
|
||||||
#process to_trash
|
|
||||||
#group by owner so we can reduce the number of notices
|
|
||||||
by_owner = {}
|
|
||||||
for binfo in to_trash:
|
|
||||||
by_owner.setdefault(binfo['owner_name'], []).append(binfo)
|
|
||||||
owners = by_owner.keys()
|
|
||||||
owners.sort()
|
|
||||||
for owner_name in owners:
|
|
||||||
builds = [(b['nvr'], b) for b in by_owner[owner_name]]
|
|
||||||
builds.sort()
|
|
||||||
send_warning_notice(owner_name, [x[1] for x in builds])
|
|
||||||
for nvr, binfo in builds:
|
|
||||||
if options.test:
|
|
||||||
print "Would have moved to trashcan: %s" % nvr
|
|
||||||
else:
|
|
||||||
if options.debug:
|
|
||||||
print "Moving to trashcan: %s" % nvr
|
|
||||||
#figure out package owner
|
|
||||||
count = {}
|
|
||||||
for pkg in session.listPackages(pkgID=binfo['name']):
|
|
||||||
count.setdefault(pkg['owner_id'], 0)
|
|
||||||
count[pkg['owner_id']] += 1
|
|
||||||
if not count:
|
|
||||||
print "Warning: no owner for %s, using build owner" % nvr
|
|
||||||
#best we can do currently
|
|
||||||
owner = binfo['owner_id']
|
|
||||||
else:
|
|
||||||
owner = max([(n, k) for k, n in count.iteritems()])[1]
|
|
||||||
session.packageListAdd(trashcan_tag, binfo['name'], owner)
|
|
||||||
session.tagBuildBypass(trashcan_tag, binfo['id'], force=True)
|
|
||||||
|
|
||||||
def protected_sig(keys):
|
|
||||||
"""Check list of keys and see if any are protected
|
|
||||||
|
|
||||||
returns True if ANY are protected (not on unprotected list)
|
|
||||||
returns False if ALL are unprotected
|
|
||||||
"""
|
|
||||||
for key in keys:
|
|
||||||
if not key:
|
|
||||||
continue
|
|
||||||
if not sigmatch(key, options.unprotected_key_patterns):
|
|
||||||
#this key is protected
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def handle_salvage():
|
|
||||||
"""Reclaim builds from trashcan
|
|
||||||
|
|
||||||
Check builds in trashcan for new tags or references and salvage them
|
|
||||||
(remove trashcan tag) if appropriate.
|
|
||||||
|
|
||||||
The delete action also does this, but this is for when you want to
|
|
||||||
run this action only."""
|
|
||||||
return handle_delete(just_salvage=True)
|
|
||||||
|
|
||||||
def salvage_build(binfo):
|
|
||||||
"""Removes trashcan tag from a build and prints a message"""
|
|
||||||
if options.test:
|
|
||||||
print "Would have untagged from trashcan: %(nvr)s" % binfo
|
|
||||||
else:
|
|
||||||
if options.debug:
|
|
||||||
print "Untagging from trashcan: %(nvr)s" % binfo
|
|
||||||
session.untagBuildBypass(options.trashcan_tag, binfo['id'], force=True)
|
|
||||||
|
|
||||||
def handle_delete(just_salvage=False):
|
|
||||||
"""Delete builds that have been in the trashcan for long enough
|
|
||||||
|
|
||||||
If just_salvage is True, goes into salvage mode. In salvage mode it only
|
|
||||||
reclaims referenced builds from the trashcan, it does not perform any
|
|
||||||
deletes
|
|
||||||
"""
|
|
||||||
print "Getting list of builds in trash..."
|
|
||||||
trashcan_tag = options.trashcan_tag
|
|
||||||
trash = [(b['nvr'], b) for b in session.listTagged(trashcan_tag)]
|
|
||||||
trash.sort()
|
|
||||||
print "...got %i builds" % len(trash)
|
|
||||||
#XXX - it would be better if there were more appropriate server calls for this
|
|
||||||
grace_period = options.grace_period
|
|
||||||
for nvr, binfo in trash:
|
|
||||||
# see if build has been tagged elsewhere
|
|
||||||
if not check_package(binfo['name']):
|
|
||||||
if options.debug:
|
|
||||||
print "Skipping package: %s" % nvr
|
|
||||||
continue
|
|
||||||
tags = [t['name'] for t in session.listTags(build=binfo['id']) if t['name'] != trashcan_tag]
|
|
||||||
if tags:
|
|
||||||
print "Build %s tagged elsewhere: %s" % (nvr, tags)
|
|
||||||
salvage_build(binfo)
|
|
||||||
continue
|
|
||||||
#check build signatures
|
|
||||||
keys = get_build_sigs(binfo['id'], cache=False)
|
|
||||||
if keys and options.debug:
|
|
||||||
print "Build: %s, Keys: %s" % (nvr, keys)
|
|
||||||
if protected_sig(keys):
|
|
||||||
print "Salvaging signed build %s. Keys: %s" % (nvr, keys)
|
|
||||||
salvage_build(binfo)
|
|
||||||
continue
|
|
||||||
if just_salvage:
|
|
||||||
# skip the rest when salvaging
|
|
||||||
continue
|
|
||||||
# determine how long this build has been in the trashcan
|
|
||||||
history = session.tagHistory(build=binfo['id'], tag=trashcan_tag)
|
|
||||||
current = [x for x in history if x['active']]
|
|
||||||
if not current:
|
|
||||||
#untagged just now?
|
|
||||||
print "Warning: history missing for %s" % nvr
|
|
||||||
pprint.pprint(binfo)
|
|
||||||
pprint.pprint(history)
|
|
||||||
continue
|
|
||||||
assert len(current) == 1 #see db constraint
|
|
||||||
current = current[0]
|
|
||||||
age = time.time() - current['create_ts']
|
|
||||||
if age < grace_period:
|
|
||||||
if options.debug:
|
|
||||||
print "Skipping build %s, age=%i" % (nvr, age)
|
|
||||||
continue
|
|
||||||
|
|
||||||
# go ahead and delete
|
|
||||||
if options.test:
|
|
||||||
print "Would have deleted build from trashcan: %s" % nvr
|
|
||||||
else:
|
|
||||||
print "Deleting build: %s" % nvr
|
|
||||||
session.untagBuildBypass(trashcan_tag, binfo['id'])
|
|
||||||
try:
|
|
||||||
session.deleteBuild(binfo['id'])
|
|
||||||
except (xmlrpclib.Fault, koji.GenericError), e:
|
|
||||||
print "Warning: deletion failed: %s" % e
|
|
||||||
#server issue
|
|
||||||
pass
|
|
||||||
#TODO - log details for delete failures
|
|
||||||
|
|
||||||
|
|
||||||
class TagPruneTest(koji.policy.MatchTest):
|
|
||||||
name = 'tag'
|
|
||||||
field = 'tagname'
|
|
||||||
|
|
||||||
|
|
||||||
class PackagePruneTest(koji.policy.MatchTest):
|
|
||||||
name = 'package'
|
|
||||||
field = 'pkgname'
|
|
||||||
|
|
||||||
|
|
||||||
class VolumePruneTest(koji.policy.MatchTest):
|
|
||||||
name = 'volume'
|
|
||||||
field = 'volname'
|
|
||||||
|
|
||||||
|
|
||||||
class SigPruneTest(koji.policy.BaseSimpleTest):
|
|
||||||
name = 'sig'
|
|
||||||
|
|
||||||
def run(self, data):
|
|
||||||
# true if any of the keys match any of the patterns
|
|
||||||
patterns = self.str.split()[1:]
|
|
||||||
for key in data['keys']:
|
|
||||||
if sigmatch(key, patterns):
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def sigmatch(key, patterns):
|
|
||||||
"""Test whether a key id matches any of the given patterns
|
|
||||||
|
|
||||||
Supports key aliases
|
|
||||||
"""
|
|
||||||
if not isinstance(patterns, (tuple, list)):
|
|
||||||
patterns = (patterns,)
|
|
||||||
for pat in patterns:
|
|
||||||
if fnmatch.fnmatch(key, pat):
|
|
||||||
return True
|
|
||||||
alias = options.key_aliases.get(key.upper())
|
|
||||||
if alias is not None and fnmatch.fnmatch(alias, pat):
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
class OrderPruneTest(koji.policy.CompareTest):
|
|
||||||
name = 'order'
|
|
||||||
field = 'order'
|
|
||||||
allow_float = False
|
|
||||||
|
|
||||||
|
|
||||||
class AgePruneTest(koji.policy.BaseSimpleTest):
|
|
||||||
name = 'age'
|
|
||||||
cmp_idx = koji.policy.CompareTest.operators
|
|
||||||
|
|
||||||
def __init__(self, str):
|
|
||||||
"""Read the test parameters from string"""
|
|
||||||
super(AgePruneTest, self).__init__(str)
|
|
||||||
self.cmp, value = str.split(None, 2)[1:]
|
|
||||||
self.func = self.cmp_idx.get(self.cmp, None)
|
|
||||||
if self.func is None:
|
|
||||||
raise Exception, "Invalid comparison in test: %s" % str
|
|
||||||
self.span = parse_duration(value)
|
|
||||||
|
|
||||||
def run(self, data):
|
|
||||||
return self.func(time.time() - data['ts'], self.span)
|
|
||||||
|
|
||||||
|
|
||||||
def read_policies(fn=None):
|
|
||||||
"""Read tag gc policies from file
|
|
||||||
|
|
||||||
The expected format as follows
|
|
||||||
test [params] [&& test [params] ...] :: (keep|untag|skip)
|
|
||||||
"""
|
|
||||||
fo = file(fn, 'r')
|
|
||||||
tests = koji.policy.findSimpleTests(globals())
|
|
||||||
ret = koji.policy.SimpleRuleSet(fo, tests)
|
|
||||||
fo.close()
|
|
||||||
return ret
|
|
||||||
|
|
||||||
def scan_policies(str):
|
|
||||||
"""Read tag gc policies from a string
|
|
||||||
|
|
||||||
The expected format as follows
|
|
||||||
test [params] [&& test [params] ...] :: (keep|untag|skip)
|
|
||||||
"""
|
|
||||||
tests = koji.policy.findSimpleTests(globals())
|
|
||||||
return koji.policy.SimpleRuleSet(str.splitlines(), tests)
|
|
||||||
|
|
||||||
build_sig_cache = {}
|
|
||||||
|
|
||||||
def get_build_sigs(build, cache=False):
|
|
||||||
if cache and build in build_sig_cache:
|
|
||||||
return build_sig_cache[build]
|
|
||||||
rpms = session.listRPMs(buildID=build)
|
|
||||||
keys = {}
|
|
||||||
if not rpms:
|
|
||||||
# for non-rpm builds we have no easy way of checking signatures
|
|
||||||
ret = build_sig_cache[build] = []
|
|
||||||
return ret
|
|
||||||
else:
|
|
||||||
#TODO - multicall helps, but it might be good to have a more robust server-side call
|
|
||||||
session.multicall = True
|
|
||||||
for rpminfo in rpms:
|
|
||||||
session.queryRPMSigs(rpm_id=rpminfo['id'])
|
|
||||||
for rpminfo, [sigs] in zip(rpms, session.multiCall()):
|
|
||||||
for sig in sigs:
|
|
||||||
if sig['sigkey']:
|
|
||||||
keys.setdefault(sig['sigkey'], 1)
|
|
||||||
ret = build_sig_cache[build] = keys.keys()
|
|
||||||
return ret
|
|
||||||
|
|
||||||
def handle_prune():
|
|
||||||
"""Untag old builds according to policy
|
|
||||||
|
|
||||||
If purge is True, will also attempt to delete the pruned builds afterwards
|
|
||||||
"""
|
|
||||||
#read policy
|
|
||||||
if not options.config or not options.config.has_option('prune', 'policy'):
|
|
||||||
print "Skipping prune step. No policies available."
|
|
||||||
return
|
|
||||||
#policies = read_policies(options.policy_file)
|
|
||||||
policies = scan_policies(options.config.get('prune', 'policy'))
|
|
||||||
for action in policies.all_actions():
|
|
||||||
if action not in ("keep", "untag", "skip"):
|
|
||||||
raise Exception, "Invalid action: %s" % action
|
|
||||||
if options.debug:
|
|
||||||
pprint.pprint(policies.ruleset)
|
|
||||||
#get tags
|
|
||||||
tags = session.listTags(queryOpts={'order': 'name'})
|
|
||||||
untagged = {}
|
|
||||||
build_ids = {}
|
|
||||||
for taginfo in tags:
|
|
||||||
tagname = taginfo['name']
|
|
||||||
if tagname == options.trashcan_tag:
|
|
||||||
if options.debug:
|
|
||||||
print "Skipping trashcan tag: %s" % tagname
|
|
||||||
continue
|
|
||||||
if not check_tag(tagname):
|
|
||||||
#if options.debug:
|
|
||||||
# print "skipping tag due to filter: %s" % tagname
|
|
||||||
continue
|
|
||||||
bypass = False
|
|
||||||
if taginfo['locked']:
|
|
||||||
if options.bypass_locks:
|
|
||||||
for pattern in options.bypass_locks:
|
|
||||||
if fnmatch.fnmatch(tagname, pattern):
|
|
||||||
bypass = True
|
|
||||||
break
|
|
||||||
if bypass:
|
|
||||||
print "Bypassing lock on tag: %s" % tagname
|
|
||||||
else:
|
|
||||||
if options.debug:
|
|
||||||
print "skipping locked tag: %s" % tagname
|
|
||||||
continue
|
|
||||||
if options.debug:
|
|
||||||
print "Pruning tag: %s" % tagname
|
|
||||||
#get builds
|
|
||||||
history = session.tagHistory(tag=tagname, active=True, queryOpts={'order': '-create_ts'})
|
|
||||||
if not history:
|
|
||||||
if options.debug:
|
|
||||||
print "No history for %s" % tagname
|
|
||||||
continue
|
|
||||||
pkghist = {}
|
|
||||||
for h in history:
|
|
||||||
if taginfo['maven_include_all'] and h['maven_build_id']:
|
|
||||||
pkghist.setdefault(h['name'] + '-' + h['version'], []).append(h)
|
|
||||||
else:
|
|
||||||
pkghist.setdefault(h['name'], []).append(h)
|
|
||||||
pkgs = pkghist.keys()
|
|
||||||
pkgs.sort()
|
|
||||||
for pkg in pkgs:
|
|
||||||
if not check_package(pkg):
|
|
||||||
#if options.debug:
|
|
||||||
# print "skipping package due to filter: %s" % pkg
|
|
||||||
continue
|
|
||||||
if options.debug:
|
|
||||||
print pkg
|
|
||||||
hist = pkghist[pkg]
|
|
||||||
#these are the *active* history entries for tag/pkg
|
|
||||||
skipped = 0
|
|
||||||
for order, entry in enumerate(hist):
|
|
||||||
# get sig data
|
|
||||||
nvr = "%(name)s-%(version)s-%(release)s" % entry
|
|
||||||
data = {
|
|
||||||
'tagname' : tagname,
|
|
||||||
'pkgname' : pkg,
|
|
||||||
'order': order - skipped,
|
|
||||||
'ts' : entry['create_ts'],
|
|
||||||
'nvr' : nvr,
|
|
||||||
}
|
|
||||||
data = LazyDict(data)
|
|
||||||
data['keys'] = LazyValue(get_build_sigs, (entry['build_id'],), {'cache':True})
|
|
||||||
data['volname'] = LazyValue(lambda x: session.getBuild(x).get('volume_name'),
|
|
||||||
(entry['build_id'],), cache=True)
|
|
||||||
build_ids[nvr] = entry['build_id']
|
|
||||||
action = policies.apply(data)
|
|
||||||
if action is None:
|
|
||||||
if options.debug:
|
|
||||||
print "No policy for %s (%s)" % (nvr, tagname)
|
|
||||||
if action == 'skip':
|
|
||||||
skipped += 1
|
|
||||||
if options.debug:
|
|
||||||
print policies.last_rule()
|
|
||||||
print "%s: %s (%s, %i)" % (action, nvr, tagname, order)
|
|
||||||
if action == 'untag':
|
|
||||||
if options.test:
|
|
||||||
print "Would have untagged %s from %s" % (nvr, tagname)
|
|
||||||
untagged.setdefault(nvr, {})[tagname] = 1
|
|
||||||
else:
|
|
||||||
print "Untagging build %s from %s" % (nvr, tagname)
|
|
||||||
try:
|
|
||||||
session.untagBuildBypass(taginfo['id'], entry['build_id'], force=bypass)
|
|
||||||
untagged.setdefault(nvr, {})[tagname] = 1
|
|
||||||
except (xmlrpclib.Fault, koji.GenericError), e:
|
|
||||||
print "Warning: untag operation failed: %s" % e
|
|
||||||
pass
|
|
||||||
# if action == 'keep' do nothing
|
|
||||||
if options.purge and untagged:
|
|
||||||
print "Attempting to purge %i builds" % len(untagged)
|
|
||||||
for nvr in untagged:
|
|
||||||
build_id = build_ids[nvr]
|
|
||||||
tags = [t['name'] for t in session.listTags(build_id)]
|
|
||||||
if options.test:
|
|
||||||
#filted out the tags we would have dropped above
|
|
||||||
tags = [t for t in tags if t not in untagged[nvr]]
|
|
||||||
if tags:
|
|
||||||
#still tagged somewhere
|
|
||||||
print "Skipping %s, still tagged: %s" % (nvr, tags)
|
|
||||||
continue
|
|
||||||
#check cached sigs first to save a little time
|
|
||||||
if build_id in build_sig_cache:
|
|
||||||
keys = build_sig_cache[build_id]
|
|
||||||
if protected_sig(keys):
|
|
||||||
print "Skipping %s, signatures: %s" % (nvr, keys)
|
|
||||||
continue
|
|
||||||
#recheck signatures in case build was signed during run
|
|
||||||
keys = get_build_sigs(build_id, cache=False)
|
|
||||||
if protected_sig(keys):
|
|
||||||
print "Skipping %s, signatures: %s" % (nvr, keys)
|
|
||||||
continue
|
|
||||||
|
|
||||||
if options.test:
|
|
||||||
print "Would have deleted build: %s" % nvr
|
|
||||||
else:
|
|
||||||
print "Deleting untagged build: %s" % nvr
|
|
||||||
try:
|
|
||||||
session.deleteBuild(build_id, strict=False)
|
|
||||||
except (xmlrpclib.Fault, koji.GenericError), e:
|
|
||||||
print "Warning: deletion failed: %s" % e
|
|
||||||
#server issue
|
|
||||||
pass
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
|
|
||||||
options, args = get_options()
|
|
||||||
|
|
||||||
session_opts = koji.grab_session_options(options)
|
|
||||||
session = koji.ClientSession(options.server, session_opts)
|
|
||||||
rv = 0
|
|
||||||
try:
|
|
||||||
if not options.skip_main:
|
|
||||||
rv = main(args)
|
|
||||||
if not rv:
|
|
||||||
rv = 0
|
|
||||||
except KeyboardInterrupt:
|
|
||||||
pass
|
|
||||||
except SystemExit:
|
|
||||||
rv = 1
|
|
||||||
#except:
|
|
||||||
# if options.debug:
|
|
||||||
# raise
|
|
||||||
# else:
|
|
||||||
# exctype, value = sys.exc_info()[:2]
|
|
||||||
# rv = 1
|
|
||||||
# print "%s: %s" % (exctype, value)
|
|
||||||
try:
|
|
||||||
session.logout()
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
if not options.skip_main:
|
|
||||||
sys.exit(rv)
|
|
||||||
|
|
@ -20,15 +20,6 @@
|
||||||
- packages
|
- packages
|
||||||
- koji_hub
|
- koji_hub
|
||||||
|
|
||||||
# Fix kerberos support for koji-gc
|
|
||||||
# https://pagure.io/koji/pull-request/246
|
|
||||||
# https://pagure.io/koji/pull-request/248
|
|
||||||
- name: hotfix - koji-gc kerberos support
|
|
||||||
copy: src=koji-gc.py-hotfix dest=/usr/sbin/koji-gc mode=0755
|
|
||||||
tags:
|
|
||||||
- koji_hub
|
|
||||||
- hotfix
|
|
||||||
|
|
||||||
- name: make koji pki directory
|
- name: make koji pki directory
|
||||||
file: state=directory path=/etc/pki/koji/ owner=root group=root
|
file: state=directory path=/etc/pki/koji/ owner=root group=root
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue