From bdea4df88d9f088c5abb2160c4453e969bdbe614 Mon Sep 17 00:00:00 2001 From: Mike McGrath Date: Wed, 27 Feb 2008 12:18:54 -0600 Subject: [PATCH] Added epel-repoclosure --- scripts/epel-repoclosure/PackageOwners.py | 277 ++++++++++++++ scripts/epel-repoclosure/demo.sh | 10 + .../epel-repoclosure/rc-epel5-20080113.txt | 166 +++++++++ scripts/epel-repoclosure/rc-modified | 282 ++++++++++++++ scripts/epel-repoclosure/rc-report-epel.cfg | 9 + scripts/epel-repoclosure/rc-report.py | 351 ++++++++++++++++++ scripts/epel-repoclosure/yum.epel.conf | 97 +++++ 7 files changed, 1192 insertions(+) create mode 100644 scripts/epel-repoclosure/PackageOwners.py create mode 100755 scripts/epel-repoclosure/demo.sh create mode 100644 scripts/epel-repoclosure/rc-epel5-20080113.txt create mode 100755 scripts/epel-repoclosure/rc-modified create mode 100644 scripts/epel-repoclosure/rc-report-epel.cfg create mode 100755 scripts/epel-repoclosure/rc-report.py create mode 100644 scripts/epel-repoclosure/yum.epel.conf diff --git a/scripts/epel-repoclosure/PackageOwners.py b/scripts/epel-repoclosure/PackageOwners.py new file mode 100644 index 0000000..8a8723e --- /dev/null +++ b/scripts/epel-repoclosure/PackageOwners.py @@ -0,0 +1,277 @@ +#!/usr/bin/python +# -*- mode: Python; indent-tabs-mode: nil; -*- +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import commands +import errno +import os, sys, time +import shutil +import tempfile +from urllib import FancyURLopener + + +class AccountsURLopener(FancyURLopener): + """Subclass of urllib.FancyURLopener to allow passing http basic auth info""" + def __init__(self, username, password): + FancyURLopener.__init__(self) + self.username = username + self.password = password + + def prompt_user_passwd(self, host, realm): + return (self.username, self.password) + + +class PackageOwners: + """interface to Fedora package owners list (and Fedora Extras owners/owners.list file)""" + + def __init__(self): + self.dict = {} + self.how = 'unknown' + + + def FromURL(self, retries=3, retrysecs=300, url='https://admin.fedoraproject.org/pkgdb/acls/bugzilla?tg_format=plain', + pkgdb=True, repoid='Fedora', username=None, password=None): + # old url='http://cvs.fedora.redhat.com/viewcvs/*checkout*/owners/owners.list?root=extras' + if pkgdb: + self.how = 'pkgdb' + else: + self.how = 'url' + self.url = url + self.repoid = repoid + self.retries = retries + self.retrysecs = retrysecs + self.username = username + self.password = password + return self._refresh() + + + def FromCVS(self, retries=3, retrysecs=300, command='LC_ALL=C CVS_RSH=ssh cvs -f -d :pserver:anonymous@cvs.fedora.redhat.com:/cvs/extras co owners', workdir='',repoid='Fedora'): + self.how = 'cvs' + self.command = command + self.repoid = repoid + self.retries = retries + self.retrysecs = retrysecs + self.workdir = workdir + self.ownersfile = os.path.join('owners', 'owners.list') + self.cwdstack = [] + return self._refresh() + + + def __getitem__(self,rpmname): + """return e-mail address from initialowner field""" + return self.GetOwner(rpmname) + + + def GetOwner(self,rpmname): + """return e-mail address from initialowner field""" + try: + r = self.dict[rpmname]['mailto'] + except KeyError: + r = '' + return r + + + def GetOwners(self,rpmname): + """return list of e-mail addresses from initialowner+initialcclist fields""" + r = self.GetCoOwnerList(rpmname) + r2 = self.GetOwner(rpmname) + if len(r2): + r.append(r2) + return r + + + def GetCoOwnerList(self,rpmname): + """return list of e-mail addresses from initialcclist field""" + try: + r = self.dict[rpmname]['cc'] + except KeyError: + r = [] + return r + + + def _enterworkdir(self): + self.cwdstack.append( os.getcwd() ) + if self.workdir != '': + os.chdir(self.workdir) + + + def _leaveworkdir(self): + if len(self.cwdstack): + os.chdir( self.cwdstack.pop() ) + + + def _refresh(self): + self.dict = {} # map package name to email address, dict[name] + return self._download() + + + def _parse(self,ownerslist): + for line in ownerslist: + if line.startswith('#') or line.isspace(): + continue + try: + (repo,pkgname,summary,emails,qacontact,cc) = line.rstrip().split('|') + # This is commented, because we don't need the summary. + #summary.replace(r'\u007c','|').replace('\u005c','\\') + + # The PkgDb includes repo's other than Fedora (Fedora EPEL, + # Fedora OLPC, and Red Hat Linux, for example). Skip them. + if repo != self.repoid: + continue + def fixaddr(a): + # Old Fedora CVS owners.list contains e-mail addresses. + # PkgDb plain output contains usernames only. + if not self.how == 'pkgdb': + return a + if not self.usermap.has_key(a): + return a + return self.usermap[a] + + addrs = [] + mailto = '' # primary pkg owner + if len(emails): + if emails.find(',')>=0: + (addrs) = emails.split(',') + mailto = addrs[0] + addrs = addrs[1:] + else: + mailto = emails + mailto = fixaddr(mailto) + + ccaddrs = [] + if len(cc): + (ccaddrs) = cc.split(',') + addrs += ccaddrs + addrs = map(lambda a: fixaddr(a), addrs) + + self.dict[pkgname] = { + 'mailto' : mailto, + 'cc' : addrs + } + except: + print 'ERROR: owners.list is broken' + print line + + + def _downloadfromcvs(self): + self._enterworkdir() + # Dumb caching. Check that file exists and is "quite recent". + cached = False + try: + fstats = os.stat(self.ownersfile) + if ( fstats.st_size and + ((time.time() - fstats.st_ctime) < 3600*2) ): + cached = True + except OSError: + pass + + if not cached: + # Remove 'owners' directory contents, if it exists. + for root, dirs, files in os.walk( 'owners', topdown=False ): + for fname in files: + os.remove(os.path.join( root, fname )) + for dname in dirs: + os.rmdir(os.path.join( root, dname )) + # Retry CVS checkout a few times. + for count in range(self.retries): + (rc, rv) = commands.getstatusoutput(self.command) + if not rc: + break + print rv + time.sleep(self.retrysecs) + if rc: + # TODO: customise behaviour on error conditions + self._leaveworkdir() + return False + + try: + f = file( self.ownersfile ) + except IOError, (err, strerr): + print 'ERROR: %s' % strerr + # TODO: customise behaviour on error conditions + self._leaveworkdir() + return err + ownerslist = f.readlines() + f.close() + self._parse(ownerslist) + self._leaveworkdir() + return True + + + def _getlinesfromurl(self,url): + err = 0 + strerr = '' + # Retry URL download a few times. + for count in range(self.retries): + if count != 0: + time.sleep(self.retrysecs) + try: + opener = AccountsURLopener(self.username, self.password) + f = opener.open(url) + rc = 0 + if 'www-authenticate' in f.headers: + rc = 1 + strerr = 'Authentication is required to access %s' % url + break + except IOError, (_err, _strerr): + rc = 1 + print url + print _strerr + (err,strerr) = (_err,_strerr) + if rc: + raise IOError, (err, strerr) + else: + l = f.readlines() + f.close() + return l + + + def _downloadfromurl(self): + self._parse(self._getlinesfromurl(self.url)) + return True + + + def _downloadfrompkgdb(self): + fasdump = self._getlinesfromurl('https://admin.fedoraproject.org/accounts/dump-group.cgi') + self.usermap = {} + for line in fasdump: + fields = line.split(',') + try: + user = fields[0] + addr = fields[1] + except IndexError: + print line + raise + if (addr.find('@') < 0): # unexpected, no addr + print 'No email in:', line + raise Exception + self.usermap[user] = addr + self._parse(self._getlinesfromurl(self.url)) + return True + + + def _download(self): + if self.how == 'url': + return self._downloadfromurl() + elif self.how == 'pkgdb': + return self._downloadfrompkgdb() + elif self.how == 'cvs': + return self._downloadfromcvs() + else: + self.__init__() + return False + + diff --git a/scripts/epel-repoclosure/demo.sh b/scripts/epel-repoclosure/demo.sh new file mode 100755 index 0000000..573a6ac --- /dev/null +++ b/scripts/epel-repoclosure/demo.sh @@ -0,0 +1,10 @@ +# Run Extras repoclosure against EPEL 5. + +# i386 +#./rc-modified -q -d mdcache -n -c yum.epel.conf -a i686 -r centos-5-i386 -r centos-updates-5-i386 -r fedora-epel-5-i386 -r fedora-epel-testing-5-i386 > rc-epel5-20080113.txt + +# x86_64 : APPEND! +#./rc-modified -q -d mdcache -n -c yum.epel.conf -a x86_64 -r centos-5-x86_64 -r centos-updates-5-x86_64 -r fedora-epel-5-x86_64 -r fedora-epel-testing-5-x86_64 >> rc-epel5-20080113.txt + +# Show summary which can be sent to mailing-list. +./rc-report.py rc-epel5-20080113.txt -k epel -c rc-report-epel.cfg -w testing diff --git a/scripts/epel-repoclosure/rc-epel5-20080113.txt b/scripts/epel-repoclosure/rc-epel5-20080113.txt new file mode 100644 index 0000000..d05c186 --- /dev/null +++ b/scripts/epel-repoclosure/rc-epel5-20080113.txt @@ -0,0 +1,166 @@ +source rpm: R-2.6.1-1.el5.src.rpm +package: R - 2.6.1-1.el5.i386 from fedora-epel-needsign-5-i386 + unresolved deps: + perl(File::Copy::Recursive) + +source rpm: bodhi-0.4.4-1.el5.src.rpm +package: bodhi-server - 0.4.4-1.el5.noarch from fedora-epel-testing-5-i386 + unresolved deps: + yum-utils >= 0:1.1.7 + mash + +source rpm: fedora-packager-0.1.1-1.el5.src.rpm +package: fedora-packager - 0.1.1-1.el5.noarch from fedora-epel-testing-5-i386 + unresolved deps: + plague-client + +source rpm: hspell-1.0-7.el5.src.rpm +package: hunspell-he - 1.0-7.el5.i386 from fedora-epel-testing-5-i386 + unresolved deps: + hunspell + +source rpm: koji-1.2.3-1.el5.src.rpm +package: koji-builder - 1.2.3-1.el5.noarch from fedora-epel-testing-5-i386 + unresolved deps: + createrepo >= 0:0.4.11 + +source rpm: moodle-1.8.2-1.el5.src.rpm +package: moodle - 1.8.2-1.el5.noarch from fedora-epel-testing-5-i386 + unresolved deps: + mimetex + +source rpm: bzr-gtk-0.93.0-2.el5.src.rpm +package: nautilus-bzr - 0.93.0-2.el5.i386 from fedora-epel-testing-5-i386 + unresolved deps: + nautilus-python >= 0:0.4.3-4 + +source rpm: nautilus-sendto-0.7-5.fc6.src.rpm +package: nautilus-sendto - 0.7-5.fc6.i386 from centos-5-i386 + unresolved deps: + libgaim.so.0 + +source rpm: perl-Test-Base-0.53-1.el5.src.rpm +package: perl-Test-Base - 0.53-1.el5.noarch from fedora-epel-testing-5-i386 + unresolved deps: + perl(Module::Install::Base) + +source rpm: perl-libwhisker2-2.4-3.el5.src.rpm +package: perl-libwhisker2 - 2.4-3.el5.noarch from fedora-epel-testing-5-i386 + unresolved deps: + perl(MD5) + +source rpm: python-Coherence-0.2.1-3.el5.src.rpm +package: python-Coherence - 0.2.1-3.el5.noarch from fedora-epel-testing-5-i386 + unresolved deps: + python-twisted-core + python-nevow + python-twisted-web + +source rpm: revisor-2.0.5-15.el5.src.rpm +package: revisor-jigdo - 2.0.5-15.el5.noarch from fedora-epel-testing-5-i386 + unresolved deps: + jigdo + +source rpm: snake-0.9-0.5git.el5.src.rpm +package: snake-server - 0.9-0.5git.el5.noarch from fedora-epel-testing-5-i386 + unresolved deps: + pykickstart >= 0:1.1 + +source rpm: translate-toolkit-0.10.1-1.el5.src.rpm +package: translate-toolkit - 0.10.1-1.el5.noarch from fedora-epel-testing-5-i386 + unresolved deps: + python-enchant + +source rpm: R-2.6.1-1.el5.src.rpm +package: R - 2.6.1-1.el5.i386 from fedora-epel-needsign-5-x86_64 + unresolved deps: + perl(File::Copy::Recursive) + +source rpm: R-2.6.1-1.el5.src.rpm +package: R - 2.6.1-1.el5.x86_64 from fedora-epel-needsign-5-x86_64 + unresolved deps: + perl(File::Copy::Recursive) + +source rpm: bodhi-0.4.4-1.el5.src.rpm +package: bodhi-server - 0.4.4-1.el5.noarch from fedora-epel-testing-5-x86_64 + unresolved deps: + yum-utils >= 0:1.1.7 + mash + +source rpm: fedora-packager-0.1.1-1.el5.src.rpm +package: fedora-packager - 0.1.1-1.el5.noarch from fedora-epel-testing-5-x86_64 + unresolved deps: + plague-client + +source rpm: hspell-1.0-7.el5.src.rpm +package: hunspell-he - 1.0-7.el5.x86_64 from fedora-epel-testing-5-x86_64 + unresolved deps: + hunspell + +source rpm: koji-1.2.3-1.el5.src.rpm +package: koji-builder - 1.2.3-1.el5.noarch from fedora-epel-testing-5-x86_64 + unresolved deps: + createrepo >= 0:0.4.11 + +source rpm: openib-1.2-6.el5.src.rpm +package: libcxgb3-devel - 1.0-6.el5.i386 from centos-5-x86_64 + unresolved deps: + libcxgb3 = 0:1.0-6.el5 + +source rpm: moodle-1.8.2-1.el5.src.rpm +package: moodle - 1.8.2-1.el5.noarch from fedora-epel-testing-5-x86_64 + unresolved deps: + mimetex + +source rpm: mozldap-6.0.4-1.el5.src.rpm +package: mozldap-devel - 6.0.4-1.el5.i386 from centos-5-x86_64 + unresolved deps: + mozldap = 0:6.0.4-1.el5 + +source rpm: nagios-2.9-1.el5.src.rpm +package: nagios-devel - 2.9-1.el5.i386 from fedora-epel-5-x86_64 + unresolved deps: + nagios = 0:2.9-1.el5 + +source rpm: bzr-gtk-0.93.0-2.el5.src.rpm +package: nautilus-bzr - 0.93.0-2.el5.x86_64 from fedora-epel-testing-5-x86_64 + unresolved deps: + nautilus-python >= 0:0.4.3-4 + +source rpm: nautilus-sendto-0.7-5.fc6.src.rpm +package: nautilus-sendto - 0.7-5.fc6.x86_64 from centos-5-x86_64 + unresolved deps: + libgaim.so.0()(64bit) + +source rpm: perl-Test-Base-0.53-1.el5.src.rpm +package: perl-Test-Base - 0.53-1.el5.noarch from fedora-epel-testing-5-x86_64 + unresolved deps: + perl(Module::Install::Base) + +source rpm: perl-libwhisker2-2.4-3.el5.src.rpm +package: perl-libwhisker2 - 2.4-3.el5.noarch from fedora-epel-testing-5-x86_64 + unresolved deps: + perl(MD5) + +source rpm: python-Coherence-0.2.1-3.el5.src.rpm +package: python-Coherence - 0.2.1-3.el5.noarch from fedora-epel-testing-5-x86_64 + unresolved deps: + python-twisted-core + python-nevow + python-twisted-web + +source rpm: revisor-2.0.5-15.el5.src.rpm +package: revisor-jigdo - 2.0.5-15.el5.noarch from fedora-epel-testing-5-x86_64 + unresolved deps: + jigdo + +source rpm: snake-0.9-0.5git.el5.src.rpm +package: snake-server - 0.9-0.5git.el5.noarch from fedora-epel-testing-5-x86_64 + unresolved deps: + pykickstart >= 0:1.1 + +source rpm: translate-toolkit-0.10.1-1.el5.src.rpm +package: translate-toolkit - 0.10.1-1.el5.noarch from fedora-epel-testing-5-x86_64 + unresolved deps: + python-enchant + diff --git a/scripts/epel-repoclosure/rc-modified b/scripts/epel-repoclosure/rc-modified new file mode 100755 index 0000000..17e8c3a --- /dev/null +++ b/scripts/epel-repoclosure/rc-modified @@ -0,0 +1,282 @@ +#!/usr/bin/python -t +# -*- mode: Python; indent-tabs-mode: nil; -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# seth vidal 2005 (c) etc etc +# mschwendt: modified for Fedora Extras buildsys + +#Read in the metadata of a series of repositories and check all the +# dependencies in all packages for resolution. Print out the list of +# packages with unresolved dependencies + +import sys +import os + +# For patched "yum" and "rpmUtils" (post 2.6.1 checkForObsolete support). +# Comment this to use system yum. +#sys.path.insert(0,'/srv/extras-push/work/buildsys-utils/pushscript') + +import yum +import yum.Errors +from yum.misc import getCacheDir +from optparse import OptionParser +import rpmUtils.arch +from yum.constants import * +if yum.__version__ < '3.0': # TODO: check + from repomd.packageSack import ListPackageSack +else: + from yum.packageSack import ListPackageSack + + +def parseArgs(): + usage = "usage: %s [-c ] [-a ] [-r ] [-r ]" % sys.argv[0] + parser = OptionParser(usage=usage) + parser.add_option("-c", "--config", default='/etc/yum.conf', + help='config file to use (defaults to /etc/yum.conf)') + parser.add_option("-a", "--arch", default=None, + help='check as if running the specified arch (default: current arch)') + parser.add_option("-r", "--repoid", default=[], action='append', + help="specify repo ids to query, can be specified multiple times (default is all enabled)") + parser.add_option("-t", "--tempcache", default=False, action="store_true", + help="Use a temp dir for storing/accessing yum-cache") + parser.add_option("-d", "--cachedir", default='', + help="specify a custom directory for storing/accessing yum-cache") + parser.add_option("-q", "--quiet", default=0, action="store_true", + help="quiet (no output to stderr)") + parser.add_option("-n", "--newest", default=0, action="store_true", + help="check only the newest packages in the repos") + (opts, args) = parser.parse_args() + return (opts, args) + +class RepoClosure(yum.YumBase): + def __init__(self, arch = None, config = "/etc/yum.conf"): + yum.YumBase.__init__(self) + + self.arch = arch + if yum.__version__ < '3.0': # TODO: check + self.doConfigSetup(fn = config) + else: + self.doConfigSetup(fn = config, init_plugins = False) + if hasattr(self.repos, 'sqlite'): + self.repos.sqlite = False + self.repos._selectSackType() + + def evrTupletoVer(self,tuple): + """convert and evr tuple to a version string, return None if nothing + to convert""" + + e, v,r = tuple + + if v is None: + return None + + val = '' + if e is not None: + val = '%s:%s' % (e, v) + + if r is not None: + val = '%s-%s' % (val, r) + + return val + + def readMetadata(self): + self.doRepoSetup() + self.doSackSetup(rpmUtils.arch.getArchList(self.arch)) + for repo in self.repos.listEnabled(): + try: # TODO: when exactly did this change to "mdtype"? + self.repos.populateSack(which=[repo.id], mdtype='filelists') + except TypeError: + self.repos.populateSack(which=[repo.id], with='filelists') + + def getBrokenDeps(self, newest=False): + unresolved = {} + resolved = {} + newpkgtuplist = [] + if newest: + if yum.__version__ >= '2.9': # TODO: check + pkgs = self.pkgSack.returnNewestByName() + else: + pkgs = [] + for l in self.pkgSack.returnNewestByName(): + pkgs.extend(l) + newestpkgtuplist = ListPackageSack(pkgs).simplePkgList() + + pkgs = self.pkgSack.returnNewestByNameArch() + else: + pkgs = self.pkgSack + self.numpkgs = len(pkgs) + + mypkgSack = ListPackageSack(pkgs) + pkgtuplist = mypkgSack.simplePkgList() + + # Support new checkForObsolete code in Yum (#190116) + # _if available_ + # so we don't examine old _obsolete_ sub-packages. + import rpmUtils.updates + self.up = rpmUtils.updates.Updates([],pkgtuplist) + self.up.rawobsoletes = mypkgSack.returnObsoletes() + + haveCheckForObsolete = hasattr(rpmUtils.updates.Updates,'checkForObsolete') + if not haveCheckForObsolete: + print 'WARNING: rpmUtils.updates.checkForObsolete missing!' + + for pkg in pkgs: + thispkgobsdict = {} + if haveCheckForObsolete: + try: + thispkgobsdict = self.up.checkForObsolete([pkg.pkgtup]) + if thispkgobsdict.has_key(pkg.pkgtup): + continue + except AttributeError: + pass + + def isnotnewest(pkg): + # Handle noarch<->arch switches in package updates, so any + # old nevra returned by returnNewestByNameArch() are skipped. + # TODO: There must be a more elegant/convenient way. + if (pkg.pkgtup[1] == 'noarch'): + if (pkg.pkgtup not in newestpkgtuplist): + return True + else: + for p in self.pkgSack.returnNewestByName(pkg.pkgtup[0]): + if (p.pkgtup[1] == 'noarch') and (p.pkgtup in newestpkgtuplist): + return True + return False + + for (req, flags, (reqe, reqv, reqr)) in pkg.returnPrco('requires'): + if req.startswith('rpmlib'): continue # ignore rpmlib deps + + ver = self.evrTupletoVer((reqe, reqv, reqr)) + if resolved.has_key((req,flags,ver)): + continue + try: + resolve_sack = self.whatProvides(req, flags, ver) + except yum.Errors.RepoError, e: + pass + + if len(resolve_sack) < 1: + if newest and isnotnewest(pkg): + break + if not unresolved.has_key(pkg): + unresolved[pkg] = [] + unresolved[pkg].append((req, flags, ver)) + continue + + kernelprovides = True # make a false assumption + # If all providers are "kernel*" packages, we allow old ones. + for (pn,pa,pe,pv,pr) in resolve_sack.simplePkgList(): + kernelprovides &= pn.startswith('kernel') + + if newest and not kernelprovides and not req.startswith('kernel'): # we allow old kernels + resolved_by_newest = False + for po in resolve_sack:# look through and make sure all our answers are newest-only + + # 2nd stage handling of obsoletes. Only keep providers, + # which are not obsolete. If no provider is left, the + # dep is unresolved. + thispkgobsdict = {} + if haveCheckForObsolete: + try: + thispkgobsdict = self.up.checkForObsolete([po.pkgtup]) + if thispkgobsdict.has_key(po.pkgtup): + continue + except AttributeError: + pass + + if po.pkgtup in pkgtuplist: + resolved_by_newest = True + break + + if resolved_by_newest: + resolved[(req,flags,ver)] = 1 + else: + if newest and isnotnewest(pkg): + break + if not unresolved.has_key(pkg): + unresolved[pkg] = [] + unresolved[pkg].append((req, flags, ver)) + + return unresolved + + + def log(self, value, msg): + pass + +def main(): + (opts, cruft) = parseArgs() + my = RepoClosure(arch = opts.arch, config = opts.config) + + if opts.repoid: + for repo in my.repos.repos.values(): + if repo.id not in opts.repoid: + repo.disable() + else: + repo.enable() + + if os.geteuid() != 0 or opts.tempcache or opts.cachedir != '': + if opts.cachedir != '': + cachedir = opts.cachedir + else: + cachedir = getCacheDir() + if cachedir is None: + print "Error: Could not make cachedir, exiting" + sys.exit(50) + + my.repos.setCacheDir(cachedir) + + if not opts.quiet: + print 'Reading in repository metadata - please wait....' + + try: + my.readMetadata() + except yum.Errors.RepoError, e: + print e + sys.exit(1) + + if not opts.quiet: + print 'Checking Dependencies' + + baddeps = my.getBrokenDeps(opts.newest) + num = my.numpkgs + + repos = my.repos.listEnabled() + + if not opts.quiet: + print 'Repos looked at: %s' % len(repos) + for repo in repos: + print ' %s' % repo + print 'Num Packages in Repos: %s' % num + + pkgs = baddeps.keys() + def sortbyname(a,b): + return cmp(a.__str__(),b.__str__()) + pkgs.sort(sortbyname) + for pkg in pkgs: + srcrpm = pkg.returnSimple('sourcerpm') + print 'source rpm: %s\npackage: %s from %s\n unresolved deps: ' % (srcrpm, pkg, pkg.repoid) + for (n, f, v) in baddeps[pkg]: + req = '%s' % n + if f: + flag = LETTERFLAGS[f] + req = '%s %s'% (req, flag) + if v: + req = '%s %s' % (req, v) + + print ' %s' % req + print + +if __name__ == "__main__": + main() + diff --git a/scripts/epel-repoclosure/rc-report-epel.cfg b/scripts/epel-repoclosure/rc-report-epel.cfg new file mode 100644 index 0000000..220153d --- /dev/null +++ b/scripts/epel-repoclosure/rc-report-epel.cfg @@ -0,0 +1,9 @@ +[FAS] +project = Fedora EPEL +#user = foo +#passwd = secret + +[Mail] +from = Fedora Extras repoclosure +replyto = epel-devel-list@redhat.com +subject = Broken dependencies in EPEL diff --git a/scripts/epel-repoclosure/rc-report.py b/scripts/epel-repoclosure/rc-report.py new file mode 100755 index 0000000..157c230 --- /dev/null +++ b/scripts/epel-repoclosure/rc-report.py @@ -0,0 +1,351 @@ +#!/usr/bin/python +# -*- mode: Python; indent-tabs-mode: nil; -*- + +import errno, os, sys, stat +import re +import smtplib +import datetime, time +from optparse import OptionParser +import ConfigParser + +from PackageOwners import PackageOwners +#from FakeOwners import FakeOwners as PackageOwners + +FAS = { + 'project' : "Fedora EPEL", + 'user' : "", + 'passwd' : "", + } + +Mail = { + 'server' : "localhost", + 'user' : "", + 'passwd' : "", + 'maxsize' : 39*1024, + 'from' : "root@localhost", + 'replyto' : "root@localhost", + 'subject' : "Broken dependencies in EPEL", +} + +class BrokenDep: + def __init__(self): + self.pkgid = None # 'name - EVR.arch' + self.repoid = None # e.g. 'fedora-core-6-i386' + self.srp_mname = None + self.age = '' # e.g. '(14 days)' + self.owner = '' + self.coowners = [] + # disabled/stripped feature + self.mail = True # whether to notify owner by mail + # disabled/stripped feature + self.new = False + self.report = [] + + def GetRequires(self): + pkgid2 = self.pkgid.replace(' ','') + r = [] + for line in self.report: + if len(line) and not line.isspace() and not line.startswith('package: ') and line.find('unresolved deps:') < 0: + r.append( ' '+pkgid2+' requires '+line.lstrip() ) + return '\n'.join(r) + + +def whiteListed(b): # Just a hook, not a generic white-list feature. + # These two in Fedora 7 Everything most likely won't be fixed. + if b.pkgid.startswith('kmod-em8300') and b.repoid.startswith('fedora-7'): + return True + elif b.pkgid.startswith('kmod-sysprof') and b.repoid.startswith('fedora-7'): + return True + elif b.pkgid.startswith('kmod'): # gah ;) temporarily catch them all + return True + else: + return False + + +def makeOwners(brokendeps): + owners = PackageOwners() + try: + #if not owners.FromURL(): + if not owners.FromURL(repoid=FAS['project'],username=FAS['user'],password=FAS['passwd']): + raise IOError('ERROR: Could not retrieve package owner data.') + except IOError, e: + print e + sys.exit(1) + for b in brokendeps: + toaddr = owners.GetOwner(b.srpm_name) + if toaddr == '': + toaddr = 'UNKNOWN OWNER' + e = 'ERROR: "%s" not in owners.list!\n\n' % b.srpm_name + if e not in errcache: + errcache.append(e) + b.owner = toaddr + b.coowners = owners.GetCoOwnerList(b.srpm_name) + + +def mail(smtp, fromaddr, toaddrs, replytoaddr, subject, body): + from email.Header import Header + from email.MIMEText import MIMEText + msg = MIMEText( body, 'plain' ) + from email.Utils import make_msgid + msg['Message-Id'] = make_msgid() + msg['Subject'] = Header(subject) + msg['From'] = Header(fromaddr) + from email.Utils import formatdate + msg['Date'] = formatdate() + if len(replytoaddr): + msg['ReplyTo'] = Header(replytoaddr) + + if isinstance(toaddrs, basestring): + toaddrs = [toaddrs] + to = '' + for t in toaddrs: + if len(to): + to += ', ' + to += t + msg['To'] = Header(to) + + try: + r = smtp.sendmail( fromaddr, toaddrs, msg.as_string(False) ) + for (name, errormsg) in r.iteritems(): + print name, ':', errormsg + except smtplib.SMTPRecipientsRefused, obj: + print 'ERROR: SMTPRecipientsRefused' + for (addr, errormsg) in obj.recipients.iteritems(): + print addr, ':', errormsg + except smtplib.SMTPException: + print 'ERROR: SMTPException' + + +def mailsplit(smtp, fromaddr, toaddrs, replytoaddr, subject, body): + # Split mail body at line positions to keep it below maxmailsize. + parts = 0 + start = 0 + end = len(body) + slices = [] + while ( start < end ): + if ( (end-start) > Mail['maxsize'] ): + nextstart = body.rfind( '\n', start, start+Mail['maxsize'] ) + if ( nextstart<0 or nextstart==start ): + print 'ERROR: cannot split mail body cleanly' + nextstart = end + else: + nextstart = end + slices.append( (start, nextstart) ) + start = nextstart + parts += 1 + + curpart = 1 + for (start,end) in slices: + if (parts>1): + subjectmodified = ( '(%d/%d) %s' % (curpart, parts, subject) ) + time.sleep(1) + else: + subjectmodified = subject + slicedbody = body[start:end] + mail(smtp,fromaddr,toaddrs,replytoaddr,subjectmodified,slicedbody) + curpart += 1 + + +def loadConfigFile(filename): + if not filename: + return + + config = ConfigParser.ConfigParser() + try: + config.readfp(open(filename)) + except IOError, (e, errstr): + print filename, ':', errstr + sys.exit(e) + + try: + if config.has_section('FAS'): + for v in ['project','user','passwd']: + if config.has_option('FAS',v): + FAS[v] = config.get('FAS',v) + if config.has_section('Mail'): + for v in ['server','user','passwd','from','replyto','subject']: + if config.has_option('Mail',v): + Mail[v] = config.get('Mail',v) + if config.has_option('Mail','maxsize'): + Mail['maxsize'] = config.getint('Mail','maxsize') + + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError), e: + print 'Configuration file error:', e + + +### main + +usage = "Usage: %s " % sys.argv[0] +parser = OptionParser(usage=usage) +parser.add_option("-c", "--config", default=None, + help="config file to use") +parser.add_option("-k", "--keyword", default=[], action='append', + help="a keyword to look for in repoids") +parser.add_option("-m", "--mail", default=[], action='append', + help="what mail to send (owner, summary)") +parser.add_option("-w", "--warn", default=[], action='append', + help="repository warnings to include (needsign, testing)") +parser.add_option("", "--noowners", default=False, action="store_true", + help="don't fetch package owner data from FAS") +(opts, args) = parser.parse_args() + +loadConfigFile(opts.config) + +domail = len(opts.mail)>0 +brokendeps = [] # list of BrokenDeps +errcache = [] # error messages to be included in the summary mail + +if not len(args): + print usage + sys.exit(errno.EINVAL) +# Parse extras-repoclosure output files and fill brokendeps array. +while len(args): + logfilename = args[0] + del args[0] + + f = file( logfilename ) + pkgre = re.compile('(?P.*)-[^-]+-[^-]+$') + inbody = False + srcrpm = '' + for line in f: + if line.startswith('source rpm: '): + w = line.rstrip().split(' ') + srcrpm = w[2] + res = pkgre.search( srcrpm ) # try to get src.rpm "name" + if not res: # only true for invalid input + inbody = False + else: + srpm_name = res.group('name') + inbody = True + continue + + elif inbody and line.startswith('package: '): + w = line.rstrip().split(' ') + repoid = w[5] + b = BrokenDep() + b.pkgid = w[1]+' - '+w[3] # name - EVR.arch + b.repoid = repoid + b.srpm_name = srpm_name + brokendeps.append(b) + + if inbody: + # Copy report per broken package. + b.report.append( line.rstrip() ) + + +def bdSortByOwnerAndName(a,b): + return cmp(a.owner+a.pkgid,b.owner+b.pkgid) + +def bdSortByRepoAndName(a,b): + return cmp(a.repoid+a.pkgid,b.repoid+b.pkgid) + + +# Filter out unwanted repoids. +for b in list(brokendeps): + for needle in opts.keyword: + if b.repoid.find( needle ) >= 0: # wanted? + break + else: + brokendeps.remove(b) + +# Filter out entries from whitelist. +for b in list(brokendeps): + if whiteListed(b): + brokendeps.remove(b) + +# Fill in package owners. +if not opts.noowners: + makeOwners(brokendeps) + +# Build full mail report per owner. Use a flag for new breakage. +reports = {} # map of lists [new,body] - a flag and the full report for a package owner +if not opts.noowners: + brokendeps.sort(bdSortByOwnerAndName) + for b in brokendeps: + if b.new: + print 'NEW breakage: %s in %s' % (b.pkgid, b.repoid) + if b.mail: + r = '\n'.join(b.report)+'\n' + reports.setdefault(b.owner,[b.new,'']) + reports[b.owner][1] += r + # Also build mails for co-owners. + for toaddr in b.coowners: + reports.setdefault(toaddr,[None,'']) + reports[toaddr][1] += r + + +sep = '='*70+'\n' +summail = '' # main summary mail text +reportssummary = '' # any NEW stuff for the summary + +def giveNeedsignMsg(): + if 'needsign' in opts.warn: + return sep+"The results in this summary consider unreleased updates in the\nbuild-system's needsign-queue!\n"+sep+'\n' + else: + return '' + +def giveTestingMsg(): + if 'testing' in opts.warn: + return sep+"The results in this summary consider Test Updates!\n"+sep+'\n' + else: + return '' + +# Create summary mail text. +reportssummary += giveNeedsignMsg() +reportssummary += giveTestingMsg() +summail += reportssummary + +if not opts.noowners and len(brokendeps): + summail += ('Summary of broken packages (by owner):\n') + brokendeps.sort(bdSortByOwnerAndName) + o = None + for b in brokendeps: + if o != b.owner: + o = b.owner + seenbefore = [] + summail += '\n '+b.owner.replace('@',' AT ')+'\n' + if b.pkgid not in seenbefore: + summail += ' '+b.pkgid+' '+b.age+'\n' + seenbefore.append(b.pkgid) + +# Broken deps sorted by repository id. +brokendeps.sort(bdSortByRepoAndName) +r = None +for b in brokendeps: + if r != b.repoid: + r = b.repoid + summail += '\n\n'+sep+('Broken packages in %s:\n\n' % b.repoid) + summail += b.GetRequires()+'\n' + +# Mail init. +if domail: + srv = smtplib.SMTP( Mail['server'] ) + if ( len(Mail['user']) and len(Mail['passwd']) ): + try: + srv.login( Mail['user'], Mail['passwd'] ) + except smtplib.SMTPException: + print 'ERROR: mailserver login failed' + sys.exit(-1) + +# Mail reports to owners. +for toaddr,(new,body) in reports.iteritems(): + # Send mail to every package owner with broken package dependencies. + mailtext = 'Your following packages in the repository suffer from broken dependencies:\n\n' + mailtext += giveNeedsignMsg() + mailtext += giveTestingMsg() + mailtext += body + if domail and ('owners' in opts.mail) and toaddr!='UNKNOWN OWNER': + subject = Mail['subject'] + ' - %s' % datetime.date.today() + mail( srv, Mail['from'], toaddr, Mail['replyto'], subject, mailtext ) + +# Mail summary to mailing-list. +if domail and ('summary' in opts.mail): + subject = Mail['subject'] + ' - %s' % datetime.date.today() + toaddr = Mail['replyto'] + mailsplit( srv, Mail['from'], toaddr, '', subject, summail ) + +if domail: + srv.quit() + +if len(summail): + print summail diff --git a/scripts/epel-repoclosure/yum.epel.conf b/scripts/epel-repoclosure/yum.epel.conf new file mode 100644 index 0000000..6742340 --- /dev/null +++ b/scripts/epel-repoclosure/yum.epel.conf @@ -0,0 +1,97 @@ +[main] +cachedir=/tmp/mdcache +debuglevel=2 +logfile=/var/log/yum.log +pkgpolicy=newest +distroverpkg=fedora-release +reposdir=/dev/null +exactarch=1 +obsoletes=1 +retries=20 + +### EL5 ### + +[centos-5-i386] +name=CentOS 5 - i386 +baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/5/os/i386/ +enabled=0 + +[centos-updates-5-i386] +name=CentOS Updates 5 - i386 +baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/5/updates/i386/ +enabled=0 + +[centos-5-x86_64] +name=CentOS 5 - x86_64 +baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/5/os/x86_64/ +enabled=0 + +[centos-updates-5-x86_64] +name=CentOS Updates 5 - x86_64 +baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/5/updates/x86_64/ +enabled=0 + + +[fedora-epel-5-i386] +name=Fedora EPEL 5 - i386 +baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/5/i386/ +enabled=0 + +[fedora-epel-testing-5-i386] +name=Fedora EPEL Test Updates 5 - i386 +baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/testing/5/i386/ +enabled=0 + +[fedora-epel-5-x86_64] +name=Fedora EPEL 5 - x86_64 +baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/5/x86_64/ +enabled=0 + +[fedora-epel-testing-5-x86_64] +name=Fedora EPEL Test Updates 5 - x86_64 +baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/testing/5/x86_64/ +enabled=0 + +### EL4 ### + +[centos-4-i386] +name=CentOS 4 - i386 +baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/4/os/i386/ +enabled=0 + +[centos-updates-4-i386] +name=CentOS Updates 4 - i386 +baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/4/updates/i386/ +enabled=0 + +[centos-4-x86_64] +name=CentOS 4 - x86_64 +baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/4/os/x86_64/ +enabled=0 + +[centos-updates-4-x86_64] +name=CentOS Updates 4 - x86_64 +baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/4/updates/x86_64/ +enabled=0 + + +[fedora-epel-4-i386] +name=Fedora EPEL 4 - i386 +baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/4/i386/ +enabled=0 + +[fedora-epel-testing-4-i386] +name=Fedora EPEL Test Updates 4 - i386 +baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/testing/4/i386/ +enabled=0 + +[fedora-epel-4-x86_64] +name=Fedora EPEL 4 - x86_64 +baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/4/x86_64/ +enabled=0 + +[fedora-epel-testing-4-x86_64] +name=Fedora EPEL Test Updates 4 - x86_64 +baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/testing/4/x86_64/ +enabled=0 +