Added epel-repoclosure

This commit is contained in:
Mike McGrath 2008-02-27 12:18:54 -06:00
parent 9e9ee86ec2
commit bdea4df88d
7 changed files with 1192 additions and 0 deletions

View file

@ -0,0 +1,277 @@
#!/usr/bin/python
# -*- mode: Python; indent-tabs-mode: nil; -*-
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
import commands
import errno
import os, sys, time
import shutil
import tempfile
from urllib import FancyURLopener
class AccountsURLopener(FancyURLopener):
"""Subclass of urllib.FancyURLopener to allow passing http basic auth info"""
def __init__(self, username, password):
FancyURLopener.__init__(self)
self.username = username
self.password = password
def prompt_user_passwd(self, host, realm):
return (self.username, self.password)
class PackageOwners:
"""interface to Fedora package owners list (and Fedora Extras owners/owners.list file)"""
def __init__(self):
self.dict = {}
self.how = 'unknown'
def FromURL(self, retries=3, retrysecs=300, url='https://admin.fedoraproject.org/pkgdb/acls/bugzilla?tg_format=plain',
pkgdb=True, repoid='Fedora', username=None, password=None):
# old url='http://cvs.fedora.redhat.com/viewcvs/*checkout*/owners/owners.list?root=extras'
if pkgdb:
self.how = 'pkgdb'
else:
self.how = 'url'
self.url = url
self.repoid = repoid
self.retries = retries
self.retrysecs = retrysecs
self.username = username
self.password = password
return self._refresh()
def FromCVS(self, retries=3, retrysecs=300, command='LC_ALL=C CVS_RSH=ssh cvs -f -d :pserver:anonymous@cvs.fedora.redhat.com:/cvs/extras co owners', workdir='',repoid='Fedora'):
self.how = 'cvs'
self.command = command
self.repoid = repoid
self.retries = retries
self.retrysecs = retrysecs
self.workdir = workdir
self.ownersfile = os.path.join('owners', 'owners.list')
self.cwdstack = []
return self._refresh()
def __getitem__(self,rpmname):
"""return e-mail address from initialowner field"""
return self.GetOwner(rpmname)
def GetOwner(self,rpmname):
"""return e-mail address from initialowner field"""
try:
r = self.dict[rpmname]['mailto']
except KeyError:
r = ''
return r
def GetOwners(self,rpmname):
"""return list of e-mail addresses from initialowner+initialcclist fields"""
r = self.GetCoOwnerList(rpmname)
r2 = self.GetOwner(rpmname)
if len(r2):
r.append(r2)
return r
def GetCoOwnerList(self,rpmname):
"""return list of e-mail addresses from initialcclist field"""
try:
r = self.dict[rpmname]['cc']
except KeyError:
r = []
return r
def _enterworkdir(self):
self.cwdstack.append( os.getcwd() )
if self.workdir != '':
os.chdir(self.workdir)
def _leaveworkdir(self):
if len(self.cwdstack):
os.chdir( self.cwdstack.pop() )
def _refresh(self):
self.dict = {} # map package name to email address, dict[name]
return self._download()
def _parse(self,ownerslist):
for line in ownerslist:
if line.startswith('#') or line.isspace():
continue
try:
(repo,pkgname,summary,emails,qacontact,cc) = line.rstrip().split('|')
# This is commented, because we don't need the summary.
#summary.replace(r'\u007c','|').replace('\u005c','\\')
# The PkgDb includes repo's other than Fedora (Fedora EPEL,
# Fedora OLPC, and Red Hat Linux, for example). Skip them.
if repo != self.repoid:
continue
def fixaddr(a):
# Old Fedora CVS owners.list contains e-mail addresses.
# PkgDb plain output contains usernames only.
if not self.how == 'pkgdb':
return a
if not self.usermap.has_key(a):
return a
return self.usermap[a]
addrs = []
mailto = '' # primary pkg owner
if len(emails):
if emails.find(',')>=0:
(addrs) = emails.split(',')
mailto = addrs[0]
addrs = addrs[1:]
else:
mailto = emails
mailto = fixaddr(mailto)
ccaddrs = []
if len(cc):
(ccaddrs) = cc.split(',')
addrs += ccaddrs
addrs = map(lambda a: fixaddr(a), addrs)
self.dict[pkgname] = {
'mailto' : mailto,
'cc' : addrs
}
except:
print 'ERROR: owners.list is broken'
print line
def _downloadfromcvs(self):
self._enterworkdir()
# Dumb caching. Check that file exists and is "quite recent".
cached = False
try:
fstats = os.stat(self.ownersfile)
if ( fstats.st_size and
((time.time() - fstats.st_ctime) < 3600*2) ):
cached = True
except OSError:
pass
if not cached:
# Remove 'owners' directory contents, if it exists.
for root, dirs, files in os.walk( 'owners', topdown=False ):
for fname in files:
os.remove(os.path.join( root, fname ))
for dname in dirs:
os.rmdir(os.path.join( root, dname ))
# Retry CVS checkout a few times.
for count in range(self.retries):
(rc, rv) = commands.getstatusoutput(self.command)
if not rc:
break
print rv
time.sleep(self.retrysecs)
if rc:
# TODO: customise behaviour on error conditions
self._leaveworkdir()
return False
try:
f = file( self.ownersfile )
except IOError, (err, strerr):
print 'ERROR: %s' % strerr
# TODO: customise behaviour on error conditions
self._leaveworkdir()
return err
ownerslist = f.readlines()
f.close()
self._parse(ownerslist)
self._leaveworkdir()
return True
def _getlinesfromurl(self,url):
err = 0
strerr = ''
# Retry URL download a few times.
for count in range(self.retries):
if count != 0:
time.sleep(self.retrysecs)
try:
opener = AccountsURLopener(self.username, self.password)
f = opener.open(url)
rc = 0
if 'www-authenticate' in f.headers:
rc = 1
strerr = 'Authentication is required to access %s' % url
break
except IOError, (_err, _strerr):
rc = 1
print url
print _strerr
(err,strerr) = (_err,_strerr)
if rc:
raise IOError, (err, strerr)
else:
l = f.readlines()
f.close()
return l
def _downloadfromurl(self):
self._parse(self._getlinesfromurl(self.url))
return True
def _downloadfrompkgdb(self):
fasdump = self._getlinesfromurl('https://admin.fedoraproject.org/accounts/dump-group.cgi')
self.usermap = {}
for line in fasdump:
fields = line.split(',')
try:
user = fields[0]
addr = fields[1]
except IndexError:
print line
raise
if (addr.find('@') < 0): # unexpected, no addr
print 'No email in:', line
raise Exception
self.usermap[user] = addr
self._parse(self._getlinesfromurl(self.url))
return True
def _download(self):
if self.how == 'url':
return self._downloadfromurl()
elif self.how == 'pkgdb':
return self._downloadfrompkgdb()
elif self.how == 'cvs':
return self._downloadfromcvs()
else:
self.__init__()
return False

View file

@ -0,0 +1,10 @@
# Run Extras repoclosure against EPEL 5.
# i386
#./rc-modified -q -d mdcache -n -c yum.epel.conf -a i686 -r centos-5-i386 -r centos-updates-5-i386 -r fedora-epel-5-i386 -r fedora-epel-testing-5-i386 > rc-epel5-20080113.txt
# x86_64 : APPEND!
#./rc-modified -q -d mdcache -n -c yum.epel.conf -a x86_64 -r centos-5-x86_64 -r centos-updates-5-x86_64 -r fedora-epel-5-x86_64 -r fedora-epel-testing-5-x86_64 >> rc-epel5-20080113.txt
# Show summary which can be sent to mailing-list.
./rc-report.py rc-epel5-20080113.txt -k epel -c rc-report-epel.cfg -w testing

View file

@ -0,0 +1,166 @@
source rpm: R-2.6.1-1.el5.src.rpm
package: R - 2.6.1-1.el5.i386 from fedora-epel-needsign-5-i386
unresolved deps:
perl(File::Copy::Recursive)
source rpm: bodhi-0.4.4-1.el5.src.rpm
package: bodhi-server - 0.4.4-1.el5.noarch from fedora-epel-testing-5-i386
unresolved deps:
yum-utils >= 0:1.1.7
mash
source rpm: fedora-packager-0.1.1-1.el5.src.rpm
package: fedora-packager - 0.1.1-1.el5.noarch from fedora-epel-testing-5-i386
unresolved deps:
plague-client
source rpm: hspell-1.0-7.el5.src.rpm
package: hunspell-he - 1.0-7.el5.i386 from fedora-epel-testing-5-i386
unresolved deps:
hunspell
source rpm: koji-1.2.3-1.el5.src.rpm
package: koji-builder - 1.2.3-1.el5.noarch from fedora-epel-testing-5-i386
unresolved deps:
createrepo >= 0:0.4.11
source rpm: moodle-1.8.2-1.el5.src.rpm
package: moodle - 1.8.2-1.el5.noarch from fedora-epel-testing-5-i386
unresolved deps:
mimetex
source rpm: bzr-gtk-0.93.0-2.el5.src.rpm
package: nautilus-bzr - 0.93.0-2.el5.i386 from fedora-epel-testing-5-i386
unresolved deps:
nautilus-python >= 0:0.4.3-4
source rpm: nautilus-sendto-0.7-5.fc6.src.rpm
package: nautilus-sendto - 0.7-5.fc6.i386 from centos-5-i386
unresolved deps:
libgaim.so.0
source rpm: perl-Test-Base-0.53-1.el5.src.rpm
package: perl-Test-Base - 0.53-1.el5.noarch from fedora-epel-testing-5-i386
unresolved deps:
perl(Module::Install::Base)
source rpm: perl-libwhisker2-2.4-3.el5.src.rpm
package: perl-libwhisker2 - 2.4-3.el5.noarch from fedora-epel-testing-5-i386
unresolved deps:
perl(MD5)
source rpm: python-Coherence-0.2.1-3.el5.src.rpm
package: python-Coherence - 0.2.1-3.el5.noarch from fedora-epel-testing-5-i386
unresolved deps:
python-twisted-core
python-nevow
python-twisted-web
source rpm: revisor-2.0.5-15.el5.src.rpm
package: revisor-jigdo - 2.0.5-15.el5.noarch from fedora-epel-testing-5-i386
unresolved deps:
jigdo
source rpm: snake-0.9-0.5git.el5.src.rpm
package: snake-server - 0.9-0.5git.el5.noarch from fedora-epel-testing-5-i386
unresolved deps:
pykickstart >= 0:1.1
source rpm: translate-toolkit-0.10.1-1.el5.src.rpm
package: translate-toolkit - 0.10.1-1.el5.noarch from fedora-epel-testing-5-i386
unresolved deps:
python-enchant
source rpm: R-2.6.1-1.el5.src.rpm
package: R - 2.6.1-1.el5.i386 from fedora-epel-needsign-5-x86_64
unresolved deps:
perl(File::Copy::Recursive)
source rpm: R-2.6.1-1.el5.src.rpm
package: R - 2.6.1-1.el5.x86_64 from fedora-epel-needsign-5-x86_64
unresolved deps:
perl(File::Copy::Recursive)
source rpm: bodhi-0.4.4-1.el5.src.rpm
package: bodhi-server - 0.4.4-1.el5.noarch from fedora-epel-testing-5-x86_64
unresolved deps:
yum-utils >= 0:1.1.7
mash
source rpm: fedora-packager-0.1.1-1.el5.src.rpm
package: fedora-packager - 0.1.1-1.el5.noarch from fedora-epel-testing-5-x86_64
unresolved deps:
plague-client
source rpm: hspell-1.0-7.el5.src.rpm
package: hunspell-he - 1.0-7.el5.x86_64 from fedora-epel-testing-5-x86_64
unresolved deps:
hunspell
source rpm: koji-1.2.3-1.el5.src.rpm
package: koji-builder - 1.2.3-1.el5.noarch from fedora-epel-testing-5-x86_64
unresolved deps:
createrepo >= 0:0.4.11
source rpm: openib-1.2-6.el5.src.rpm
package: libcxgb3-devel - 1.0-6.el5.i386 from centos-5-x86_64
unresolved deps:
libcxgb3 = 0:1.0-6.el5
source rpm: moodle-1.8.2-1.el5.src.rpm
package: moodle - 1.8.2-1.el5.noarch from fedora-epel-testing-5-x86_64
unresolved deps:
mimetex
source rpm: mozldap-6.0.4-1.el5.src.rpm
package: mozldap-devel - 6.0.4-1.el5.i386 from centos-5-x86_64
unresolved deps:
mozldap = 0:6.0.4-1.el5
source rpm: nagios-2.9-1.el5.src.rpm
package: nagios-devel - 2.9-1.el5.i386 from fedora-epel-5-x86_64
unresolved deps:
nagios = 0:2.9-1.el5
source rpm: bzr-gtk-0.93.0-2.el5.src.rpm
package: nautilus-bzr - 0.93.0-2.el5.x86_64 from fedora-epel-testing-5-x86_64
unresolved deps:
nautilus-python >= 0:0.4.3-4
source rpm: nautilus-sendto-0.7-5.fc6.src.rpm
package: nautilus-sendto - 0.7-5.fc6.x86_64 from centos-5-x86_64
unresolved deps:
libgaim.so.0()(64bit)
source rpm: perl-Test-Base-0.53-1.el5.src.rpm
package: perl-Test-Base - 0.53-1.el5.noarch from fedora-epel-testing-5-x86_64
unresolved deps:
perl(Module::Install::Base)
source rpm: perl-libwhisker2-2.4-3.el5.src.rpm
package: perl-libwhisker2 - 2.4-3.el5.noarch from fedora-epel-testing-5-x86_64
unresolved deps:
perl(MD5)
source rpm: python-Coherence-0.2.1-3.el5.src.rpm
package: python-Coherence - 0.2.1-3.el5.noarch from fedora-epel-testing-5-x86_64
unresolved deps:
python-twisted-core
python-nevow
python-twisted-web
source rpm: revisor-2.0.5-15.el5.src.rpm
package: revisor-jigdo - 2.0.5-15.el5.noarch from fedora-epel-testing-5-x86_64
unresolved deps:
jigdo
source rpm: snake-0.9-0.5git.el5.src.rpm
package: snake-server - 0.9-0.5git.el5.noarch from fedora-epel-testing-5-x86_64
unresolved deps:
pykickstart >= 0:1.1
source rpm: translate-toolkit-0.10.1-1.el5.src.rpm
package: translate-toolkit - 0.10.1-1.el5.noarch from fedora-epel-testing-5-x86_64
unresolved deps:
python-enchant

View file

@ -0,0 +1,282 @@
#!/usr/bin/python -t
# -*- mode: Python; indent-tabs-mode: nil; -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# seth vidal 2005 (c) etc etc
# mschwendt: modified for Fedora Extras buildsys
#Read in the metadata of a series of repositories and check all the
# dependencies in all packages for resolution. Print out the list of
# packages with unresolved dependencies
import sys
import os
# For patched "yum" and "rpmUtils" (post 2.6.1 checkForObsolete support).
# Comment this to use system yum.
#sys.path.insert(0,'/srv/extras-push/work/buildsys-utils/pushscript')
import yum
import yum.Errors
from yum.misc import getCacheDir
from optparse import OptionParser
import rpmUtils.arch
from yum.constants import *
if yum.__version__ < '3.0': # TODO: check
from repomd.packageSack import ListPackageSack
else:
from yum.packageSack import ListPackageSack
def parseArgs():
usage = "usage: %s [-c <config file>] [-a <arch>] [-r <repoid>] [-r <repoid2>]" % sys.argv[0]
parser = OptionParser(usage=usage)
parser.add_option("-c", "--config", default='/etc/yum.conf',
help='config file to use (defaults to /etc/yum.conf)')
parser.add_option("-a", "--arch", default=None,
help='check as if running the specified arch (default: current arch)')
parser.add_option("-r", "--repoid", default=[], action='append',
help="specify repo ids to query, can be specified multiple times (default is all enabled)")
parser.add_option("-t", "--tempcache", default=False, action="store_true",
help="Use a temp dir for storing/accessing yum-cache")
parser.add_option("-d", "--cachedir", default='',
help="specify a custom directory for storing/accessing yum-cache")
parser.add_option("-q", "--quiet", default=0, action="store_true",
help="quiet (no output to stderr)")
parser.add_option("-n", "--newest", default=0, action="store_true",
help="check only the newest packages in the repos")
(opts, args) = parser.parse_args()
return (opts, args)
class RepoClosure(yum.YumBase):
def __init__(self, arch = None, config = "/etc/yum.conf"):
yum.YumBase.__init__(self)
self.arch = arch
if yum.__version__ < '3.0': # TODO: check
self.doConfigSetup(fn = config)
else:
self.doConfigSetup(fn = config, init_plugins = False)
if hasattr(self.repos, 'sqlite'):
self.repos.sqlite = False
self.repos._selectSackType()
def evrTupletoVer(self,tuple):
"""convert and evr tuple to a version string, return None if nothing
to convert"""
e, v,r = tuple
if v is None:
return None
val = ''
if e is not None:
val = '%s:%s' % (e, v)
if r is not None:
val = '%s-%s' % (val, r)
return val
def readMetadata(self):
self.doRepoSetup()
self.doSackSetup(rpmUtils.arch.getArchList(self.arch))
for repo in self.repos.listEnabled():
try: # TODO: when exactly did this change to "mdtype"?
self.repos.populateSack(which=[repo.id], mdtype='filelists')
except TypeError:
self.repos.populateSack(which=[repo.id], with='filelists')
def getBrokenDeps(self, newest=False):
unresolved = {}
resolved = {}
newpkgtuplist = []
if newest:
if yum.__version__ >= '2.9': # TODO: check
pkgs = self.pkgSack.returnNewestByName()
else:
pkgs = []
for l in self.pkgSack.returnNewestByName():
pkgs.extend(l)
newestpkgtuplist = ListPackageSack(pkgs).simplePkgList()
pkgs = self.pkgSack.returnNewestByNameArch()
else:
pkgs = self.pkgSack
self.numpkgs = len(pkgs)
mypkgSack = ListPackageSack(pkgs)
pkgtuplist = mypkgSack.simplePkgList()
# Support new checkForObsolete code in Yum (#190116)
# _if available_
# so we don't examine old _obsolete_ sub-packages.
import rpmUtils.updates
self.up = rpmUtils.updates.Updates([],pkgtuplist)
self.up.rawobsoletes = mypkgSack.returnObsoletes()
haveCheckForObsolete = hasattr(rpmUtils.updates.Updates,'checkForObsolete')
if not haveCheckForObsolete:
print 'WARNING: rpmUtils.updates.checkForObsolete missing!'
for pkg in pkgs:
thispkgobsdict = {}
if haveCheckForObsolete:
try:
thispkgobsdict = self.up.checkForObsolete([pkg.pkgtup])
if thispkgobsdict.has_key(pkg.pkgtup):
continue
except AttributeError:
pass
def isnotnewest(pkg):
# Handle noarch<->arch switches in package updates, so any
# old nevra returned by returnNewestByNameArch() are skipped.
# TODO: There must be a more elegant/convenient way.
if (pkg.pkgtup[1] == 'noarch'):
if (pkg.pkgtup not in newestpkgtuplist):
return True
else:
for p in self.pkgSack.returnNewestByName(pkg.pkgtup[0]):
if (p.pkgtup[1] == 'noarch') and (p.pkgtup in newestpkgtuplist):
return True
return False
for (req, flags, (reqe, reqv, reqr)) in pkg.returnPrco('requires'):
if req.startswith('rpmlib'): continue # ignore rpmlib deps
ver = self.evrTupletoVer((reqe, reqv, reqr))
if resolved.has_key((req,flags,ver)):
continue
try:
resolve_sack = self.whatProvides(req, flags, ver)
except yum.Errors.RepoError, e:
pass
if len(resolve_sack) < 1:
if newest and isnotnewest(pkg):
break
if not unresolved.has_key(pkg):
unresolved[pkg] = []
unresolved[pkg].append((req, flags, ver))
continue
kernelprovides = True # make a false assumption
# If all providers are "kernel*" packages, we allow old ones.
for (pn,pa,pe,pv,pr) in resolve_sack.simplePkgList():
kernelprovides &= pn.startswith('kernel')
if newest and not kernelprovides and not req.startswith('kernel'): # we allow old kernels
resolved_by_newest = False
for po in resolve_sack:# look through and make sure all our answers are newest-only
# 2nd stage handling of obsoletes. Only keep providers,
# which are not obsolete. If no provider is left, the
# dep is unresolved.
thispkgobsdict = {}
if haveCheckForObsolete:
try:
thispkgobsdict = self.up.checkForObsolete([po.pkgtup])
if thispkgobsdict.has_key(po.pkgtup):
continue
except AttributeError:
pass
if po.pkgtup in pkgtuplist:
resolved_by_newest = True
break
if resolved_by_newest:
resolved[(req,flags,ver)] = 1
else:
if newest and isnotnewest(pkg):
break
if not unresolved.has_key(pkg):
unresolved[pkg] = []
unresolved[pkg].append((req, flags, ver))
return unresolved
def log(self, value, msg):
pass
def main():
(opts, cruft) = parseArgs()
my = RepoClosure(arch = opts.arch, config = opts.config)
if opts.repoid:
for repo in my.repos.repos.values():
if repo.id not in opts.repoid:
repo.disable()
else:
repo.enable()
if os.geteuid() != 0 or opts.tempcache or opts.cachedir != '':
if opts.cachedir != '':
cachedir = opts.cachedir
else:
cachedir = getCacheDir()
if cachedir is None:
print "Error: Could not make cachedir, exiting"
sys.exit(50)
my.repos.setCacheDir(cachedir)
if not opts.quiet:
print 'Reading in repository metadata - please wait....'
try:
my.readMetadata()
except yum.Errors.RepoError, e:
print e
sys.exit(1)
if not opts.quiet:
print 'Checking Dependencies'
baddeps = my.getBrokenDeps(opts.newest)
num = my.numpkgs
repos = my.repos.listEnabled()
if not opts.quiet:
print 'Repos looked at: %s' % len(repos)
for repo in repos:
print ' %s' % repo
print 'Num Packages in Repos: %s' % num
pkgs = baddeps.keys()
def sortbyname(a,b):
return cmp(a.__str__(),b.__str__())
pkgs.sort(sortbyname)
for pkg in pkgs:
srcrpm = pkg.returnSimple('sourcerpm')
print 'source rpm: %s\npackage: %s from %s\n unresolved deps: ' % (srcrpm, pkg, pkg.repoid)
for (n, f, v) in baddeps[pkg]:
req = '%s' % n
if f:
flag = LETTERFLAGS[f]
req = '%s %s'% (req, flag)
if v:
req = '%s %s' % (req, v)
print ' %s' % req
print
if __name__ == "__main__":
main()

View file

@ -0,0 +1,9 @@
[FAS]
project = Fedora EPEL
#user = foo
#passwd = secret
[Mail]
from = Fedora Extras repoclosure <buildsys@fedoraproject.org>
replyto = epel-devel-list@redhat.com
subject = Broken dependencies in EPEL

View file

@ -0,0 +1,351 @@
#!/usr/bin/python
# -*- mode: Python; indent-tabs-mode: nil; -*-
import errno, os, sys, stat
import re
import smtplib
import datetime, time
from optparse import OptionParser
import ConfigParser
from PackageOwners import PackageOwners
#from FakeOwners import FakeOwners as PackageOwners
FAS = {
'project' : "Fedora EPEL",
'user' : "",
'passwd' : "",
}
Mail = {
'server' : "localhost",
'user' : "",
'passwd' : "",
'maxsize' : 39*1024,
'from' : "root@localhost",
'replyto' : "root@localhost",
'subject' : "Broken dependencies in EPEL",
}
class BrokenDep:
def __init__(self):
self.pkgid = None # 'name - EVR.arch'
self.repoid = None # e.g. 'fedora-core-6-i386'
self.srp_mname = None
self.age = '' # e.g. '(14 days)'
self.owner = ''
self.coowners = []
# disabled/stripped feature
self.mail = True # whether to notify owner by mail
# disabled/stripped feature
self.new = False
self.report = []
def GetRequires(self):
pkgid2 = self.pkgid.replace(' ','')
r = []
for line in self.report:
if len(line) and not line.isspace() and not line.startswith('package: ') and line.find('unresolved deps:') < 0:
r.append( ' '+pkgid2+' requires '+line.lstrip() )
return '\n'.join(r)
def whiteListed(b): # Just a hook, not a generic white-list feature.
# These two in Fedora 7 Everything most likely won't be fixed.
if b.pkgid.startswith('kmod-em8300') and b.repoid.startswith('fedora-7'):
return True
elif b.pkgid.startswith('kmod-sysprof') and b.repoid.startswith('fedora-7'):
return True
elif b.pkgid.startswith('kmod'): # gah ;) temporarily catch them all
return True
else:
return False
def makeOwners(brokendeps):
owners = PackageOwners()
try:
#if not owners.FromURL():
if not owners.FromURL(repoid=FAS['project'],username=FAS['user'],password=FAS['passwd']):
raise IOError('ERROR: Could not retrieve package owner data.')
except IOError, e:
print e
sys.exit(1)
for b in brokendeps:
toaddr = owners.GetOwner(b.srpm_name)
if toaddr == '':
toaddr = 'UNKNOWN OWNER'
e = 'ERROR: "%s" not in owners.list!\n\n' % b.srpm_name
if e not in errcache:
errcache.append(e)
b.owner = toaddr
b.coowners = owners.GetCoOwnerList(b.srpm_name)
def mail(smtp, fromaddr, toaddrs, replytoaddr, subject, body):
from email.Header import Header
from email.MIMEText import MIMEText
msg = MIMEText( body, 'plain' )
from email.Utils import make_msgid
msg['Message-Id'] = make_msgid()
msg['Subject'] = Header(subject)
msg['From'] = Header(fromaddr)
from email.Utils import formatdate
msg['Date'] = formatdate()
if len(replytoaddr):
msg['ReplyTo'] = Header(replytoaddr)
if isinstance(toaddrs, basestring):
toaddrs = [toaddrs]
to = ''
for t in toaddrs:
if len(to):
to += ', '
to += t
msg['To'] = Header(to)
try:
r = smtp.sendmail( fromaddr, toaddrs, msg.as_string(False) )
for (name, errormsg) in r.iteritems():
print name, ':', errormsg
except smtplib.SMTPRecipientsRefused, obj:
print 'ERROR: SMTPRecipientsRefused'
for (addr, errormsg) in obj.recipients.iteritems():
print addr, ':', errormsg
except smtplib.SMTPException:
print 'ERROR: SMTPException'
def mailsplit(smtp, fromaddr, toaddrs, replytoaddr, subject, body):
# Split mail body at line positions to keep it below maxmailsize.
parts = 0
start = 0
end = len(body)
slices = []
while ( start < end ):
if ( (end-start) > Mail['maxsize'] ):
nextstart = body.rfind( '\n', start, start+Mail['maxsize'] )
if ( nextstart<0 or nextstart==start ):
print 'ERROR: cannot split mail body cleanly'
nextstart = end
else:
nextstart = end
slices.append( (start, nextstart) )
start = nextstart
parts += 1
curpart = 1
for (start,end) in slices:
if (parts>1):
subjectmodified = ( '(%d/%d) %s' % (curpart, parts, subject) )
time.sleep(1)
else:
subjectmodified = subject
slicedbody = body[start:end]
mail(smtp,fromaddr,toaddrs,replytoaddr,subjectmodified,slicedbody)
curpart += 1
def loadConfigFile(filename):
if not filename:
return
config = ConfigParser.ConfigParser()
try:
config.readfp(open(filename))
except IOError, (e, errstr):
print filename, ':', errstr
sys.exit(e)
try:
if config.has_section('FAS'):
for v in ['project','user','passwd']:
if config.has_option('FAS',v):
FAS[v] = config.get('FAS',v)
if config.has_section('Mail'):
for v in ['server','user','passwd','from','replyto','subject']:
if config.has_option('Mail',v):
Mail[v] = config.get('Mail',v)
if config.has_option('Mail','maxsize'):
Mail['maxsize'] = config.getint('Mail','maxsize')
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError), e:
print 'Configuration file error:', e
### main
usage = "Usage: %s <options> <Extras repoclosure report file(s)>" % sys.argv[0]
parser = OptionParser(usage=usage)
parser.add_option("-c", "--config", default=None,
help="config file to use")
parser.add_option("-k", "--keyword", default=[], action='append',
help="a keyword to look for in repoids")
parser.add_option("-m", "--mail", default=[], action='append',
help="what mail to send (owner, summary)")
parser.add_option("-w", "--warn", default=[], action='append',
help="repository warnings to include (needsign, testing)")
parser.add_option("", "--noowners", default=False, action="store_true",
help="don't fetch package owner data from FAS")
(opts, args) = parser.parse_args()
loadConfigFile(opts.config)
domail = len(opts.mail)>0
brokendeps = [] # list of BrokenDeps
errcache = [] # error messages to be included in the summary mail
if not len(args):
print usage
sys.exit(errno.EINVAL)
# Parse extras-repoclosure output files and fill brokendeps array.
while len(args):
logfilename = args[0]
del args[0]
f = file( logfilename )
pkgre = re.compile('(?P<name>.*)-[^-]+-[^-]+$')
inbody = False
srcrpm = ''
for line in f:
if line.startswith('source rpm: '):
w = line.rstrip().split(' ')
srcrpm = w[2]
res = pkgre.search( srcrpm ) # try to get src.rpm "name"
if not res: # only true for invalid input
inbody = False
else:
srpm_name = res.group('name')
inbody = True
continue
elif inbody and line.startswith('package: '):
w = line.rstrip().split(' ')
repoid = w[5]
b = BrokenDep()
b.pkgid = w[1]+' - '+w[3] # name - EVR.arch
b.repoid = repoid
b.srpm_name = srpm_name
brokendeps.append(b)
if inbody:
# Copy report per broken package.
b.report.append( line.rstrip() )
def bdSortByOwnerAndName(a,b):
return cmp(a.owner+a.pkgid,b.owner+b.pkgid)
def bdSortByRepoAndName(a,b):
return cmp(a.repoid+a.pkgid,b.repoid+b.pkgid)
# Filter out unwanted repoids.
for b in list(brokendeps):
for needle in opts.keyword:
if b.repoid.find( needle ) >= 0: # wanted?
break
else:
brokendeps.remove(b)
# Filter out entries from whitelist.
for b in list(brokendeps):
if whiteListed(b):
brokendeps.remove(b)
# Fill in package owners.
if not opts.noowners:
makeOwners(brokendeps)
# Build full mail report per owner. Use a flag for new breakage.
reports = {} # map of lists [new,body] - a flag and the full report for a package owner
if not opts.noowners:
brokendeps.sort(bdSortByOwnerAndName)
for b in brokendeps:
if b.new:
print 'NEW breakage: %s in %s' % (b.pkgid, b.repoid)
if b.mail:
r = '\n'.join(b.report)+'\n'
reports.setdefault(b.owner,[b.new,''])
reports[b.owner][1] += r
# Also build mails for co-owners.
for toaddr in b.coowners:
reports.setdefault(toaddr,[None,''])
reports[toaddr][1] += r
sep = '='*70+'\n'
summail = '' # main summary mail text
reportssummary = '' # any NEW stuff for the summary
def giveNeedsignMsg():
if 'needsign' in opts.warn:
return sep+"The results in this summary consider unreleased updates in the\nbuild-system's needsign-queue!\n"+sep+'\n'
else:
return ''
def giveTestingMsg():
if 'testing' in opts.warn:
return sep+"The results in this summary consider Test Updates!\n"+sep+'\n'
else:
return ''
# Create summary mail text.
reportssummary += giveNeedsignMsg()
reportssummary += giveTestingMsg()
summail += reportssummary
if not opts.noowners and len(brokendeps):
summail += ('Summary of broken packages (by owner):\n')
brokendeps.sort(bdSortByOwnerAndName)
o = None
for b in brokendeps:
if o != b.owner:
o = b.owner
seenbefore = []
summail += '\n '+b.owner.replace('@',' AT ')+'\n'
if b.pkgid not in seenbefore:
summail += ' '+b.pkgid+' '+b.age+'\n'
seenbefore.append(b.pkgid)
# Broken deps sorted by repository id.
brokendeps.sort(bdSortByRepoAndName)
r = None
for b in brokendeps:
if r != b.repoid:
r = b.repoid
summail += '\n\n'+sep+('Broken packages in %s:\n\n' % b.repoid)
summail += b.GetRequires()+'\n'
# Mail init.
if domail:
srv = smtplib.SMTP( Mail['server'] )
if ( len(Mail['user']) and len(Mail['passwd']) ):
try:
srv.login( Mail['user'], Mail['passwd'] )
except smtplib.SMTPException:
print 'ERROR: mailserver login failed'
sys.exit(-1)
# Mail reports to owners.
for toaddr,(new,body) in reports.iteritems():
# Send mail to every package owner with broken package dependencies.
mailtext = 'Your following packages in the repository suffer from broken dependencies:\n\n'
mailtext += giveNeedsignMsg()
mailtext += giveTestingMsg()
mailtext += body
if domail and ('owners' in opts.mail) and toaddr!='UNKNOWN OWNER':
subject = Mail['subject'] + ' - %s' % datetime.date.today()
mail( srv, Mail['from'], toaddr, Mail['replyto'], subject, mailtext )
# Mail summary to mailing-list.
if domail and ('summary' in opts.mail):
subject = Mail['subject'] + ' - %s' % datetime.date.today()
toaddr = Mail['replyto']
mailsplit( srv, Mail['from'], toaddr, '', subject, summail )
if domail:
srv.quit()
if len(summail):
print summail

View file

@ -0,0 +1,97 @@
[main]
cachedir=/tmp/mdcache
debuglevel=2
logfile=/var/log/yum.log
pkgpolicy=newest
distroverpkg=fedora-release
reposdir=/dev/null
exactarch=1
obsoletes=1
retries=20
### EL5 ###
[centos-5-i386]
name=CentOS 5 - i386
baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/5/os/i386/
enabled=0
[centos-updates-5-i386]
name=CentOS Updates 5 - i386
baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/5/updates/i386/
enabled=0
[centos-5-x86_64]
name=CentOS 5 - x86_64
baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/5/os/x86_64/
enabled=0
[centos-updates-5-x86_64]
name=CentOS Updates 5 - x86_64
baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/5/updates/x86_64/
enabled=0
[fedora-epel-5-i386]
name=Fedora EPEL 5 - i386
baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/5/i386/
enabled=0
[fedora-epel-testing-5-i386]
name=Fedora EPEL Test Updates 5 - i386
baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/testing/5/i386/
enabled=0
[fedora-epel-5-x86_64]
name=Fedora EPEL 5 - x86_64
baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/5/x86_64/
enabled=0
[fedora-epel-testing-5-x86_64]
name=Fedora EPEL Test Updates 5 - x86_64
baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/testing/5/x86_64/
enabled=0
### EL4 ###
[centos-4-i386]
name=CentOS 4 - i386
baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/4/os/i386/
enabled=0
[centos-updates-4-i386]
name=CentOS Updates 4 - i386
baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/4/updates/i386/
enabled=0
[centos-4-x86_64]
name=CentOS 4 - x86_64
baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/4/os/x86_64/
enabled=0
[centos-updates-4-x86_64]
name=CentOS Updates 4 - x86_64
baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/4/updates/x86_64/
enabled=0
[fedora-epel-4-i386]
name=Fedora EPEL 4 - i386
baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/4/i386/
enabled=0
[fedora-epel-testing-4-i386]
name=Fedora EPEL Test Updates 4 - i386
baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/testing/4/i386/
enabled=0
[fedora-epel-4-x86_64]
name=Fedora EPEL 4 - x86_64
baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/4/x86_64/
enabled=0
[fedora-epel-testing-4-x86_64]
name=Fedora EPEL Test Updates 4 - x86_64
baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/testing/4/x86_64/
enabled=0