Added epel-repoclosure

scripts/epel-repoclosure/PackageOwners.py

@@ -0,0 +1,277 @@
+#!/usr/bin/python
+# -*- mode: Python; indent-tabs-mode: nil; -*-
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import commands
+import errno
+import os, sys, time
+import shutil
+import tempfile
+from urllib import FancyURLopener
+
+
+class AccountsURLopener(FancyURLopener):
+    """Subclass of urllib.FancyURLopener to allow passing http basic auth info"""
+    def __init__(self, username, password):
+        FancyURLopener.__init__(self)
+        self.username = username
+        self.password = password
+
+    def prompt_user_passwd(self, host, realm):
+        return (self.username, self.password)
+
+
+class PackageOwners:
+    """interface to Fedora package owners list (and Fedora Extras owners/owners.list file)"""
+
+    def __init__(self):
+        self.dict = {}
+        self.how = 'unknown'
+
+
+    def FromURL(self, retries=3, retrysecs=300, url='https://admin.fedoraproject.org/pkgdb/acls/bugzilla?tg_format=plain',
+                pkgdb=True, repoid='Fedora', username=None, password=None):
+        # old url='http://cvs.fedora.redhat.com/viewcvs/*checkout*/owners/owners.list?root=extras'
+        if pkgdb:
+            self.how = 'pkgdb'
+        else:
+            self.how = 'url'
+        self.url = url
+        self.repoid = repoid
+        self.retries = retries
+        self.retrysecs = retrysecs
+        self.username = username
+        self.password = password
+        return self._refresh()
+
+
+    def FromCVS(self, retries=3, retrysecs=300, command='LC_ALL=C CVS_RSH=ssh cvs -f -d :pserver:anonymous@cvs.fedora.redhat.com:/cvs/extras co owners', workdir='',repoid='Fedora'):
+        self.how = 'cvs'
+        self.command = command
+        self.repoid = repoid
+        self.retries = retries
+        self.retrysecs = retrysecs
+        self.workdir = workdir
+        self.ownersfile = os.path.join('owners', 'owners.list')
+        self.cwdstack = []
+        return self._refresh()
+
+
+    def __getitem__(self,rpmname):
+        """return e-mail address from initialowner field"""
+        return self.GetOwner(rpmname)
+
+
+    def GetOwner(self,rpmname):
+        """return e-mail address from initialowner field"""
+        try:
+            r = self.dict[rpmname]['mailto']
+        except KeyError:
+            r = ''
+        return r
+
+
+    def GetOwners(self,rpmname):
+        """return list of e-mail addresses from initialowner+initialcclist fields"""
+        r = self.GetCoOwnerList(rpmname)
+        r2 = self.GetOwner(rpmname)
+        if len(r2):
+            r.append(r2)
+        return r
+
+
+    def GetCoOwnerList(self,rpmname):
+        """return list of e-mail addresses from initialcclist field"""
+        try:
+            r = self.dict[rpmname]['cc']
+        except KeyError:
+            r = []
+        return r
+
+
+    def _enterworkdir(self):
+        self.cwdstack.append( os.getcwd() )
+        if self.workdir != '':
+            os.chdir(self.workdir)
+
+
+    def _leaveworkdir(self):
+        if len(self.cwdstack):
+            os.chdir( self.cwdstack.pop() )
+
+
+    def _refresh(self):
+        self.dict = {}  # map package name to email address, dict[name]
+        return self._download()
+
+
+    def _parse(self,ownerslist):
+        for line in ownerslist:
+            if line.startswith('#') or line.isspace():
+                continue
+            try:
+                (repo,pkgname,summary,emails,qacontact,cc) = line.rstrip().split('|')
+                # This is commented, because we don't need the summary.
+                #summary.replace(r'\u007c','|').replace('\u005c','\\')
+
+                # The PkgDb includes repo's other than Fedora (Fedora EPEL,
+                # Fedora OLPC, and Red Hat Linux, for example).  Skip them.
+                if repo != self.repoid:
+                    continue
+                def fixaddr(a):
+                    # Old Fedora CVS owners.list contains e-mail addresses.
+                    # PkgDb plain output contains usernames only.
+                    if not self.how == 'pkgdb':
+                        return a
+                    if not self.usermap.has_key(a):
+                        return a
+                    return self.usermap[a]
+
+                addrs = []
+                mailto = '' # primary pkg owner
+                if len(emails):
+                    if emails.find(',')>=0:
+                        (addrs) = emails.split(',')
+                        mailto = addrs[0]
+                        addrs = addrs[1:]
+                    else:
+                        mailto = emails
+                    mailto = fixaddr(mailto)
+
+                ccaddrs = []
+                if len(cc):
+                    (ccaddrs) = cc.split(',')
+                addrs += ccaddrs
+                addrs = map(lambda a: fixaddr(a), addrs)
+
+                self.dict[pkgname] = {
+                    'mailto' : mailto,
+                    'cc' : addrs
+                    }
+            except:
+                print 'ERROR: owners.list is broken'
+                print line
+
+
+    def _downloadfromcvs(self):
+        self._enterworkdir()
+        # Dumb caching. Check that file exists and is "quite recent".
+        cached = False
+        try:
+            fstats = os.stat(self.ownersfile)
+            if ( fstats.st_size and
+                 ((time.time() - fstats.st_ctime) < 3600*2) ):
+                cached = True
+        except OSError:
+            pass
+
+        if not cached:
+            # Remove 'owners' directory contents, if it exists.
+            for root, dirs, files in os.walk( 'owners', topdown=False ):
+                for fname in files:
+                    os.remove(os.path.join( root, fname ))
+                for dname in dirs:
+                    os.rmdir(os.path.join( root, dname ))
+            # Retry CVS checkout a few times.
+            for count in range(self.retries):
+                (rc, rv) = commands.getstatusoutput(self.command)
+                if not rc:
+                    break
+                print rv
+                time.sleep(self.retrysecs)
+            if rc:
+                # TODO: customise behaviour on error conditions
+                self._leaveworkdir()
+                return False
+
+        try:
+            f = file( self.ownersfile )
+        except IOError, (err, strerr):
+            print 'ERROR: %s' % strerr
+            # TODO: customise behaviour on error conditions
+            self._leaveworkdir()
+            return err
+        ownerslist = f.readlines()
+        f.close()
+        self._parse(ownerslist)
+        self._leaveworkdir()
+        return True
+
+
+    def _getlinesfromurl(self,url):
+        err = 0
+        strerr = ''
+        # Retry URL download a few times.
+        for count in range(self.retries):
+            if count != 0:
+                time.sleep(self.retrysecs)
+            try:
+                opener = AccountsURLopener(self.username, self.password)
+                f = opener.open(url)
+                rc = 0
+                if 'www-authenticate' in f.headers:
+                    rc = 1
+                    strerr = 'Authentication is required to access %s' % url
+                break
+            except IOError, (_err, _strerr):
+                rc = 1
+                print url
+                print _strerr
+                (err,strerr) = (_err,_strerr)
+        if rc:
+            raise IOError, (err, strerr)
+        else:
+            l = f.readlines()
+            f.close()
+            return l
+
+
+    def _downloadfromurl(self):
+        self._parse(self._getlinesfromurl(self.url))
+        return True
+
+
+    def _downloadfrompkgdb(self):
+        fasdump = self._getlinesfromurl('https://admin.fedoraproject.org/accounts/dump-group.cgi')
+        self.usermap = {}
+        for line in fasdump:
+            fields = line.split(',')
+            try:
+                user = fields[0]
+                addr = fields[1]
+            except IndexError:
+                print line
+                raise
+            if (addr.find('@') < 0):  # unexpected, no addr
+                print 'No email in:', line
+                raise Exception
+            self.usermap[user] = addr
+        self._parse(self._getlinesfromurl(self.url))
+        return True
+
+
+    def _download(self):
+        if self.how == 'url':
+            return self._downloadfromurl()
+        elif self.how == 'pkgdb':
+            return self._downloadfrompkgdb()
+        elif self.how == 'cvs':
+            return self._downloadfromcvs()
+        else:
+            self.__init__()
+            return False
+
+

scripts/epel-repoclosure/demo.sh

@@ -0,0 +1,10 @@
+# Run Extras repoclosure against EPEL 5.
+
+# i386
+#./rc-modified -q -d mdcache -n -c yum.epel.conf -a i686 -r centos-5-i386 -r centos-updates-5-i386 -r fedora-epel-5-i386 -r fedora-epel-testing-5-i386 > rc-epel5-20080113.txt
+
+# x86_64 : APPEND!
+#./rc-modified -q -d mdcache -n -c yum.epel.conf -a x86_64 -r centos-5-x86_64 -r centos-updates-5-x86_64 -r fedora-epel-5-x86_64 -r fedora-epel-testing-5-x86_64 >> rc-epel5-20080113.txt
+
+# Show summary which can be sent to mailing-list.
+./rc-report.py rc-epel5-20080113.txt -k epel -c rc-report-epel.cfg -w testing

scripts/epel-repoclosure/rc-epel5-20080113.txt

@@ -0,0 +1,166 @@
+source rpm: R-2.6.1-1.el5.src.rpm
+package: R - 2.6.1-1.el5.i386 from fedora-epel-needsign-5-i386
+  unresolved deps: 
+     perl(File::Copy::Recursive)
+
+source rpm: bodhi-0.4.4-1.el5.src.rpm
+package: bodhi-server - 0.4.4-1.el5.noarch from fedora-epel-testing-5-i386
+  unresolved deps: 
+     yum-utils >= 0:1.1.7
+     mash
+
+source rpm: fedora-packager-0.1.1-1.el5.src.rpm
+package: fedora-packager - 0.1.1-1.el5.noarch from fedora-epel-testing-5-i386
+  unresolved deps: 
+     plague-client
+
+source rpm: hspell-1.0-7.el5.src.rpm
+package: hunspell-he - 1.0-7.el5.i386 from fedora-epel-testing-5-i386
+  unresolved deps: 
+     hunspell
+
+source rpm: koji-1.2.3-1.el5.src.rpm
+package: koji-builder - 1.2.3-1.el5.noarch from fedora-epel-testing-5-i386
+  unresolved deps: 
+     createrepo >= 0:0.4.11
+
+source rpm: moodle-1.8.2-1.el5.src.rpm
+package: moodle - 1.8.2-1.el5.noarch from fedora-epel-testing-5-i386
+  unresolved deps: 
+     mimetex
+
+source rpm: bzr-gtk-0.93.0-2.el5.src.rpm
+package: nautilus-bzr - 0.93.0-2.el5.i386 from fedora-epel-testing-5-i386
+  unresolved deps: 
+     nautilus-python >= 0:0.4.3-4
+
+source rpm: nautilus-sendto-0.7-5.fc6.src.rpm
+package: nautilus-sendto - 0.7-5.fc6.i386 from centos-5-i386
+  unresolved deps: 
+     libgaim.so.0
+
+source rpm: perl-Test-Base-0.53-1.el5.src.rpm
+package: perl-Test-Base - 0.53-1.el5.noarch from fedora-epel-testing-5-i386
+  unresolved deps: 
+     perl(Module::Install::Base)
+
+source rpm: perl-libwhisker2-2.4-3.el5.src.rpm
+package: perl-libwhisker2 - 2.4-3.el5.noarch from fedora-epel-testing-5-i386
+  unresolved deps: 
+     perl(MD5)
+
+source rpm: python-Coherence-0.2.1-3.el5.src.rpm
+package: python-Coherence - 0.2.1-3.el5.noarch from fedora-epel-testing-5-i386
+  unresolved deps: 
+     python-twisted-core
+     python-nevow
+     python-twisted-web
+
+source rpm: revisor-2.0.5-15.el5.src.rpm
+package: revisor-jigdo - 2.0.5-15.el5.noarch from fedora-epel-testing-5-i386
+  unresolved deps: 
+     jigdo
+
+source rpm: snake-0.9-0.5git.el5.src.rpm
+package: snake-server - 0.9-0.5git.el5.noarch from fedora-epel-testing-5-i386
+  unresolved deps: 
+     pykickstart >= 0:1.1
+
+source rpm: translate-toolkit-0.10.1-1.el5.src.rpm
+package: translate-toolkit - 0.10.1-1.el5.noarch from fedora-epel-testing-5-i386
+  unresolved deps: 
+     python-enchant
+
+source rpm: R-2.6.1-1.el5.src.rpm
+package: R - 2.6.1-1.el5.i386 from fedora-epel-needsign-5-x86_64
+  unresolved deps: 
+     perl(File::Copy::Recursive)
+
+source rpm: R-2.6.1-1.el5.src.rpm
+package: R - 2.6.1-1.el5.x86_64 from fedora-epel-needsign-5-x86_64
+  unresolved deps: 
+     perl(File::Copy::Recursive)
+
+source rpm: bodhi-0.4.4-1.el5.src.rpm
+package: bodhi-server - 0.4.4-1.el5.noarch from fedora-epel-testing-5-x86_64
+  unresolved deps: 
+     yum-utils >= 0:1.1.7
+     mash
+
+source rpm: fedora-packager-0.1.1-1.el5.src.rpm
+package: fedora-packager - 0.1.1-1.el5.noarch from fedora-epel-testing-5-x86_64
+  unresolved deps: 
+     plague-client
+
+source rpm: hspell-1.0-7.el5.src.rpm
+package: hunspell-he - 1.0-7.el5.x86_64 from fedora-epel-testing-5-x86_64
+  unresolved deps: 
+     hunspell
+
+source rpm: koji-1.2.3-1.el5.src.rpm
+package: koji-builder - 1.2.3-1.el5.noarch from fedora-epel-testing-5-x86_64
+  unresolved deps: 
+     createrepo >= 0:0.4.11
+
+source rpm: openib-1.2-6.el5.src.rpm
+package: libcxgb3-devel - 1.0-6.el5.i386 from centos-5-x86_64
+  unresolved deps: 
+     libcxgb3 = 0:1.0-6.el5
+
+source rpm: moodle-1.8.2-1.el5.src.rpm
+package: moodle - 1.8.2-1.el5.noarch from fedora-epel-testing-5-x86_64
+  unresolved deps: 
+     mimetex
+
+source rpm: mozldap-6.0.4-1.el5.src.rpm
+package: mozldap-devel - 6.0.4-1.el5.i386 from centos-5-x86_64
+  unresolved deps: 
+     mozldap = 0:6.0.4-1.el5
+
+source rpm: nagios-2.9-1.el5.src.rpm
+package: nagios-devel - 2.9-1.el5.i386 from fedora-epel-5-x86_64
+  unresolved deps: 
+     nagios = 0:2.9-1.el5
+
+source rpm: bzr-gtk-0.93.0-2.el5.src.rpm
+package: nautilus-bzr - 0.93.0-2.el5.x86_64 from fedora-epel-testing-5-x86_64
+  unresolved deps: 
+     nautilus-python >= 0:0.4.3-4
+
+source rpm: nautilus-sendto-0.7-5.fc6.src.rpm
+package: nautilus-sendto - 0.7-5.fc6.x86_64 from centos-5-x86_64
+  unresolved deps: 
+     libgaim.so.0()(64bit)
+
+source rpm: perl-Test-Base-0.53-1.el5.src.rpm
+package: perl-Test-Base - 0.53-1.el5.noarch from fedora-epel-testing-5-x86_64
+  unresolved deps: 
+     perl(Module::Install::Base)
+
+source rpm: perl-libwhisker2-2.4-3.el5.src.rpm
+package: perl-libwhisker2 - 2.4-3.el5.noarch from fedora-epel-testing-5-x86_64
+  unresolved deps: 
+     perl(MD5)
+
+source rpm: python-Coherence-0.2.1-3.el5.src.rpm
+package: python-Coherence - 0.2.1-3.el5.noarch from fedora-epel-testing-5-x86_64
+  unresolved deps: 
+     python-twisted-core
+     python-nevow
+     python-twisted-web
+
+source rpm: revisor-2.0.5-15.el5.src.rpm
+package: revisor-jigdo - 2.0.5-15.el5.noarch from fedora-epel-testing-5-x86_64
+  unresolved deps: 
+     jigdo
+
+source rpm: snake-0.9-0.5git.el5.src.rpm
+package: snake-server - 0.9-0.5git.el5.noarch from fedora-epel-testing-5-x86_64
+  unresolved deps: 
+     pykickstart >= 0:1.1
+
+source rpm: translate-toolkit-0.10.1-1.el5.src.rpm
+package: translate-toolkit - 0.10.1-1.el5.noarch from fedora-epel-testing-5-x86_64
+  unresolved deps: 
+     python-enchant
+

scripts/epel-repoclosure/rc-modified

@@ -0,0 +1,282 @@
+#!/usr/bin/python -t
+# -*- mode: Python; indent-tabs-mode: nil; -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+# seth vidal 2005 (c) etc etc
+# mschwendt: modified for Fedora Extras buildsys
+
+#Read in the metadata of a series of repositories and check all the
+#   dependencies in all packages for resolution. Print out the list of
+#   packages with unresolved dependencies
+
+import sys
+import os
+
+# For patched "yum" and "rpmUtils" (post 2.6.1 checkForObsolete support).
+# Comment this to use system yum.
+#sys.path.insert(0,'/srv/extras-push/work/buildsys-utils/pushscript')
+
+import yum
+import yum.Errors
+from yum.misc import getCacheDir
+from optparse import OptionParser
+import rpmUtils.arch
+from yum.constants import *
+if yum.__version__ < '3.0':  # TODO: check
+    from repomd.packageSack import ListPackageSack
+else:
+    from yum.packageSack import ListPackageSack
+
+
+def parseArgs():
+    usage = "usage: %s [-c <config file>] [-a <arch>] [-r <repoid>] [-r <repoid2>]" % sys.argv[0]
+    parser = OptionParser(usage=usage)
+    parser.add_option("-c", "--config", default='/etc/yum.conf',
+        help='config file to use (defaults to /etc/yum.conf)')
+    parser.add_option("-a", "--arch", default=None,
+        help='check as if running the specified arch (default: current arch)')
+    parser.add_option("-r", "--repoid", default=[], action='append',
+        help="specify repo ids to query, can be specified multiple times (default is all enabled)")
+    parser.add_option("-t", "--tempcache", default=False, action="store_true", 
+        help="Use a temp dir for storing/accessing yum-cache")
+    parser.add_option("-d", "--cachedir", default='', 
+        help="specify a custom directory for storing/accessing yum-cache")
+    parser.add_option("-q", "--quiet", default=0, action="store_true", 
+                      help="quiet (no output to stderr)")
+    parser.add_option("-n", "--newest", default=0, action="store_true",
+                      help="check only the newest packages in the repos")
+    (opts, args) = parser.parse_args()
+    return (opts, args)
+
+class RepoClosure(yum.YumBase):
+    def __init__(self, arch = None, config = "/etc/yum.conf"):
+        yum.YumBase.__init__(self)
+
+        self.arch = arch
+        if yum.__version__ < '3.0':  # TODO: check
+            self.doConfigSetup(fn = config)
+        else:
+            self.doConfigSetup(fn = config, init_plugins = False)
+        if hasattr(self.repos, 'sqlite'):
+            self.repos.sqlite = False
+            self.repos._selectSackType()
+    
+    def evrTupletoVer(self,tuple):
+        """convert and evr tuple to a version string, return None if nothing
+        to convert"""
+    
+        e, v,r = tuple
+
+        if v is None:
+            return None
+    
+        val = ''
+        if e is not None:
+            val = '%s:%s' % (e, v)
+    
+        if r is not None:
+            val = '%s-%s' % (val, r)
+    
+        return val
+    
+    def readMetadata(self):
+        self.doRepoSetup()
+        self.doSackSetup(rpmUtils.arch.getArchList(self.arch))
+        for repo in self.repos.listEnabled():
+            try:  # TODO: when exactly did this change to "mdtype"?
+                self.repos.populateSack(which=[repo.id], mdtype='filelists')
+            except TypeError:
+                self.repos.populateSack(which=[repo.id], with='filelists')
+
+    def getBrokenDeps(self, newest=False):
+        unresolved = {}
+        resolved = {}
+        newpkgtuplist = []
+        if newest:
+            if yum.__version__ >= '2.9':  # TODO: check
+                pkgs = self.pkgSack.returnNewestByName()
+            else:
+                pkgs = []
+                for l in self.pkgSack.returnNewestByName():
+                    pkgs.extend(l)
+            newestpkgtuplist = ListPackageSack(pkgs).simplePkgList()
+
+            pkgs = self.pkgSack.returnNewestByNameArch()
+        else:
+            pkgs = self.pkgSack
+        self.numpkgs = len(pkgs)
+
+        mypkgSack = ListPackageSack(pkgs)
+        pkgtuplist = mypkgSack.simplePkgList()
+        
+        # Support new checkForObsolete code in Yum (#190116)
+        # _if available_
+        # so we don't examine old _obsolete_ sub-packages.
+        import rpmUtils.updates
+        self.up = rpmUtils.updates.Updates([],pkgtuplist)
+        self.up.rawobsoletes = mypkgSack.returnObsoletes()
+
+        haveCheckForObsolete = hasattr(rpmUtils.updates.Updates,'checkForObsolete')
+        if not haveCheckForObsolete:
+            print 'WARNING: rpmUtils.updates.checkForObsolete missing!'
+
+        for pkg in pkgs:
+            thispkgobsdict = {}
+            if haveCheckForObsolete:
+                try:
+                    thispkgobsdict = self.up.checkForObsolete([pkg.pkgtup])
+                    if thispkgobsdict.has_key(pkg.pkgtup):
+                        continue
+                except AttributeError:
+                    pass
+
+            def isnotnewest(pkg):
+                # Handle noarch<->arch switches in package updates, so any
+                # old nevra returned by returnNewestByNameArch() are skipped.
+                # TODO: There must be a more elegant/convenient way.
+                if (pkg.pkgtup[1] == 'noarch'):
+                    if (pkg.pkgtup not in newestpkgtuplist):
+                        return True
+                else:
+                    for p in self.pkgSack.returnNewestByName(pkg.pkgtup[0]):
+                        if (p.pkgtup[1] == 'noarch') and (p.pkgtup in newestpkgtuplist):
+                            return True
+                return False
+
+            for (req, flags, (reqe, reqv, reqr)) in pkg.returnPrco('requires'):
+                if req.startswith('rpmlib'): continue # ignore rpmlib deps
+            
+                ver = self.evrTupletoVer((reqe, reqv, reqr))
+                if resolved.has_key((req,flags,ver)):
+                    continue
+                try:
+                    resolve_sack = self.whatProvides(req, flags, ver)
+                except yum.Errors.RepoError, e:
+                    pass
+            
+                if len(resolve_sack) < 1:
+                    if newest and isnotnewest(pkg):
+                        break
+                    if not unresolved.has_key(pkg):
+                        unresolved[pkg] = []
+                    unresolved[pkg].append((req, flags, ver))
+                    continue
+                    
+                kernelprovides = True  # make a false assumption
+                # If all providers are "kernel*" packages, we allow old ones.
+                for (pn,pa,pe,pv,pr) in resolve_sack.simplePkgList():
+                    kernelprovides &= pn.startswith('kernel')
+                
+                if newest and not kernelprovides and not req.startswith('kernel'): # we allow old kernels
+                    resolved_by_newest = False
+                    for po in resolve_sack:# look through and make sure all our answers are newest-only
+
+                        # 2nd stage handling of obsoletes. Only keep providers,
+                        # which are not obsolete. If no provider is left, the
+                        # dep is unresolved.
+                        thispkgobsdict = {}
+                        if haveCheckForObsolete:
+                            try:
+                                thispkgobsdict = self.up.checkForObsolete([po.pkgtup])
+                                if thispkgobsdict.has_key(po.pkgtup):
+                                    continue
+                            except AttributeError:
+                                pass
+                        
+                        if po.pkgtup in pkgtuplist:
+                            resolved_by_newest = True
+                            break
+
+                    if resolved_by_newest:                    
+                        resolved[(req,flags,ver)] = 1
+                    else:
+                        if newest and isnotnewest(pkg):
+                            break
+                        if not unresolved.has_key(pkg):
+                            unresolved[pkg] = []
+                        unresolved[pkg].append((req, flags, ver))                        
+                        
+        return unresolved
+    
+    
+    def log(self, value, msg):
+        pass
+
+def main():
+    (opts, cruft) = parseArgs()
+    my = RepoClosure(arch = opts.arch, config = opts.config)
+    
+    if opts.repoid:
+        for repo in my.repos.repos.values():
+            if repo.id not in opts.repoid:
+                repo.disable()
+            else:
+                repo.enable()
+
+    if os.geteuid() != 0 or opts.tempcache or opts.cachedir != '':
+        if opts.cachedir != '':
+            cachedir = opts.cachedir
+        else:
+            cachedir = getCacheDir()
+            if cachedir is None:
+                print "Error: Could not make cachedir, exiting"
+                sys.exit(50)
+            
+        my.repos.setCacheDir(cachedir)
+
+    if not opts.quiet:
+        print 'Reading in repository metadata - please wait....'
+
+    try:
+        my.readMetadata()
+    except yum.Errors.RepoError, e:
+        print e
+        sys.exit(1)
+
+    if not opts.quiet:
+        print 'Checking Dependencies'
+
+    baddeps = my.getBrokenDeps(opts.newest)
+    num = my.numpkgs
+        
+    repos = my.repos.listEnabled()
+
+    if not opts.quiet:
+        print 'Repos looked at: %s' % len(repos)
+        for repo in repos:
+            print '   %s' % repo
+        print 'Num Packages in Repos: %s' % num
+    
+    pkgs = baddeps.keys()
+    def sortbyname(a,b):
+        return cmp(a.__str__(),b.__str__())
+    pkgs.sort(sortbyname)
+    for pkg in pkgs:
+        srcrpm = pkg.returnSimple('sourcerpm')
+        print 'source rpm: %s\npackage: %s from %s\n  unresolved deps: ' % (srcrpm, pkg, pkg.repoid)
+        for (n, f, v) in baddeps[pkg]:
+            req = '%s' % n
+            if f: 
+                flag = LETTERFLAGS[f]
+                req = '%s %s'% (req, flag)
+            if v:
+                req = '%s %s' % (req, v)
+            
+            print '     %s' % req
+        print
+
+if __name__ == "__main__":
+    main()
+        

scripts/epel-repoclosure/rc-report-epel.cfg

@@ -0,0 +1,9 @@
+[FAS]
+project = Fedora EPEL
+#user = foo
+#passwd = secret
+
+[Mail]
+from = Fedora Extras repoclosure <buildsys@fedoraproject.org>
+replyto = epel-devel-list@redhat.com
+subject = Broken dependencies in EPEL

scripts/epel-repoclosure/rc-report.py

@@ -0,0 +1,351 @@
+#!/usr/bin/python
+# -*- mode: Python; indent-tabs-mode: nil; -*-
+
+import errno, os, sys, stat
+import re
+import smtplib
+import datetime, time
+from optparse import OptionParser
+import ConfigParser
+
+from PackageOwners import PackageOwners
+#from FakeOwners import FakeOwners as PackageOwners
+
+FAS = {
+    'project' : "Fedora EPEL",
+    'user' : "",
+    'passwd' : "",
+    }
+
+Mail = {
+    'server' : "localhost",
+    'user' : "",
+    'passwd' : "",
+    'maxsize' : 39*1024,
+    'from' : "root@localhost",
+    'replyto' : "root@localhost",
+    'subject' : "Broken dependencies in EPEL",
+}
+
+class BrokenDep:
+    def __init__(self):
+        self.pkgid = None  # 'name - EVR.arch'
+        self.repoid = None  # e.g. 'fedora-core-6-i386'
+        self.srp_mname = None
+        self.age = ''  # e.g. '(14 days)'
+        self.owner = ''
+        self.coowners = []
+        # disabled/stripped feature
+        self.mail = True  # whether to notify owner by mail
+        # disabled/stripped feature
+        self.new = False
+        self.report = []
+
+    def GetRequires(self):
+        pkgid2 = self.pkgid.replace(' ','')
+        r = []
+        for line in self.report:
+            if len(line) and not line.isspace() and not line.startswith('package: ') and line.find('unresolved deps:') < 0:
+                r.append( '    '+pkgid2+'  requires  '+line.lstrip() )
+        return '\n'.join(r)
+
+
+def whiteListed(b): # Just a hook, not a generic white-list feature.
+    # These two in Fedora 7 Everything most likely won't be fixed.
+    if b.pkgid.startswith('kmod-em8300') and b.repoid.startswith('fedora-7'):
+        return True
+    elif b.pkgid.startswith('kmod-sysprof') and b.repoid.startswith('fedora-7'):
+        return True
+    elif b.pkgid.startswith('kmod'):  # gah ;)  temporarily catch them all
+        return True
+    else:
+        return False
+
+
+def makeOwners(brokendeps):
+    owners = PackageOwners()
+    try:
+        #if not owners.FromURL():
+        if not owners.FromURL(repoid=FAS['project'],username=FAS['user'],password=FAS['passwd']):
+            raise IOError('ERROR: Could not retrieve package owner data.')
+    except IOError, e:
+        print e
+        sys.exit(1)
+    for b in brokendeps:
+        toaddr = owners.GetOwner(b.srpm_name)
+        if toaddr == '':
+            toaddr = 'UNKNOWN OWNER'
+            e = 'ERROR: "%s" not in owners.list!\n\n' % b.srpm_name
+            if e not in errcache:
+                errcache.append(e)
+        b.owner = toaddr
+        b.coowners = owners.GetCoOwnerList(b.srpm_name)
+
+
+def mail(smtp, fromaddr, toaddrs, replytoaddr, subject, body):
+    from email.Header import Header
+    from email.MIMEText import MIMEText
+    msg = MIMEText( body, 'plain' )
+    from email.Utils import make_msgid
+    msg['Message-Id'] = make_msgid()
+    msg['Subject'] = Header(subject)
+    msg['From'] = Header(fromaddr)
+    from email.Utils import formatdate
+    msg['Date'] = formatdate()
+    if len(replytoaddr):
+        msg['ReplyTo'] = Header(replytoaddr)
+
+    if isinstance(toaddrs, basestring):
+        toaddrs = [toaddrs]
+    to = ''
+    for t in toaddrs:
+        if len(to):
+            to += ', '
+        to += t
+    msg['To'] = Header(to)
+
+    try:
+        r = smtp.sendmail( fromaddr, toaddrs, msg.as_string(False) )
+        for (name, errormsg) in r.iteritems():
+            print name, ':', errormsg
+    except smtplib.SMTPRecipientsRefused, obj:
+        print 'ERROR: SMTPRecipientsRefused'
+        for (addr, errormsg) in obj.recipients.iteritems():
+            print addr, ':', errormsg
+    except smtplib.SMTPException:
+        print 'ERROR: SMTPException'
+
+
+def mailsplit(smtp, fromaddr, toaddrs, replytoaddr, subject, body):
+    # Split mail body at line positions to keep it below maxmailsize.
+    parts = 0
+    start = 0
+    end = len(body)
+    slices = []
+    while ( start < end ):
+        if ( (end-start) > Mail['maxsize'] ):
+            nextstart = body.rfind( '\n', start, start+Mail['maxsize'] )
+            if ( nextstart<0 or nextstart==start ):
+                print 'ERROR: cannot split mail body cleanly'
+                nextstart = end
+        else:
+            nextstart = end
+        slices.append( (start, nextstart) )
+        start = nextstart
+        parts += 1
+
+    curpart = 1
+    for (start,end) in slices:
+        if (parts>1):
+            subjectmodified = ( '(%d/%d) %s' % (curpart, parts, subject) )
+            time.sleep(1)
+        else:
+            subjectmodified = subject
+        slicedbody = body[start:end]
+        mail(smtp,fromaddr,toaddrs,replytoaddr,subjectmodified,slicedbody)
+        curpart += 1
+
+
+def loadConfigFile(filename):
+    if not filename:
+        return
+
+    config = ConfigParser.ConfigParser()
+    try:
+        config.readfp(open(filename))
+    except IOError, (e, errstr):
+        print filename, ':', errstr
+        sys.exit(e)
+    
+    try:
+        if config.has_section('FAS'):
+            for v in ['project','user','passwd']:
+                if config.has_option('FAS',v):
+                    FAS[v] = config.get('FAS',v)
+        if config.has_section('Mail'):
+            for v in ['server','user','passwd','from','replyto','subject']:
+                if config.has_option('Mail',v):
+                    Mail[v] = config.get('Mail',v)
+                if config.has_option('Mail','maxsize'):
+                    Mail['maxsize'] = config.getint('Mail','maxsize')
+
+    except (ConfigParser.NoSectionError, ConfigParser.NoOptionError), e:
+        print 'Configuration file error:', e
+
+
+### main
+
+usage = "Usage: %s <options> <Extras repoclosure report file(s)>" % sys.argv[0]
+parser = OptionParser(usage=usage)
+parser.add_option("-c", "--config", default=None,
+                  help="config file to use")
+parser.add_option("-k", "--keyword", default=[], action='append',
+                  help="a keyword to look for in repoids")
+parser.add_option("-m", "--mail", default=[], action='append',
+                  help="what mail to send (owner, summary)")
+parser.add_option("-w", "--warn", default=[], action='append',
+                  help="repository warnings to include (needsign, testing)")
+parser.add_option("", "--noowners", default=False, action="store_true",
+                  help="don't fetch package owner data from FAS")
+(opts, args) = parser.parse_args()
+
+loadConfigFile(opts.config)
+
+domail = len(opts.mail)>0
+brokendeps = []  # list of BrokenDeps
+errcache = []  # error messages to be included in the summary mail
+
+if not len(args):
+    print usage
+    sys.exit(errno.EINVAL)
+# Parse extras-repoclosure output files and fill brokendeps array.
+while len(args):
+    logfilename = args[0]
+    del args[0]
+
+    f = file( logfilename )
+    pkgre = re.compile('(?P<name>.*)-[^-]+-[^-]+$')
+    inbody = False
+    srcrpm = ''
+    for line in f:
+        if line.startswith('source rpm: '):
+            w = line.rstrip().split(' ')
+            srcrpm = w[2]
+            res = pkgre.search( srcrpm )  # try to get src.rpm "name"
+            if not res:  # only true for invalid input
+                inbody = False
+            else:
+                srpm_name = res.group('name')
+                inbody = True
+            continue
+
+        elif inbody and line.startswith('package: '):
+            w = line.rstrip().split(' ')
+            repoid = w[5]
+            b = BrokenDep()
+            b.pkgid = w[1]+' - '+w[3]  # name - EVR.arch
+            b.repoid = repoid
+            b.srpm_name = srpm_name
+            brokendeps.append(b)
+
+        if inbody:
+            # Copy report per broken package.
+            b.report.append( line.rstrip() )
+
+
+def bdSortByOwnerAndName(a,b):
+    return cmp(a.owner+a.pkgid,b.owner+b.pkgid)
+
+def bdSortByRepoAndName(a,b):
+    return cmp(a.repoid+a.pkgid,b.repoid+b.pkgid)
+
+
+# Filter out unwanted repoids.
+for b in list(brokendeps):
+    for needle in opts.keyword:
+        if b.repoid.find( needle ) >= 0:  # wanted?
+            break
+    else:
+        brokendeps.remove(b)
+
+# Filter out entries from whitelist.
+for b in list(brokendeps):
+    if whiteListed(b):
+        brokendeps.remove(b)
+
+# Fill in package owners.
+if not opts.noowners:
+    makeOwners(brokendeps)
+
+# Build full mail report per owner. Use a flag for new breakage.
+reports = {}  # map of lists [new,body] - a flag and the full report for a package owner
+if not opts.noowners:
+    brokendeps.sort(bdSortByOwnerAndName)
+    for b in brokendeps:
+        if b.new:
+            print 'NEW breakage: %s in %s' % (b.pkgid, b.repoid)
+        if b.mail:
+            r = '\n'.join(b.report)+'\n'
+            reports.setdefault(b.owner,[b.new,''])
+            reports[b.owner][1] += r
+            # Also build mails for co-owners.
+            for toaddr in b.coowners:
+                reports.setdefault(toaddr,[None,''])
+                reports[toaddr][1] += r
+
+
+sep = '='*70+'\n'
+summail = ''  # main summary mail text
+reportssummary = ''  # any NEW stuff for the summary
+
+def giveNeedsignMsg():
+    if 'needsign' in opts.warn:
+        return sep+"The results in this summary consider unreleased updates in the\nbuild-system's needsign-queue!\n"+sep+'\n'
+    else:
+        return ''
+
+def giveTestingMsg():
+    if 'testing' in opts.warn:
+        return sep+"The results in this summary consider Test Updates!\n"+sep+'\n'
+    else:
+        return ''
+
+# Create summary mail text.
+reportssummary += giveNeedsignMsg()
+reportssummary += giveTestingMsg()
+summail += reportssummary
+
+if not opts.noowners and len(brokendeps):
+    summail += ('Summary of broken packages (by owner):\n')
+    brokendeps.sort(bdSortByOwnerAndName)
+    o = None
+    for b in brokendeps:
+        if o != b.owner:
+            o = b.owner
+            seenbefore = []
+            summail += '\n    '+b.owner.replace('@',' AT ')+'\n'
+        if b.pkgid not in seenbefore:
+            summail += '        '+b.pkgid+'    '+b.age+'\n'
+            seenbefore.append(b.pkgid)
+
+# Broken deps sorted by repository id.
+brokendeps.sort(bdSortByRepoAndName)
+r = None
+for b in brokendeps:
+    if r != b.repoid:
+        r = b.repoid
+        summail += '\n\n'+sep+('Broken packages in %s:\n\n' % b.repoid)
+    summail += b.GetRequires()+'\n'
+
+# Mail init.
+if domail:
+    srv = smtplib.SMTP( Mail['server'] )
+    if ( len(Mail['user']) and len(Mail['passwd']) ):
+        try:
+            srv.login( Mail['user'], Mail['passwd'] )
+        except smtplib.SMTPException:
+            print 'ERROR: mailserver login failed'
+            sys.exit(-1)
+
+# Mail reports to owners.
+for toaddr,(new,body) in reports.iteritems():
+    # Send mail to every package owner with broken package dependencies.
+    mailtext = 'Your following packages in the repository suffer from broken dependencies:\n\n'
+    mailtext += giveNeedsignMsg()
+    mailtext += giveTestingMsg()
+    mailtext += body
+    if domail and ('owners' in opts.mail) and toaddr!='UNKNOWN OWNER':
+        subject = Mail['subject'] + ' - %s' % datetime.date.today()
+        mail( srv, Mail['from'], toaddr, Mail['replyto'], subject, mailtext )
+
+# Mail summary to mailing-list.
+if domail and ('summary' in opts.mail):
+    subject = Mail['subject'] + ' - %s' % datetime.date.today()
+    toaddr = Mail['replyto']
+    mailsplit( srv, Mail['from'], toaddr, '', subject, summail )
+
+if domail:
+    srv.quit()
+
+if len(summail):
+    print summail

scripts/epel-repoclosure/yum.epel.conf

@@ -0,0 +1,97 @@
+[main]
+cachedir=/tmp/mdcache
+debuglevel=2
+logfile=/var/log/yum.log
+pkgpolicy=newest
+distroverpkg=fedora-release
+reposdir=/dev/null
+exactarch=1
+obsoletes=1
+retries=20
+
+### EL5 ###
+
+[centos-5-i386]
+name=CentOS 5 - i386
+baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/5/os/i386/
+enabled=0
+
+[centos-updates-5-i386]
+name=CentOS Updates 5 - i386
+baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/5/updates/i386/
+enabled=0
+
+[centos-5-x86_64]
+name=CentOS 5 - x86_64
+baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/5/os/x86_64/
+enabled=0
+
+[centos-updates-5-x86_64]
+name=CentOS Updates 5 - x86_64
+baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/5/updates/x86_64/
+enabled=0
+
+
+[fedora-epel-5-i386]
+name=Fedora EPEL 5 - i386
+baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/5/i386/
+enabled=0
+
+[fedora-epel-testing-5-i386]
+name=Fedora EPEL Test Updates 5 - i386
+baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/testing/5/i386/
+enabled=0
+
+[fedora-epel-5-x86_64]
+name=Fedora EPEL 5 - x86_64
+baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/5/x86_64/
+enabled=0
+
+[fedora-epel-testing-5-x86_64]
+name=Fedora EPEL Test Updates 5 - x86_64
+baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/testing/5/x86_64/
+enabled=0
+
+### EL4 ###
+
+[centos-4-i386]
+name=CentOS 4 - i386
+baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/4/os/i386/
+enabled=0
+
+[centos-updates-4-i386]
+name=CentOS Updates 4 - i386
+baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/4/updates/i386/
+enabled=0
+
+[centos-4-x86_64]
+name=CentOS 4 - x86_64
+baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/4/os/x86_64/
+enabled=0
+
+[centos-updates-4-x86_64]
+name=CentOS Updates 4 - x86_64
+baseurl=http://wftp.tu-chemnitz.de/pub/linux/centos/4/updates/x86_64/
+enabled=0
+
+
+[fedora-epel-4-i386]
+name=Fedora EPEL 4 - i386
+baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/4/i386/
+enabled=0
+
+[fedora-epel-testing-4-i386]
+name=Fedora EPEL Test Updates 4 - i386
+baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/testing/4/i386/
+enabled=0
+
+[fedora-epel-4-x86_64]
+name=Fedora EPEL 4 - x86_64
+baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/4/x86_64/
+enabled=0
+
+[fedora-epel-testing-4-x86_64]
+name=Fedora EPEL Test Updates 4 - x86_64
+baseurl=http://wftp.tu-chemnitz.de/pub/linux/fedora-epel/testing/4/x86_64/
+enabled=0
+