add mockchain so I can get fixes in place for it for copr

files/copr/provision/builderpb.yml

@@ -70,7 +70,7 @@
 
   - name: mockbuilder user   
     action: user name=mockbuilder groups=mock
-
+h
   - name: mockbuilder .ssh
     action: file state=directory path=/home/mockbuilder/.ssh mode=0700 owner=mockbuilder group=mockbuilder
 
@@ -84,4 +84,7 @@
     - epel-5-x86_64.cfg
     - epel-5-i386.cfg
 
+  - name: put updated   mockchain into /usr/bin
+    action: copy src=$files/mockchain dest=/usr/bin/mockchain mode=0755 owner=root group=root
+
 

files/copr/provision/files/mockchain

@@ -0,0 +1,337 @@
+#!/usr/bin/python -tt
+# by skvidal@fedoraproject.org
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA.
+# copyright 2012 Red Hat, Inc.
+
+# SUMMARY
+# mockchain
+# take a mock config and a series of srpms
+# rebuild them one at a time
+# adding each to a local repo
+# so they are available as build deps to next pkg being built
+
+import sys
+import subprocess
+import os
+import optparse
+import tempfile
+import shutil
+from urlgrabber import grabber
+import time
+
+mockconfig_path='/etc/mock'
+
+def createrepo(path):
+    if os.path.exists(path + '/repodata/repomd.xml'):
+        comm = ['/usr/bin/createrepo', '--update', path]
+    else:
+        comm = ['/usr/bin/createrepo', path]
+    cmd = subprocess.Popen(comm,
+             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    out, err = cmd.communicate()
+    return out, err
+
+def parse_args(args):
+    parser = optparse.OptionParser('\nmockchain -r mockcfg pkg1 [pkg2] [pkg3]')
+    parser.add_option('-r', '--root', default=None, dest='chroot',
+            help="chroot config name/base to use in the mock build")
+    parser.add_option('-l', '--localrepo', default=None, 
+            help="local path for the local repo, defaults to making its own")
+    parser.add_option('-c', '--continue', default=False, action='store_true',
+            dest='cont',
+            help="if a pkg fails to build, continue to the next one")
+    parser.add_option('-a','--addrepo', default=[], action='append',
+            dest='repos',
+            help="add these repo baseurls to the chroot's yum config")
+    parser.add_option('--recurse', default=False, action='store_true',
+            help="if more than one pkg and it fails to build, try to build the rest and come back to it")
+    parser.add_option('--log', default=None, dest='logfile',
+            help="log to the file named by this option, defaults to not logging")
+    parser.add_option('--tmp_prefix', default=None, dest='tmp_prefix',
+            help="tmp dir prefix - will default to username-pid if not specified")
+
+            
+    #FIXME?
+    # figure out how to pass other args to mock?
+    
+    opts, args = parser.parse_args(args)
+    if opts.recurse:
+        opts.cont = True
+    
+    if not opts.chroot:
+        print "You must provide an argument to -r for the mock chroot"
+        sys.exit(1)
+        
+        
+    if len(sys.argv) < 3:
+        print "You must specifiy at least 1 package to build"
+        sys.exit(1)
+
+
+    return opts, args
+    
+def add_local_repo(infile, destfile, baseurl, repoid=None):
+    """take a mock chroot config and add a repo to it's yum.conf
+       infile = mock chroot config file
+       destfile = where to save out the result
+       baseurl = baseurl of repo you wish to add"""
+        
+    try:
+        config_opts = {}
+        execfile(infile)
+        if not repoid:
+            repoid=baseurl.split('//')[1].replace('/','_')
+        localyumrepo="""
+[%s]
+name=%s
+baseurl=%s
+enabled=1
+skip_if_unavailable=1
+metadata_expire=30
+cost=1
+""" % (repoid, baseurl, baseurl)
+
+        config_opts['yum.conf'] += localyumrepo
+        br_dest = open(destfile, 'w')
+        for k,v in config_opts.items():
+            br_dest.write("config_opts[%r] = %r\n" % (k, v))
+        br_dest.close()
+        return True, ''
+    except (IOError, OSError):
+        return False, "Could not write mock config to %s" % destfile
+
+    return True, ''
+
+def do_build(opts, cfg, pkg):
+
+    # returns 0, cmd, out, err = failure
+    # returns 1, cmd, out, err  = success
+    # returns 2, None, None, None = already built
+    
+    s_pkg = os.path.basename(pkg)
+    pdn = s_pkg.replace('.src.rpm', '')
+    resdir = '%s/%s' % (opts.local_repo_dir, pdn)
+    resdir = os.path.normpath(resdir)
+    if not os.path.exists(resdir):
+        os.makedirs(resdir)
+
+    success_file = resdir + '/success'
+    fail_file = resdir + '/fail'
+    
+    if os.path.exists(success_file):
+        return 2, None, None, None
+    
+    # clean it up if we're starting over :)
+    if os.path.exists(fail_file):
+        os.unlink(fail_file)
+        
+    mockcmd = ['/usr/bin/mock', 
+               '--configdir', opts.config_path,
+               '--resultdir', resdir,
+               '--uniqueext', opts.uniqueext,
+               '-r', cfg, ]
+    print 'building %s' % s_pkg
+    mockcmd.append(pkg)
+    cmd = subprocess.Popen(mockcmd,
+           stdout=subprocess.PIPE, 
+           stderr=subprocess.PIPE )
+    out, err = cmd.communicate()
+    if cmd.returncode == 0:
+        open(success_file, 'w').write('done\n')
+        ret = 1
+    else:
+        open(fail_file, 'w').write('undone\n')
+        ret = 0
+    
+    return ret, cmd, out, err
+
+
+def log(lf, msg):
+    if lf:
+        now = time.time()
+        try:
+            open(lf, 'a').write(str(now) + ':' + msg + '\n')
+        except (IOError, OSError), e:
+            print 'Could not write to logfile %s - %s' % (lf, str(e))
+    print msg
+    
+    
+    
+def main(args):
+    opts, args = parse_args(args)
+
+    # take mock config + list of pkgs
+    cfg=opts.chroot
+    pkgs=args[1:]
+    mockcfg = mockconfig_path + '/' + cfg + '.cfg'
+    
+    if not os.path.exists(mockcfg):
+        print "could not find config: %s" % mockcfg
+        sys.exit(1)
+
+
+    if not opts.tmp_prefix:
+        try:
+            opts.tmp_prefix = os.getlogin()
+        except OSError, e:
+            print "Could not find login name for tmp dir prefix add --tmp_prefix"
+            sys.exit(1)
+    pid = os.getpid()
+    opts.uniqueext = '%s-%s' % (opts.tmp_prefix, pid)
+
+
+    # create a tempdir for our local info
+    if opts.localrepo:
+        local_tmp_dir = os.path.abspath(opts.localrepo)
+        if not os.path.exists(local_tmp_dir):
+            os.makedirs(local_tmp_dir)
+    else:
+        pre = 'mock-chain-%s-' % opts.uniqueext
+        local_tmp_dir = tempfile.mkdtemp(prefix=pre, dir='/var/tmp')
+
+    os.chmod(local_tmp_dir, 0755)
+
+    if opts.logfile:
+        opts.logfile = os.path.join(local_tmp_dir, opts.logfile)
+        if os.path.exists(opts.logfile):
+            os.unlink(opts.logfile)
+            
+    log(opts.logfile, "starting logfile: %s" % opts.logfile)
+    opts.local_repo_dir = os.path.normpath(local_tmp_dir + '/results/' + cfg + '/')
+
+    if not os.path.exists(opts.local_repo_dir):
+        os.makedirs(opts.local_repo_dir, mode=0755)
+    
+    local_baseurl="file://%s" % opts.local_repo_dir
+    log(opts.logfile, "results dir: %s" % opts.local_repo_dir)
+    opts.config_path = os.path.normpath(local_tmp_dir + '/configs/' + cfg + '/')
+    
+    if not os.path.exists(opts.config_path):
+        os.makedirs(opts.config_path, mode=0755)
+
+    log(opts.logfile, "config dir: %s" % opts.config_path)
+
+    my_mock_config = opts.config_path + '/' + os.path.basename(mockcfg)
+    
+    # modify with localrepo
+    res, msg = add_local_repo(mockcfg, my_mock_config, local_baseurl, 'local_build_repo')
+    if not res:
+         log(opts.logfile, "Error: Could not write out local config: %s" % msg)
+         sys.exit(1)
+
+    for baseurl in opts.repos:
+        res, msg =  add_local_repo(my_mock_config, my_mock_config, baseurl)
+        if not res:
+            log(opts.logfile, "Error: Could not add: %s to yum config in mock chroot: %s" % (baseurl, msg))
+            sys.exit(1)
+
+        
+    # these files needed from the mock.config dir to make mock run
+    for fn in ['site-defaults.cfg', 'logging.ini']:
+        pth = mockconfig_path + '/' + fn
+        shutil.copyfile(pth, opts.config_path + '/' + fn)
+
+   
+    # createrepo on it
+    out, err = createrepo(opts.local_repo_dir)
+    if err.strip():
+        log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir)
+        log(opts.logfile, "Err: %s" % err)
+        sys.exit(1)
+
+ 
+    download_dir = tempfile.mkdtemp()
+    downloaded_pkgs = {}
+    built_pkgs = []
+    try_again = True
+    to_be_built = pkgs
+    while try_again:
+        failed = []
+        for pkg in to_be_built:
+            if not pkg.endswith('.rpm'):
+                log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg)
+                failed.append(pkg)
+                continue
+            
+            elif pkg.startswith('http://') or pkg.startswith('https://'):
+                url = pkg
+                cwd = os.getcwd()
+                os.chdir(download_dir)
+                try:
+                    log(opts.logfile, 'Fetching %s' % url)
+                    ug = grabber.URLGrabber()
+                    fn = ug.urlgrab(url)
+                    pkg = download_dir + '/' + fn
+                except Exception, e:
+                    log(opts.logfile, 'Error Downloading %s: %s' % (url, str(e)))
+                    failed.append(url)
+                    os.chdir(cwd)
+                    continue
+                else:
+                    os.chdir(cwd)
+                    downloaded_pkgs[pkg] = url
+            log(opts.logfile, "Start build: %s" % pkg)
+            ret, cmd, out, err = do_build(opts, cfg, pkg)
+            log(opts.logfile, "End build: %s" % pkg) 
+            if ret == 0:
+                if opts.recurse:
+                    failed.append(pkg)
+                    log(opts.logfile, "Error building %s, will try again" % os.path.basename(pkg))
+                else:
+                    log(opts.logfile,"Error building %s" % os.path.basename(pkg))
+                    log(opts.logfile,"See logs/results in %s" % opts.local_repo_dir)
+                    if not opts.cont:
+                        sys.exit(1)
+                    
+            elif ret == 1:
+                log(opts.logfile, "Success building %s" % os.path.basename(pkg))
+                built_pkgs.append(pkg)
+                # createrepo with the new pkgs
+                out, err = createrepo(opts.local_repo_dir)
+                if err.strip():
+                    log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir)
+                    log(opts.logfile, "Err: %s" % err)
+            elif ret == 2:
+                log(opts.logfile, "Skipping already built pkg %s" % os.path.basename(pkg))
+        
+        if failed:
+            if len(failed) != len(to_be_built):
+                to_be_built = failed
+                try_again = True
+                log(opts.logfile, 'Trying to rebuild %s failed pkgs' % len(failed))
+            else:
+                log(opts.logfile, "Tried twice - following pkgs could not be successfully built:")
+                for pkg in failed:
+                    msg = pkg
+                    if pkg in downloaded_pkgs:
+                        msg = downloaded_pkgs[pkg]
+                    log(opts.logfile, msg)
+
+                try_again = False
+        else:
+            try_again = False
+            
+    # cleaning up our download dir
+    shutil.rmtree(download_dir, ignore_errors=True)
+    
+    log(opts.logfile, "Results out to: %s" % opts.local_repo_dir)
+    log(opts.logfile, "Pkgs built: %s" % len(built_pkgs))
+    log(opts.logfile, "Packages successfully built in this order:")
+    for pkg in built_pkgs:
+        log(opts.logfile, pkg)
+
+if __name__ == "__main__":
+    main(sys.argv)
+    sys.exit(0)