From 554d9dad52807830acb42f16ea81f2e20d4810ce Mon Sep 17 00:00:00 2001 From: Seth Vidal Date: Wed, 16 Jan 2013 05:35:41 +0000 Subject: [PATCH] add mockchain so I can get fixes in place for it for copr --- files/copr/provision/builderpb.yml | 7 +- files/copr/provision/files/mockchain | 337 +++++++++++++++++++++++++++ 2 files changed, 342 insertions(+), 2 deletions(-) create mode 100755 files/copr/provision/files/mockchain diff --git a/files/copr/provision/builderpb.yml b/files/copr/provision/builderpb.yml index fb1910ae18..031f372587 100644 --- a/files/copr/provision/builderpb.yml +++ b/files/copr/provision/builderpb.yml @@ -70,7 +70,7 @@ - name: mockbuilder user action: user name=mockbuilder groups=mock - +h - name: mockbuilder .ssh action: file state=directory path=/home/mockbuilder/.ssh mode=0700 owner=mockbuilder group=mockbuilder @@ -83,5 +83,8 @@ - site-defaults.cfg - epel-5-x86_64.cfg - epel-5-i386.cfg - + + - name: put updated mockchain into /usr/bin + action: copy src=$files/mockchain dest=/usr/bin/mockchain mode=0755 owner=root group=root + diff --git a/files/copr/provision/files/mockchain b/files/copr/provision/files/mockchain new file mode 100755 index 0000000000..ebc6581232 --- /dev/null +++ b/files/copr/provision/files/mockchain @@ -0,0 +1,337 @@ +#!/usr/bin/python -tt +# by skvidal@fedoraproject.org +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# copyright 2012 Red Hat, Inc. + +# SUMMARY +# mockchain +# take a mock config and a series of srpms +# rebuild them one at a time +# adding each to a local repo +# so they are available as build deps to next pkg being built + +import sys +import subprocess +import os +import optparse +import tempfile +import shutil +from urlgrabber import grabber +import time + +mockconfig_path='/etc/mock' + +def createrepo(path): + if os.path.exists(path + '/repodata/repomd.xml'): + comm = ['/usr/bin/createrepo', '--update', path] + else: + comm = ['/usr/bin/createrepo', path] + cmd = subprocess.Popen(comm, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = cmd.communicate() + return out, err + +def parse_args(args): + parser = optparse.OptionParser('\nmockchain -r mockcfg pkg1 [pkg2] [pkg3]') + parser.add_option('-r', '--root', default=None, dest='chroot', + help="chroot config name/base to use in the mock build") + parser.add_option('-l', '--localrepo', default=None, + help="local path for the local repo, defaults to making its own") + parser.add_option('-c', '--continue', default=False, action='store_true', + dest='cont', + help="if a pkg fails to build, continue to the next one") + parser.add_option('-a','--addrepo', default=[], action='append', + dest='repos', + help="add these repo baseurls to the chroot's yum config") + parser.add_option('--recurse', default=False, action='store_true', + help="if more than one pkg and it fails to build, try to build the rest and come back to it") + parser.add_option('--log', default=None, dest='logfile', + help="log to the file named by this option, defaults to not logging") + parser.add_option('--tmp_prefix', default=None, dest='tmp_prefix', + help="tmp dir prefix - will default to username-pid if not specified") + + + #FIXME? + # figure out how to pass other args to mock? + + opts, args = parser.parse_args(args) + if opts.recurse: + opts.cont = True + + if not opts.chroot: + print "You must provide an argument to -r for the mock chroot" + sys.exit(1) + + + if len(sys.argv) < 3: + print "You must specifiy at least 1 package to build" + sys.exit(1) + + + return opts, args + +def add_local_repo(infile, destfile, baseurl, repoid=None): + """take a mock chroot config and add a repo to it's yum.conf + infile = mock chroot config file + destfile = where to save out the result + baseurl = baseurl of repo you wish to add""" + + try: + config_opts = {} + execfile(infile) + if not repoid: + repoid=baseurl.split('//')[1].replace('/','_') + localyumrepo=""" +[%s] +name=%s +baseurl=%s +enabled=1 +skip_if_unavailable=1 +metadata_expire=30 +cost=1 +""" % (repoid, baseurl, baseurl) + + config_opts['yum.conf'] += localyumrepo + br_dest = open(destfile, 'w') + for k,v in config_opts.items(): + br_dest.write("config_opts[%r] = %r\n" % (k, v)) + br_dest.close() + return True, '' + except (IOError, OSError): + return False, "Could not write mock config to %s" % destfile + + return True, '' + +def do_build(opts, cfg, pkg): + + # returns 0, cmd, out, err = failure + # returns 1, cmd, out, err = success + # returns 2, None, None, None = already built + + s_pkg = os.path.basename(pkg) + pdn = s_pkg.replace('.src.rpm', '') + resdir = '%s/%s' % (opts.local_repo_dir, pdn) + resdir = os.path.normpath(resdir) + if not os.path.exists(resdir): + os.makedirs(resdir) + + success_file = resdir + '/success' + fail_file = resdir + '/fail' + + if os.path.exists(success_file): + return 2, None, None, None + + # clean it up if we're starting over :) + if os.path.exists(fail_file): + os.unlink(fail_file) + + mockcmd = ['/usr/bin/mock', + '--configdir', opts.config_path, + '--resultdir', resdir, + '--uniqueext', opts.uniqueext, + '-r', cfg, ] + print 'building %s' % s_pkg + mockcmd.append(pkg) + cmd = subprocess.Popen(mockcmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE ) + out, err = cmd.communicate() + if cmd.returncode == 0: + open(success_file, 'w').write('done\n') + ret = 1 + else: + open(fail_file, 'w').write('undone\n') + ret = 0 + + return ret, cmd, out, err + + +def log(lf, msg): + if lf: + now = time.time() + try: + open(lf, 'a').write(str(now) + ':' + msg + '\n') + except (IOError, OSError), e: + print 'Could not write to logfile %s - %s' % (lf, str(e)) + print msg + + + +def main(args): + opts, args = parse_args(args) + + # take mock config + list of pkgs + cfg=opts.chroot + pkgs=args[1:] + mockcfg = mockconfig_path + '/' + cfg + '.cfg' + + if not os.path.exists(mockcfg): + print "could not find config: %s" % mockcfg + sys.exit(1) + + + if not opts.tmp_prefix: + try: + opts.tmp_prefix = os.getlogin() + except OSError, e: + print "Could not find login name for tmp dir prefix add --tmp_prefix" + sys.exit(1) + pid = os.getpid() + opts.uniqueext = '%s-%s' % (opts.tmp_prefix, pid) + + + # create a tempdir for our local info + if opts.localrepo: + local_tmp_dir = os.path.abspath(opts.localrepo) + if not os.path.exists(local_tmp_dir): + os.makedirs(local_tmp_dir) + else: + pre = 'mock-chain-%s-' % opts.uniqueext + local_tmp_dir = tempfile.mkdtemp(prefix=pre, dir='/var/tmp') + + os.chmod(local_tmp_dir, 0755) + + if opts.logfile: + opts.logfile = os.path.join(local_tmp_dir, opts.logfile) + if os.path.exists(opts.logfile): + os.unlink(opts.logfile) + + log(opts.logfile, "starting logfile: %s" % opts.logfile) + opts.local_repo_dir = os.path.normpath(local_tmp_dir + '/results/' + cfg + '/') + + if not os.path.exists(opts.local_repo_dir): + os.makedirs(opts.local_repo_dir, mode=0755) + + local_baseurl="file://%s" % opts.local_repo_dir + log(opts.logfile, "results dir: %s" % opts.local_repo_dir) + opts.config_path = os.path.normpath(local_tmp_dir + '/configs/' + cfg + '/') + + if not os.path.exists(opts.config_path): + os.makedirs(opts.config_path, mode=0755) + + log(opts.logfile, "config dir: %s" % opts.config_path) + + my_mock_config = opts.config_path + '/' + os.path.basename(mockcfg) + + # modify with localrepo + res, msg = add_local_repo(mockcfg, my_mock_config, local_baseurl, 'local_build_repo') + if not res: + log(opts.logfile, "Error: Could not write out local config: %s" % msg) + sys.exit(1) + + for baseurl in opts.repos: + res, msg = add_local_repo(my_mock_config, my_mock_config, baseurl) + if not res: + log(opts.logfile, "Error: Could not add: %s to yum config in mock chroot: %s" % (baseurl, msg)) + sys.exit(1) + + + # these files needed from the mock.config dir to make mock run + for fn in ['site-defaults.cfg', 'logging.ini']: + pth = mockconfig_path + '/' + fn + shutil.copyfile(pth, opts.config_path + '/' + fn) + + + # createrepo on it + out, err = createrepo(opts.local_repo_dir) + if err.strip(): + log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir) + log(opts.logfile, "Err: %s" % err) + sys.exit(1) + + + download_dir = tempfile.mkdtemp() + downloaded_pkgs = {} + built_pkgs = [] + try_again = True + to_be_built = pkgs + while try_again: + failed = [] + for pkg in to_be_built: + if not pkg.endswith('.rpm'): + log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg) + failed.append(pkg) + continue + + elif pkg.startswith('http://') or pkg.startswith('https://'): + url = pkg + cwd = os.getcwd() + os.chdir(download_dir) + try: + log(opts.logfile, 'Fetching %s' % url) + ug = grabber.URLGrabber() + fn = ug.urlgrab(url) + pkg = download_dir + '/' + fn + except Exception, e: + log(opts.logfile, 'Error Downloading %s: %s' % (url, str(e))) + failed.append(url) + os.chdir(cwd) + continue + else: + os.chdir(cwd) + downloaded_pkgs[pkg] = url + log(opts.logfile, "Start build: %s" % pkg) + ret, cmd, out, err = do_build(opts, cfg, pkg) + log(opts.logfile, "End build: %s" % pkg) + if ret == 0: + if opts.recurse: + failed.append(pkg) + log(opts.logfile, "Error building %s, will try again" % os.path.basename(pkg)) + else: + log(opts.logfile,"Error building %s" % os.path.basename(pkg)) + log(opts.logfile,"See logs/results in %s" % opts.local_repo_dir) + if not opts.cont: + sys.exit(1) + + elif ret == 1: + log(opts.logfile, "Success building %s" % os.path.basename(pkg)) + built_pkgs.append(pkg) + # createrepo with the new pkgs + out, err = createrepo(opts.local_repo_dir) + if err.strip(): + log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir) + log(opts.logfile, "Err: %s" % err) + elif ret == 2: + log(opts.logfile, "Skipping already built pkg %s" % os.path.basename(pkg)) + + if failed: + if len(failed) != len(to_be_built): + to_be_built = failed + try_again = True + log(opts.logfile, 'Trying to rebuild %s failed pkgs' % len(failed)) + else: + log(opts.logfile, "Tried twice - following pkgs could not be successfully built:") + for pkg in failed: + msg = pkg + if pkg in downloaded_pkgs: + msg = downloaded_pkgs[pkg] + log(opts.logfile, msg) + + try_again = False + else: + try_again = False + + # cleaning up our download dir + shutil.rmtree(download_dir, ignore_errors=True) + + log(opts.logfile, "Results out to: %s" % opts.local_repo_dir) + log(opts.logfile, "Pkgs built: %s" % len(built_pkgs)) + log(opts.logfile, "Packages successfully built in this order:") + for pkg in built_pkgs: + log(opts.logfile, pkg) + +if __name__ == "__main__": + main(sys.argv) + sys.exit(0)