diff --git a/scripts/distgit/mkbranch b/scripts/distgit/mkbranch new file mode 100755 index 0000000..58a5da8 --- /dev/null +++ b/scripts/distgit/mkbranch @@ -0,0 +1,183 @@ +#!/bin/bash +# +# Create a new development branch for a module. +# THIS HAS TO BE RUN ON THE GIT SERVER! + +# WARNING: +# This file is maintained within puppet? +# All local changes will be lost. + + +# Figure out the environment we're running in +RUNDIR=$(cd $(dirname $0) && pwd) +GITROOT=/srv/git/rpms + +# check if a moron is driving me +if [ ! -d $GITROOT ] ; then + # we're not on the git server (this check is fragile) + echo "ERROR: This script has to be run on the git server." + echo "ERROR: Homer sez 'Duh'." + exit -9 +fi + +# where are the packages kept +TOPLEVEL=rpms + +# Local variables +VERBOSE=0 +TEST= +IGNORE= +BRANCH="" +PACKAGES="" +SRC_BRANCH="master" +AUTHOR="Fedora Release Engineering " + +Usage() { + cat <] ... + + Creates a new branch for the list of s. + The /master suffix on branch names is assumed. + +Options: + -s,--source= Use as the source branch. + Defaults is master + /master suffix on other branches assumed + -n,--test Don't do nothing, only test + -i,--ignore Ignore erroneous modules + -h,--help This help message + -v,--verbose Increase verbosity +EOF +} + +# parse the arguments +while [ -n "$1" ] ; do + case "$1" in + -h | --help ) + Usage + exit 0 + ;; + + -v | --verbose ) + VERBOSE=$(($VERBOSE + 1)) + ;; + + -i | --ignore ) + IGNORE="yes" + ;; + + -n | --test ) + TEST="yes" + ;; + + -s | --source ) + shift + SRC_BRANCH=$1 + ;; + + -b | --branch ) + shift + BRANCH=$1/master + ;; + + * ) + if [ -z "$BRANCH" ] ; then + BRANCH="$1" + else + PACKAGES="$PACKAGES $1" + fi + ;; + esac + shift +done + +# check the arguments +if [ -z "$BRANCH" -o -z "$PACKAGES" ] ; then + Usage + exit -1 +fi + +# Translate the source branch +if [ $SRC_BRANCH == 'master' ] ; then + echo > /dev/null +else + $SRC_BRANCH = $SRC_BRANCH/master +fi + +# Sanity checks before we start doing damage +NEWP= +for p in $PACKAGES ; do + [ $VERBOSE -gt 1 ] && echo "Checking package $p..." + if [ ! -d $GITROOT/$p.git ] ; then + echo "ERROR: Package module $p is invalid" >&2 + [ "$IGNORE" = "yes" ] && continue || exit -1 + fi + $(GIT_DIR=$GITROOT/$p.git git rev-parse -q --verify \ + $SRC_BRANCH >/dev/null) || \ + (echo "ERROR: Invalid source branch '$SRC_BRANCH' for package $p" >&2; \ + [ "$IGNORE" = "yes" ] && continue || exit -1) + $(GIT_DIR=$GITROOT/$p.git git rev-parse -q --verify \ + $BRANCH >/dev/null) && \ + (echo "IGNORING: Package module $p already has a branch $BRANCH" >&2; \ + [ "$IGNORE" = "yes" ] && continue || exit -1) + NEWP="$NEWP $p" +done +PACKAGES="$(echo $NEWP)" +if [ -z "$PACKAGES" ] ; then + echo "NOOP: no valid packages found to process" + exit -1 +fi + +if [ -n "$TEST" ] ; then + echo "Branch $BRANCH valid for $PACKAGES" + exit 0 +fi + +# This account must have the proper permissions as to not screw up the +# repository work. +if [ "$(id -un)" = "root" ] ; then + echo "Please run this script as yourself" + exit -3 +fi +#### Change this to check for proper git-admin rights + +# "global" permissions check +if [ ! -w $GITROOT ] ; then + echo "ERROR: You can not write to $GITROOT" + echo "ERROR: You can not perform branching operations" + exit -1 +fi + +# Now start working on creating those branches + +# For every module, "create" the branch +for NAME in $PACKAGES ; do + echo + echo "Creating new module branch '$BRANCH' for '$NAME' from branch '$SRC_BRANCH'..." + + # permissions checks for this particular module + if [ ! -w $GITROOT/$NAME.git/refs/heads/ ] ; then + echo "ERROR: You can not write to $d" + echo "ERROR: $NAME can not be branched by you" + continue + fi + #### Replace the above with a gitolite permission check + #[ $VERBOSE -gt 0 ] && echo "Creating $BRANCH-split tag for $NAME/$SRC_BRANCH..." + # Is the above needed? + #cvs -Q rtag -f "$BRANCH-split" $TOPLEVEL/$NAME/$SRC_BRANCH || { + #echo "ERROR: Branch split tag for $NAME/$SRC_BRANCH could not be created" >&2 + #exit -2 + #} + [ $VERBOSE -gt 0 ] && echo "Creating $NAME $BRANCH from $NAME $SRC_BRANCH..." + $(pushd $GITROOT/$NAME.git >/dev/null && \ + git branch --no-track $BRANCH $SRC_BRANCH && \ + popd >/dev/null) || { + echo "ERROR: Branch $NAME $BRANCH could not be created" >&2 + popd >/dev/null + exit -2 + } +done + +echo +echo "Done." diff --git a/scripts/distgit/pkgdb2branch.py b/scripts/distgit/pkgdb2branch.py new file mode 100755 index 0000000..dee7e4e --- /dev/null +++ b/scripts/distgit/pkgdb2branch.py @@ -0,0 +1,348 @@ +#!/usr/bin/python -t +# Author: Toshio Kuratomi +# Copyright: 2007-2008 Red Hat Software +# License: GPLv2 +# This needs a proper license and copyright here +__version__ = '0.3' + +import sys +import os +import optparse + +import subprocess + +from fedora.client import PackageDB, FedoraServiceError + +GITDIR='/srv/git/rpms' +BASEURL = os.environ.get('PACKAGEDBURL') or 'https://admin.fedoraproject.org/pkgdb/' +MKBRANCH='/usr/local/bin/mkbranch' +SETUP_PACKAGE='/usr/local/bin/setup_git_package' +BRANCHES = {'el4': 'master', 'el5': 'master', 'el6': 'f12', + 'olpc2': 'f7', + 'olpc3': 'f11', + 'master': None, + 'fc6': 'master', + 'f7': 'master', + 'f8': 'master', + 'f9': 'master', + 'f10': 'master', + 'f11': 'master', + 'f12': 'master', + 'f13': 'master', 'f14': 'master', + 'f15': 'master' + } + +# The branch names we get out of pkgdb have to be translated to git +GITBRANCHES = {'EL-4': 'el4', 'EL-5': 'el5', 'EL-6': 'el6', 'OLPC-2': 'olpc2', + 'FC-6': 'fc6', 'F-7': 'f7', 'F-8': 'f8', 'F-9': 'f9', + 'F-10': 'f10', 'OLPC-3': 'olpc3', + 'F-11': 'f11', 'F-12': 'f12', 'F-13': 'f13', 'f14': 'f14', 'f15': 'f15', + 'devel': 'master'} + +# The branch options we get from the CLI have to be translated to pkgdb +BRANCHBYGIT = dict([(v, k) for (k, v) in GITBRANCHES.iteritems()]) + +class InternalError(Exception): + pass + +class PackageDBError(InternalError): + pass + +class ProcessError(InternalError): + pass + +class ArgumentsError(InternalError): + pass + +class InvalidBranchError(PackageDBError): + pass + +class PackageDBClient(PackageDB): + def __init__(self, baseURL, cache=False, debug=False): + '''Initialize the connection. + + Args: + :baseURL: URL from which the packageDB is accessed + :cache: Whether to download a list of all vcs acls. + ''' + # We're only performing read operations so we don't need a username + super(PackageDBClient, self).__init__(baseURL, useragent=None, + debug=debug, insecure=True) + self.cacheEnabled = cache + self.__cache = None + self.branchCache = {} + + def _cache(self): + '''cache property which returns returns all the package acls. + ''' + if self.__cache: + return self.__cache + data = self.get_vcs_acls() + + if self.cacheEnabled: + self.__cache = data + else: + self.__cache = None + return self.__cache + cache = property(_cache) + + def get_package_branches(self, pkgname): + '''Return the branches to which a package belongs. + + Args: + :pkgname: The package to retrieve branch information about + ''' + if self.cacheEnabled: + # If the cache is enabled, the information is in the whole + # package dump + try: + return self.cache[pkgname].keys() + except KeyError: + raise PackageDBError('%s is not a known package' % pkgname) + + data = self.get_package_info(pkgname) + branches = [] + for packageListing in data.packageListings: + branches.append(packageListing['collection']['branchname']) + return branches + + def get_package_list(self, branchName): + '''Retrieve all the packages in a specific branch. + + Args: + :branchName: to return the packages for + ''' + ### FIXME: At some point we could enhance the server to filter by + # branchName + try: + # If branch is in the cache use that + return self.branchCache[branchName] + except KeyError: + pass + pkgList = [] + for pkg in self.cache: + # If the package has a branch record, we'll branch it + if branchName in self.cache[pkg]: + pkgList.append(pkg) + + if self.cacheEnabled: + self.branchCache[branchName] = pkgList + return pkgList + +class Brancher(object): + ''' Make branches in the GIT Repository.''' + + def __init__(self, pkgdburl, cache, verbose): + # Connect to the package database + self.verbose = verbose + self.client = PackageDBClient(BASEURL, cache=cache, debug=verbose) + + def _invoke(self, program, args): + '''Run a command and raise an exception if an error occurred. + + Args: + :program: The program to invoke + :args: List of arguments to pass to the program + + raises ProcessError if there's a problem. + ''' + cmdLine = [program] + cmdLine.extend(args) + print ' '.join(cmdLine) + + stdoutfd = subprocess.PIPE + if self.verbose: + program = subprocess.Popen(cmdLine, stderr=subprocess.STDOUT) + else: + program = subprocess.Popen(cmdLine, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + retCode = program.wait() + if retCode != 0: + e = ProcessError() + e.returnCode = retCode + e.cmd = ' '.join(cmdLine) + if self.verbose: + output = program.stdout.read() + e.message = 'Error, "%s" returned %s: %s' % (e.cmd, e.returnCode, output) + else: + e.message = 'Error, "%s" returned %s' % (e.cmd, e.returnCode) + raise e + + def _create_branch(self, pkgname, branch): + '''Create a specific branch for a package. + + Args: + :pkgname: Name of the package to branch + :branch: Name of the branch to create + + raises InvalidBranchError if a branchname is unknown. + + Will ignore a branch which is EOL. + ''' + try: + branchFrom = '%s/master' % BRANCHES[branch] + except KeyError: + raise InvalidBranchError( + 'PackageDB returned an invalid branch %s for %s' % + (branch, pkgname)) + + # Add the master to the branch + branch = '%s/master' % branch + # If branchFrom is None, this is an EOL release + # If the directory already exists, no need to invoke mkbranch + if branchFrom: + # Fall back to branching from master. + frombranchpath = os.path.join(GITDIR, '%s.git' % pkgname, + 'refs/heads', branchFrom) + if not os.path.exists(frombranchpath): + branchFrom = 'master' + + branchpath = os.path.join(GITDIR, '%s.git' % pkgname, + 'refs/heads', branch) + if not os.path.exists(branchpath): + try: + self._invoke(MKBRANCH, ['-s', branchFrom, branch, pkgname]) + except ProcessError, e: + if e.returnCode == 255: + # This is a warning, not an error + return + raise + + def branch_package(self, pkgname): + '''Create all the branches that are listed in the pkgdb for a package. + + Args: + :pkgname: The package to create branches for + + Note: this will ignore branches which are EOL. + + raises PackageDBError if the package is not present in the Package + Database. + ''' + # Retrieve branch information + try: + branches = self.client.get_package_branches(pkgname) + except FedoraServiceError, e: + raise PackageDBError( + 'Unable to retrieve information about %s: %s' % + (pkgname, str(e))) + + # Create the devel branch if necessary + if not os.path.exists(os.path.join(GITDIR, + '%s.git' % pkgname)): + self._invoke(SETUP_PACKAGE, [pkgname]) + # Create all the required branches for the package + # Use the translated branch name until pkgdb falls inline + for branch in branches: + if branch == 'devel': + continue + if not branch in GITBRANCHES.keys(): + print 'Skipping unknown branch %s' % branch + continue + self._create_branch(pkgname, GITBRANCHES[branch]) + + def mass_branch(self, branchName): + '''Make sure all packages listed for a specific branch in the PackageDB + have a CVS branch. + + Args: + :branchName: The branch to ensure. + ''' + # Retrieve all the packages in this branch + pkglist = self.client.get_package_list(branchName) + pkglist.sort() + for pkg in pkglist: + # Create a branch for this release for each of them + # Use the translated branch name until pkgdb falls inline + self._create_branch(pkg, GITBRANCHES[branchName]) + +def parse_commands(): + parser = optparse.OptionParser(version=__version__, usage='''pkgdb2branch.py [options] PACKAGENAME [packagename, ...] [-] + pkgdb2branch.py [options] --branchfor BRANCH + +pkgdb2branch reads package information from the packagedb and creates branches +on the git server based on what branches are listed there. pkgdb2branch can +read the list of packages from stdin if you specify '-' as an argument. + +pkgdb2branch has two modes of operation. In the first mode, you specify which +packages you want to branch. This mode is more efficient for a small number +of packages. + +In the second mode, pkgdb2branch will find every package lacking a BRANCH and +will create one if the pkgdb says it's needed. This mode is very efficient for +mass branching. This implies --cache-branches. + +For those with a moderate number of packages, using a list of packages and +--cache-branches may be fastest.''') + parser.add_option('-b', '--branch-for', + dest='branchFor', + action='store', + help='Make sure all the packages have been branched for BRANCHFOR. Implies -c.') + parser.add_option('-c', '--cache-branches', + dest='enableCache', + action='store_true', + help='Download a complete cache of packages') + parser.add_option('--verbose', + dest='verbose', + action='store_true', + help='Enable verbose output') + (opts, args) = parser.parse_args() + + if opts.branchFor: + if args: + raise ArgumentsError('Cannot specify packages with --branchFor') + opts.enableCache = True + + if '-' in args: + opts.fromStdin = True + del (args[args.index('-')]) + else: + opts.fromStdin = False + + if not (args or opts.fromStdin or opts.branchFor): + raise ArgumentsError('You must list packages to operate on') + + return opts, args + +if __name__ == '__main__': + try: + options, packages = parse_commands() + except ArgumentsError, e: + print e + sys.exit(1) + + unbranchedPackages = [] + brancher = Brancher(BASEURL, options.enableCache, options.verbose) + + if options.branchFor: + try: + unbranchedPackages = \ + brancher.mass_branch(BRANCHBYGIT[options.branchFor]) + except PackageDBError, e: + print 'Unable contact the PackageDB. Error: %s' % str(e) + sys.exit(1) + else: + # Process packages specified on the cmdline + for pkgname in packages: + try: + brancher.branch_package(pkgname) + except InternalError, e: + print str(e) + unbranchedPackages.append(pkgname) + + # Process packages from stdin + if options.fromStdin: + for pkgname in sys.stdin: + pkgname = pkgname.strip() + try: + brancher.branch_package(pkgname) + except InternalError, e: + print str(e) + unbranchedPackages.append(pkgname) + + if unbranchedPackages: + print 'The following packages were unbranched:' + print '\n'.join(unbranchedPackages) + sys.exit(100) + + sys.exit(0) diff --git a/scripts/distgit/setup_git_package b/scripts/distgit/setup_git_package new file mode 100755 index 0000000..4b438b2 --- /dev/null +++ b/scripts/distgit/setup_git_package @@ -0,0 +1,117 @@ +#!/bin/bash +# +# Create a new repo. +# THIS HAS TO BE RUN ON THE GIT SERVER! + +# WARNING: +# This file is maintained within puppet? +# All local changes will be lost. + + +# Figure out the environment we're running in +GITROOT=/srv/git/rpms + +# check if a moron is driving me +if [ ! -d $GITROOT ] ; then + # we're not on the git server (this check is fragile) + echo "ERROR: This script has to be run on the git server." + echo "ERROR: Homer sez 'Duh'." + exit -9 +fi + +# Local variables +VERBOSE=0 +TEST= +IGNORE= +AUTHOR="Fedora Release Engineering " +GIT_SSH_URL="ssh://localhost" + +Usage() { + cat < + + Creates a new repo for + +Options: + -h,--help This help message +EOF +} + +if [ $# -gt 2 ]; then + Usage + exit -1 +fi + +# parse the arguments +while [ -n "$1" ] ; do + case "$1" in + -h | --help ) + Usage + exit 0 + ;; + + * ) + PACKAGE="$1" + ;; + esac + shift +done + +# I hate shell scripting. I'm sure the above is totally wrong + +# check the arguments +if [ -z "$PACKAGE" ] ; then + Usage + exit -1 +fi + +# Sanity checks before we start doing damage +[ $VERBOSE -gt 1 ] && echo "Checking package $PACKAGE..." +if [ -d $GITROOT/$PACKAGE.git ] ; then + echo "ERROR: Package module $PACKAGE already exists!" >&2 + exit -1 +fi + +# Just don't run as root, mmkey? +if [ "$(id -un)" = "root" ] ; then + echo "Please run this script as yourself" + exit -3 +fi + +# "global" permissions check +if [ ! -w $GITROOT ] ; then + echo "ERROR: You can not write to $GITROOT" + echo "ERROR: You can not create repos" + exit -1 +fi + +# Now start working on creating those branches +# Create a tmpdir to do some git work in +TMPDIR=$(mktemp -d /tmp/tmpXXXXXX) + +# First create the master repo +mkdir $GITROOT/$PACKAGE.git +pushd $GITROOT/$PACKAGE.git >/dev/null +git init -q --shared --bare +echo "$PACKAGE" > description # This is used to figure out who to send mail to. +git config --add hooks.mailinglist "$PACKAGE-owner@fedoraproject.org,scm-commits@lists.fedoraproject.org" +git config --add hooks.maildomain fedoraproject.org +popd >/dev/null + +# Now clone that repo and create the .gitignore and sources file +git clone -q /srv/git/rpms/$PACKAGE.git $TMPDIR/$PACKAGE +pushd $TMPDIR/$PACKAGE >/dev/null +touch .gitignore sources +git add . +git commit -q -m 'Initial setup of the repo' --author "$AUTHOR" +git push -q origin master +popd >/dev/null + +# Put our special update hooks in place +ln -s /usr/share/gitolite/hooks/common/update $GITROOT/$PACKAGE.git/hooks/ +ln -s /usr/share/git-core/mail-hooks/gnome-post-receive-email \ + $GITROOT/$PACKAGE.git/hooks/post-receive + +rm -rf $TMPDIR +echo "Done."