Add various distgit scripts

scripts/distgit/mkbranch

@@ -0,0 +1,183 @@
+#!/bin/bash
+#
+# Create a new development branch for a module.
+# THIS HAS TO BE RUN ON THE GIT SERVER!
+
+# WARNING:
+# This file is maintained within puppet?
+# All local changes will be lost.
+
+
+# Figure out the environment we're running in
+RUNDIR=$(cd $(dirname $0) && pwd)
+GITROOT=/srv/git/rpms
+
+# check if a moron is driving me
+if [ ! -d $GITROOT ] ; then
+    # we're not on the git server (this check is fragile)
+    echo "ERROR: This script has to be run on the git server."
+    echo "ERROR: Homer sez 'Duh'."
+    exit -9
+fi
+
+# where are the packages kept
+TOPLEVEL=rpms
+
+# Local variables
+VERBOSE=0
+TEST=
+IGNORE=
+BRANCH=""
+PACKAGES=""
+SRC_BRANCH="master"
+AUTHOR="Fedora Release Engineering <rel-eng@lists.fedoraproject.org>"
+
+Usage() {
+    cat <<EOF
+Usage:
+    $0 [ -s <src_branch>] <branch> <package_name>...
+
+    Creates a new branch <branch> for the list of <package_name>s.
+    The /master suffix on branch names is assumed.
+
+Options:
+    -s,--source=<src_branch>	Use <src_branch> as the source branch.
+				Defaults is master
+                                /master suffix on other branches assumed
+    -n,--test			Don't do nothing, only test
+    -i,--ignore                 Ignore erroneous modules
+    -h,--help			This help message
+    -v,--verbose		Increase verbosity
+EOF
+}
+
+# parse the arguments
+while [ -n "$1" ] ; do
+    case "$1" in
+	-h | --help )
+	    Usage
+	    exit 0
+	    ;;
+
+	-v | --verbose )
+	    VERBOSE=$(($VERBOSE + 1))
+	    ;;
+
+	-i | --ignore )
+	    IGNORE="yes"
+	    ;;
+
+	-n | --test )
+	    TEST="yes"
+	    ;;
+
+	-s | --source )
+	    shift
+	    SRC_BRANCH=$1
+	    ;;
+
+	-b | --branch )
+	    shift
+	    BRANCH=$1/master
+	    ;;
+
+	* )
+	    if [ -z "$BRANCH" ] ; then
+		BRANCH="$1"
+	    else
+		PACKAGES="$PACKAGES $1"
+	    fi
+	    ;;
+    esac
+    shift
+done
+
+# check the arguments
+if [ -z "$BRANCH" -o -z "$PACKAGES" ] ; then
+    Usage
+    exit -1
+fi
+
+# Translate the source branch
+if [ $SRC_BRANCH == 'master' ] ; then
+    echo > /dev/null
+else
+    $SRC_BRANCH = $SRC_BRANCH/master
+fi
+
+# Sanity checks before we start doing damage
+NEWP=
+for p in $PACKAGES ; do
+    [ $VERBOSE -gt 1 ] && echo "Checking package $p..."
+    if [ ! -d $GITROOT/$p.git ] ; then
+	echo "ERROR: Package module $p is invalid" >&2
+	[ "$IGNORE" = "yes" ] && continue || exit -1
+    fi
+    $(GIT_DIR=$GITROOT/$p.git git rev-parse -q --verify \
+      $SRC_BRANCH >/dev/null) || \
+	(echo "ERROR: Invalid source branch '$SRC_BRANCH' for package $p" >&2; \
+	[ "$IGNORE" = "yes" ] && continue || exit -1)
+    $(GIT_DIR=$GITROOT/$p.git git rev-parse -q --verify \
+      $BRANCH >/dev/null) && \
+	(echo "IGNORING: Package module $p already has a branch $BRANCH" >&2; \
+	[ "$IGNORE" = "yes" ] && continue || exit -1)
+    NEWP="$NEWP $p"
+done
+PACKAGES="$(echo $NEWP)"
+if [ -z "$PACKAGES" ] ; then
+    echo "NOOP: no valid packages found to process"
+    exit -1
+fi
+
+if [ -n "$TEST" ] ; then
+    echo "Branch $BRANCH valid for $PACKAGES"
+    exit 0
+fi
+
+# This account must have the proper permissions as to not screw up the
+# repository work.
+if [ "$(id -un)" = "root" ] ; then
+    echo "Please run this script as yourself"
+    exit -3
+fi
+#### Change this to check for proper git-admin rights
+
+# "global" permissions check
+if [ ! -w $GITROOT ] ; then
+    echo "ERROR: You can not write to $GITROOT"
+    echo "ERROR: You can not perform branching operations"
+    exit -1
+fi
+
+# Now start working on creating those branches
+
+# For every module, "create" the branch
+for NAME in $PACKAGES ; do
+    echo
+    echo "Creating new module branch '$BRANCH' for '$NAME' from branch '$SRC_BRANCH'..."
+
+    # permissions checks for this particular module
+    if [ ! -w $GITROOT/$NAME.git/refs/heads/  ] ; then
+        echo "ERROR: You can not write to $d"
+        echo "ERROR: $NAME can not be branched by you"
+        continue
+    fi
+    ####  Replace the above with a gitolite permission check
+    #[ $VERBOSE -gt 0 ] && echo "Creating $BRANCH-split tag for $NAME/$SRC_BRANCH..."
+    # Is the above needed?
+    #cvs -Q rtag -f "$BRANCH-split" $TOPLEVEL/$NAME/$SRC_BRANCH || {
+	#echo "ERROR: Branch split tag for $NAME/$SRC_BRANCH could not be created" >&2
+	#exit -2
+    #}
+    [ $VERBOSE -gt 0 ] && echo "Creating $NAME $BRANCH from $NAME $SRC_BRANCH..."
+    $(pushd $GITROOT/$NAME.git >/dev/null && \
+    git branch --no-track $BRANCH $SRC_BRANCH && \
+    popd >/dev/null) || {
+	echo "ERROR: Branch $NAME $BRANCH could not be created" >&2
+        popd >/dev/null
+	exit -2
+    }
+done
+
+echo
+echo "Done."

scripts/distgit/pkgdb2branch.py

@@ -0,0 +1,348 @@
+#!/usr/bin/python -t
+# Author: Toshio Kuratomi
+# Copyright: 2007-2008 Red Hat Software
+# License: GPLv2
+# This needs a proper license and copyright here
+__version__ = '0.3'
+
+import sys
+import os
+import optparse
+
+import subprocess
+
+from fedora.client import PackageDB, FedoraServiceError
+
+GITDIR='/srv/git/rpms'
+BASEURL = os.environ.get('PACKAGEDBURL') or 'https://admin.fedoraproject.org/pkgdb/'
+MKBRANCH='/usr/local/bin/mkbranch'
+SETUP_PACKAGE='/usr/local/bin/setup_git_package'
+BRANCHES = {'el4': 'master', 'el5': 'master', 'el6': 'f12',
+        'olpc2': 'f7',
+        'olpc3': 'f11',
+        'master': None,
+        'fc6': 'master',
+        'f7': 'master',
+        'f8': 'master',
+        'f9': 'master',
+        'f10': 'master',
+        'f11': 'master',
+        'f12': 'master',
+        'f13': 'master', 'f14': 'master',
+        'f15': 'master'
+        }
+
+# The branch names we get out of pkgdb have to be translated to git
+GITBRANCHES = {'EL-4': 'el4', 'EL-5': 'el5', 'EL-6': 'el6', 'OLPC-2': 'olpc2',
+               'FC-6': 'fc6', 'F-7': 'f7', 'F-8': 'f8', 'F-9': 'f9',
+               'F-10': 'f10', 'OLPC-3': 'olpc3',
+               'F-11': 'f11', 'F-12': 'f12', 'F-13': 'f13', 'f14': 'f14', 'f15': 'f15',
+               'devel': 'master'}
+
+# The branch options we get from the CLI have to be translated to pkgdb
+BRANCHBYGIT = dict([(v, k) for (k, v) in GITBRANCHES.iteritems()])
+
+class InternalError(Exception):
+    pass
+
+class PackageDBError(InternalError):
+    pass
+
+class ProcessError(InternalError):
+    pass
+
+class ArgumentsError(InternalError):
+    pass
+
+class InvalidBranchError(PackageDBError):
+    pass
+
+class PackageDBClient(PackageDB):
+    def __init__(self, baseURL, cache=False, debug=False):
+        '''Initialize the connection.
+
+        Args:
+        :baseURL: URL from which the packageDB is accessed
+        :cache: Whether to download a list of all vcs acls.
+        '''
+        # We're only performing read operations so we don't need a username
+        super(PackageDBClient, self).__init__(baseURL, useragent=None,
+            debug=debug, insecure=True)
+        self.cacheEnabled = cache
+        self.__cache = None
+        self.branchCache = {}
+
+    def _cache(self):
+        '''cache property which returns returns all the package acls.
+        '''
+        if self.__cache:
+            return self.__cache
+        data = self.get_vcs_acls()
+        
+        if self.cacheEnabled:
+            self.__cache = data
+        else:
+            self.__cache = None
+        return self.__cache
+    cache = property(_cache)
+        
+    def get_package_branches(self, pkgname):
+        '''Return the branches to which a package belongs.
+
+        Args:
+        :pkgname: The package to retrieve branch information about
+        '''
+        if self.cacheEnabled:
+            # If the cache is enabled, the information is in the whole
+            # package dump
+            try:
+                return self.cache[pkgname].keys()
+            except KeyError:
+                raise PackageDBError('%s is not a known package' % pkgname)
+            
+        data = self.get_package_info(pkgname)
+        branches = []
+        for packageListing in data.packageListings:
+            branches.append(packageListing['collection']['branchname'])
+        return branches
+        
+    def get_package_list(self, branchName):
+        '''Retrieve all the packages in a specific branch.
+
+        Args:
+        :branchName: to return the packages for
+        '''
+        ### FIXME: At some point we could enhance the server to filter by
+        # branchName
+        try:
+            # If branch is in the cache use that
+            return self.branchCache[branchName]
+        except KeyError:
+            pass
+        pkgList = []
+        for pkg in self.cache:
+            # If the package has a branch record, we'll branch it
+            if branchName in self.cache[pkg]:
+                pkgList.append(pkg)
+
+        if self.cacheEnabled:
+            self.branchCache[branchName] = pkgList
+        return pkgList
+
+class Brancher(object):
+    ''' Make branches in the GIT Repository.'''
+
+    def __init__(self, pkgdburl, cache, verbose):
+        # Connect to the package database
+        self.verbose = verbose
+        self.client = PackageDBClient(BASEURL, cache=cache, debug=verbose)
+
+    def _invoke(self, program, args):
+        '''Run a command and raise an exception if an error occurred.
+        
+        Args:
+        :program: The program to invoke
+        :args: List of arguments to pass to the program
+
+        raises ProcessError if there's a problem.
+        '''
+        cmdLine = [program]
+        cmdLine.extend(args)
+        print ' '.join(cmdLine)
+
+        stdoutfd = subprocess.PIPE
+        if self.verbose:
+            program = subprocess.Popen(cmdLine, stderr=subprocess.STDOUT)
+        else:
+            program = subprocess.Popen(cmdLine, stdout=subprocess.PIPE,
+                     stderr=subprocess.STDOUT)
+        retCode = program.wait()
+        if retCode != 0:
+            e = ProcessError()
+            e.returnCode = retCode
+            e.cmd = ' '.join(cmdLine)
+            if self.verbose:
+                output = program.stdout.read()
+                e.message = 'Error, "%s" returned %s: %s' % (e.cmd, e.returnCode, output)
+            else:
+                e.message = 'Error, "%s" returned %s' % (e.cmd, e.returnCode)
+            raise e
+
+    def _create_branch(self, pkgname, branch):
+        '''Create a specific branch for a package.
+        
+        Args:
+        :pkgname: Name of the package to branch
+        :branch: Name of the branch to create
+
+        raises InvalidBranchError if a branchname is unknown.
+
+        Will ignore a branch which is EOL.
+        '''
+        try:
+            branchFrom = '%s/master' % BRANCHES[branch]
+        except KeyError:
+            raise InvalidBranchError(
+                    'PackageDB returned an invalid branch %s for %s' %
+                    (branch, pkgname))
+
+        # Add the master to the branch
+        branch = '%s/master' % branch
+        # If branchFrom is None, this is an EOL release
+        # If the directory already exists, no need to invoke mkbranch
+        if branchFrom:
+            # Fall back to branching from master.
+            frombranchpath = os.path.join(GITDIR, '%s.git' % pkgname,
+                                          'refs/heads', branchFrom)
+            if not os.path.exists(frombranchpath):
+                branchFrom = 'master'
+
+            branchpath = os.path.join(GITDIR, '%s.git' % pkgname,
+                                      'refs/heads', branch)
+            if not os.path.exists(branchpath):
+                try:
+                    self._invoke(MKBRANCH, ['-s', branchFrom, branch, pkgname])
+                except ProcessError, e:
+                    if e.returnCode == 255:
+                        # This is a warning, not an error
+                        return
+                    raise
+
+    def branch_package(self, pkgname):
+        '''Create all the branches that are listed in the pkgdb for a package.
+
+        Args:
+        :pkgname: The package to create branches for
+
+        Note: this will ignore branches which are EOL.
+
+        raises PackageDBError if the package is not present in the Package
+        Database.
+        '''
+        # Retrieve branch information
+        try:
+            branches = self.client.get_package_branches(pkgname)
+        except FedoraServiceError, e:
+            raise PackageDBError(
+                    'Unable to retrieve information about %s: %s' %
+                    (pkgname, str(e)))
+
+        # Create the devel branch if necessary
+        if not os.path.exists(os.path.join(GITDIR,
+                                           '%s.git' % pkgname)):
+            self._invoke(SETUP_PACKAGE, [pkgname])
+        # Create all the required branches for the package
+        # Use the translated branch name until pkgdb falls inline
+        for branch in branches:
+            if branch == 'devel':
+                continue
+            if not branch in GITBRANCHES.keys():
+                print 'Skipping unknown branch %s' % branch
+                continue
+            self._create_branch(pkgname, GITBRANCHES[branch])
+
+    def mass_branch(self, branchName):
+        '''Make sure all packages listed for a specific branch in the PackageDB
+        have a CVS branch.
+        
+        Args:
+        :branchName: The branch to ensure.
+        '''
+        # Retrieve all the packages in this branch
+        pkglist = self.client.get_package_list(branchName)
+        pkglist.sort()
+        for pkg in pkglist:
+            # Create a branch for this release for each of them
+            # Use the translated branch name until pkgdb falls inline
+            self._create_branch(pkg, GITBRANCHES[branchName])
+        
+def parse_commands():
+    parser = optparse.OptionParser(version=__version__, usage='''pkgdb2branch.py [options] PACKAGENAME [packagename, ...] [-]
+       pkgdb2branch.py [options] --branchfor BRANCH
+
+pkgdb2branch reads package information from the packagedb and creates branches
+on the git server based on what branches are listed there.  pkgdb2branch can
+read the list of packages from stdin if you specify '-' as an argument.
+
+pkgdb2branch has two modes of operation.  In the first mode, you specify which
+packages you want to branch.  This mode is more efficient for a small number
+of packages.
+
+In the second mode, pkgdb2branch will find every package lacking a BRANCH and
+will create one if the pkgdb says it's needed.  This mode is very efficient for
+mass branching.  This implies --cache-branches.
+
+For those with a moderate number of packages, using a list of packages and
+--cache-branches may be fastest.''')
+    parser.add_option('-b', '--branch-for',
+        dest='branchFor',
+        action='store',
+        help='Make sure all the packages have been branched for BRANCHFOR. Implies -c.')
+    parser.add_option('-c', '--cache-branches',
+        dest='enableCache',
+        action='store_true',
+        help='Download a complete cache of packages')
+    parser.add_option('--verbose',
+        dest='verbose',
+        action='store_true',
+        help='Enable verbose output')
+    (opts, args) = parser.parse_args()
+
+    if opts.branchFor:
+        if args:
+            raise ArgumentsError('Cannot specify packages with --branchFor')
+        opts.enableCache = True
+
+    if '-' in args:
+        opts.fromStdin = True
+        del (args[args.index('-')])
+    else:
+        opts.fromStdin = False
+
+    if not (args or opts.fromStdin or opts.branchFor):
+        raise ArgumentsError('You must list packages to operate on')
+
+    return opts, args
+    
+if __name__ == '__main__':
+    try:
+        options, packages = parse_commands()
+    except ArgumentsError, e:
+        print e
+        sys.exit(1)
+
+    unbranchedPackages = []
+    brancher = Brancher(BASEURL, options.enableCache, options.verbose)
+
+    if options.branchFor:
+        try:
+            unbranchedPackages = \
+                        brancher.mass_branch(BRANCHBYGIT[options.branchFor])
+        except PackageDBError, e:
+            print 'Unable contact the PackageDB. Error: %s' % str(e)
+            sys.exit(1)
+    else:
+        # Process packages specified on the cmdline
+        for pkgname in packages:
+            try:
+                brancher.branch_package(pkgname)
+            except InternalError, e:
+                print str(e)
+                unbranchedPackages.append(pkgname)
+
+        # Process packages from stdin
+        if options.fromStdin:
+            for pkgname in sys.stdin:
+                pkgname = pkgname.strip()
+                try:
+                    brancher.branch_package(pkgname)
+                except InternalError, e:
+                    print str(e)
+                    unbranchedPackages.append(pkgname)
+        
+    if unbranchedPackages:
+        print 'The following packages were unbranched:'
+        print '\n'.join(unbranchedPackages)
+        sys.exit(100)
+
+    sys.exit(0)

scripts/distgit/setup_git_package

@@ -0,0 +1,117 @@
+#!/bin/bash
+#
+# Create a new repo.
+# THIS HAS TO BE RUN ON THE GIT SERVER!
+
+# WARNING:
+# This file is maintained within puppet?
+# All local changes will be lost.
+
+
+# Figure out the environment we're running in
+GITROOT=/srv/git/rpms
+
+# check if a moron is driving me
+if [ ! -d $GITROOT ] ; then
+    # we're not on the git server (this check is fragile)
+    echo "ERROR: This script has to be run on the git server."
+    echo "ERROR: Homer sez 'Duh'."
+    exit -9
+fi
+
+# Local variables
+VERBOSE=0
+TEST=
+IGNORE=
+AUTHOR="Fedora Release Engineering <rel-eng@lists.fedoraproject.org>"
+GIT_SSH_URL="ssh://localhost"
+
+Usage() {
+    cat <<EOF
+Usage:
+    $0 <package_name>
+
+    Creates a new repo for <package_name>
+
+Options:
+    -h,--help			This help message
+EOF
+}
+
+if [ $# -gt 2 ]; then
+    Usage
+    exit -1
+fi
+
+# parse the arguments
+while [ -n "$1" ] ; do
+    case "$1" in
+	-h | --help )
+	    Usage
+	    exit 0
+	    ;;
+
+	* )
+	    PACKAGE="$1"
+	    ;;
+    esac
+    shift
+done
+
+# I hate shell scripting.  I'm sure the above is totally wrong
+
+# check the arguments
+if [ -z "$PACKAGE" ] ; then
+    Usage
+    exit -1
+fi
+
+# Sanity checks before we start doing damage
+[ $VERBOSE -gt 1 ] && echo "Checking package $PACKAGE..."
+if [ -d $GITROOT/$PACKAGE.git ] ; then
+    echo "ERROR: Package module $PACKAGE already exists!" >&2
+    exit -1
+fi
+
+# Just don't run as root, mmkey?
+if [ "$(id -un)" = "root" ] ; then
+    echo "Please run this script as yourself"
+    exit -3
+fi
+
+# "global" permissions check
+if [ ! -w $GITROOT ] ; then
+    echo "ERROR: You can not write to $GITROOT"
+    echo "ERROR: You can not create repos"
+    exit -1
+fi
+
+# Now start working on creating those branches
+# Create a tmpdir to do some git work in
+TMPDIR=$(mktemp -d /tmp/tmpXXXXXX)
+
+# First create the master repo
+mkdir $GITROOT/$PACKAGE.git
+pushd $GITROOT/$PACKAGE.git >/dev/null
+git init -q --shared --bare
+echo "$PACKAGE" > description # This is used to figure out who to send mail to.
+git config --add hooks.mailinglist "$PACKAGE-owner@fedoraproject.org,scm-commits@lists.fedoraproject.org"
+git config --add hooks.maildomain fedoraproject.org
+popd >/dev/null
+
+# Now clone that repo and create the .gitignore and sources file
+git clone -q /srv/git/rpms/$PACKAGE.git $TMPDIR/$PACKAGE
+pushd $TMPDIR/$PACKAGE >/dev/null
+touch .gitignore sources
+git add .
+git commit -q -m 'Initial setup of the repo' --author "$AUTHOR"
+git push -q origin master
+popd >/dev/null
+
+# Put our special update hooks in place
+ln -s /usr/share/gitolite/hooks/common/update $GITROOT/$PACKAGE.git/hooks/
+ln -s /usr/share/git-core/mail-hooks/gnome-post-receive-email \
+    $GITROOT/$PACKAGE.git/hooks/post-receive
+
+rm -rf $TMPDIR
+echo "Done."