Add various distgit scripts

This commit is contained in:
Jon 2011-05-08 13:45:07 -04:00
parent 9965e2d3d1
commit d1e5c04b7d
3 changed files with 648 additions and 0 deletions

183
scripts/distgit/mkbranch Executable file
View file

@ -0,0 +1,183 @@
#!/bin/bash
#
# Create a new development branch for a module.
# THIS HAS TO BE RUN ON THE GIT SERVER!
# WARNING:
# This file is maintained within puppet?
# All local changes will be lost.
# Figure out the environment we're running in
RUNDIR=$(cd $(dirname $0) && pwd)
GITROOT=/srv/git/rpms
# check if a moron is driving me
if [ ! -d $GITROOT ] ; then
# we're not on the git server (this check is fragile)
echo "ERROR: This script has to be run on the git server."
echo "ERROR: Homer sez 'Duh'."
exit -9
fi
# where are the packages kept
TOPLEVEL=rpms
# Local variables
VERBOSE=0
TEST=
IGNORE=
BRANCH=""
PACKAGES=""
SRC_BRANCH="master"
AUTHOR="Fedora Release Engineering <rel-eng@lists.fedoraproject.org>"
Usage() {
cat <<EOF
Usage:
$0 [ -s <src_branch>] <branch> <package_name>...
Creates a new branch <branch> for the list of <package_name>s.
The /master suffix on branch names is assumed.
Options:
-s,--source=<src_branch> Use <src_branch> as the source branch.
Defaults is master
/master suffix on other branches assumed
-n,--test Don't do nothing, only test
-i,--ignore Ignore erroneous modules
-h,--help This help message
-v,--verbose Increase verbosity
EOF
}
# parse the arguments
while [ -n "$1" ] ; do
case "$1" in
-h | --help )
Usage
exit 0
;;
-v | --verbose )
VERBOSE=$(($VERBOSE + 1))
;;
-i | --ignore )
IGNORE="yes"
;;
-n | --test )
TEST="yes"
;;
-s | --source )
shift
SRC_BRANCH=$1
;;
-b | --branch )
shift
BRANCH=$1/master
;;
* )
if [ -z "$BRANCH" ] ; then
BRANCH="$1"
else
PACKAGES="$PACKAGES $1"
fi
;;
esac
shift
done
# check the arguments
if [ -z "$BRANCH" -o -z "$PACKAGES" ] ; then
Usage
exit -1
fi
# Translate the source branch
if [ $SRC_BRANCH == 'master' ] ; then
echo > /dev/null
else
$SRC_BRANCH = $SRC_BRANCH/master
fi
# Sanity checks before we start doing damage
NEWP=
for p in $PACKAGES ; do
[ $VERBOSE -gt 1 ] && echo "Checking package $p..."
if [ ! -d $GITROOT/$p.git ] ; then
echo "ERROR: Package module $p is invalid" >&2
[ "$IGNORE" = "yes" ] && continue || exit -1
fi
$(GIT_DIR=$GITROOT/$p.git git rev-parse -q --verify \
$SRC_BRANCH >/dev/null) || \
(echo "ERROR: Invalid source branch '$SRC_BRANCH' for package $p" >&2; \
[ "$IGNORE" = "yes" ] && continue || exit -1)
$(GIT_DIR=$GITROOT/$p.git git rev-parse -q --verify \
$BRANCH >/dev/null) && \
(echo "IGNORING: Package module $p already has a branch $BRANCH" >&2; \
[ "$IGNORE" = "yes" ] && continue || exit -1)
NEWP="$NEWP $p"
done
PACKAGES="$(echo $NEWP)"
if [ -z "$PACKAGES" ] ; then
echo "NOOP: no valid packages found to process"
exit -1
fi
if [ -n "$TEST" ] ; then
echo "Branch $BRANCH valid for $PACKAGES"
exit 0
fi
# This account must have the proper permissions as to not screw up the
# repository work.
if [ "$(id -un)" = "root" ] ; then
echo "Please run this script as yourself"
exit -3
fi
#### Change this to check for proper git-admin rights
# "global" permissions check
if [ ! -w $GITROOT ] ; then
echo "ERROR: You can not write to $GITROOT"
echo "ERROR: You can not perform branching operations"
exit -1
fi
# Now start working on creating those branches
# For every module, "create" the branch
for NAME in $PACKAGES ; do
echo
echo "Creating new module branch '$BRANCH' for '$NAME' from branch '$SRC_BRANCH'..."
# permissions checks for this particular module
if [ ! -w $GITROOT/$NAME.git/refs/heads/ ] ; then
echo "ERROR: You can not write to $d"
echo "ERROR: $NAME can not be branched by you"
continue
fi
#### Replace the above with a gitolite permission check
#[ $VERBOSE -gt 0 ] && echo "Creating $BRANCH-split tag for $NAME/$SRC_BRANCH..."
# Is the above needed?
#cvs -Q rtag -f "$BRANCH-split" $TOPLEVEL/$NAME/$SRC_BRANCH || {
#echo "ERROR: Branch split tag for $NAME/$SRC_BRANCH could not be created" >&2
#exit -2
#}
[ $VERBOSE -gt 0 ] && echo "Creating $NAME $BRANCH from $NAME $SRC_BRANCH..."
$(pushd $GITROOT/$NAME.git >/dev/null && \
git branch --no-track $BRANCH $SRC_BRANCH && \
popd >/dev/null) || {
echo "ERROR: Branch $NAME $BRANCH could not be created" >&2
popd >/dev/null
exit -2
}
done
echo
echo "Done."

348
scripts/distgit/pkgdb2branch.py Executable file
View file

@ -0,0 +1,348 @@
#!/usr/bin/python -t
# Author: Toshio Kuratomi
# Copyright: 2007-2008 Red Hat Software
# License: GPLv2
# This needs a proper license and copyright here
__version__ = '0.3'
import sys
import os
import optparse
import subprocess
from fedora.client import PackageDB, FedoraServiceError
GITDIR='/srv/git/rpms'
BASEURL = os.environ.get('PACKAGEDBURL') or 'https://admin.fedoraproject.org/pkgdb/'
MKBRANCH='/usr/local/bin/mkbranch'
SETUP_PACKAGE='/usr/local/bin/setup_git_package'
BRANCHES = {'el4': 'master', 'el5': 'master', 'el6': 'f12',
'olpc2': 'f7',
'olpc3': 'f11',
'master': None,
'fc6': 'master',
'f7': 'master',
'f8': 'master',
'f9': 'master',
'f10': 'master',
'f11': 'master',
'f12': 'master',
'f13': 'master', 'f14': 'master',
'f15': 'master'
}
# The branch names we get out of pkgdb have to be translated to git
GITBRANCHES = {'EL-4': 'el4', 'EL-5': 'el5', 'EL-6': 'el6', 'OLPC-2': 'olpc2',
'FC-6': 'fc6', 'F-7': 'f7', 'F-8': 'f8', 'F-9': 'f9',
'F-10': 'f10', 'OLPC-3': 'olpc3',
'F-11': 'f11', 'F-12': 'f12', 'F-13': 'f13', 'f14': 'f14', 'f15': 'f15',
'devel': 'master'}
# The branch options we get from the CLI have to be translated to pkgdb
BRANCHBYGIT = dict([(v, k) for (k, v) in GITBRANCHES.iteritems()])
class InternalError(Exception):
pass
class PackageDBError(InternalError):
pass
class ProcessError(InternalError):
pass
class ArgumentsError(InternalError):
pass
class InvalidBranchError(PackageDBError):
pass
class PackageDBClient(PackageDB):
def __init__(self, baseURL, cache=False, debug=False):
'''Initialize the connection.
Args:
:baseURL: URL from which the packageDB is accessed
:cache: Whether to download a list of all vcs acls.
'''
# We're only performing read operations so we don't need a username
super(PackageDBClient, self).__init__(baseURL, useragent=None,
debug=debug, insecure=True)
self.cacheEnabled = cache
self.__cache = None
self.branchCache = {}
def _cache(self):
'''cache property which returns returns all the package acls.
'''
if self.__cache:
return self.__cache
data = self.get_vcs_acls()
if self.cacheEnabled:
self.__cache = data
else:
self.__cache = None
return self.__cache
cache = property(_cache)
def get_package_branches(self, pkgname):
'''Return the branches to which a package belongs.
Args:
:pkgname: The package to retrieve branch information about
'''
if self.cacheEnabled:
# If the cache is enabled, the information is in the whole
# package dump
try:
return self.cache[pkgname].keys()
except KeyError:
raise PackageDBError('%s is not a known package' % pkgname)
data = self.get_package_info(pkgname)
branches = []
for packageListing in data.packageListings:
branches.append(packageListing['collection']['branchname'])
return branches
def get_package_list(self, branchName):
'''Retrieve all the packages in a specific branch.
Args:
:branchName: to return the packages for
'''
### FIXME: At some point we could enhance the server to filter by
# branchName
try:
# If branch is in the cache use that
return self.branchCache[branchName]
except KeyError:
pass
pkgList = []
for pkg in self.cache:
# If the package has a branch record, we'll branch it
if branchName in self.cache[pkg]:
pkgList.append(pkg)
if self.cacheEnabled:
self.branchCache[branchName] = pkgList
return pkgList
class Brancher(object):
''' Make branches in the GIT Repository.'''
def __init__(self, pkgdburl, cache, verbose):
# Connect to the package database
self.verbose = verbose
self.client = PackageDBClient(BASEURL, cache=cache, debug=verbose)
def _invoke(self, program, args):
'''Run a command and raise an exception if an error occurred.
Args:
:program: The program to invoke
:args: List of arguments to pass to the program
raises ProcessError if there's a problem.
'''
cmdLine = [program]
cmdLine.extend(args)
print ' '.join(cmdLine)
stdoutfd = subprocess.PIPE
if self.verbose:
program = subprocess.Popen(cmdLine, stderr=subprocess.STDOUT)
else:
program = subprocess.Popen(cmdLine, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
retCode = program.wait()
if retCode != 0:
e = ProcessError()
e.returnCode = retCode
e.cmd = ' '.join(cmdLine)
if self.verbose:
output = program.stdout.read()
e.message = 'Error, "%s" returned %s: %s' % (e.cmd, e.returnCode, output)
else:
e.message = 'Error, "%s" returned %s' % (e.cmd, e.returnCode)
raise e
def _create_branch(self, pkgname, branch):
'''Create a specific branch for a package.
Args:
:pkgname: Name of the package to branch
:branch: Name of the branch to create
raises InvalidBranchError if a branchname is unknown.
Will ignore a branch which is EOL.
'''
try:
branchFrom = '%s/master' % BRANCHES[branch]
except KeyError:
raise InvalidBranchError(
'PackageDB returned an invalid branch %s for %s' %
(branch, pkgname))
# Add the master to the branch
branch = '%s/master' % branch
# If branchFrom is None, this is an EOL release
# If the directory already exists, no need to invoke mkbranch
if branchFrom:
# Fall back to branching from master.
frombranchpath = os.path.join(GITDIR, '%s.git' % pkgname,
'refs/heads', branchFrom)
if not os.path.exists(frombranchpath):
branchFrom = 'master'
branchpath = os.path.join(GITDIR, '%s.git' % pkgname,
'refs/heads', branch)
if not os.path.exists(branchpath):
try:
self._invoke(MKBRANCH, ['-s', branchFrom, branch, pkgname])
except ProcessError, e:
if e.returnCode == 255:
# This is a warning, not an error
return
raise
def branch_package(self, pkgname):
'''Create all the branches that are listed in the pkgdb for a package.
Args:
:pkgname: The package to create branches for
Note: this will ignore branches which are EOL.
raises PackageDBError if the package is not present in the Package
Database.
'''
# Retrieve branch information
try:
branches = self.client.get_package_branches(pkgname)
except FedoraServiceError, e:
raise PackageDBError(
'Unable to retrieve information about %s: %s' %
(pkgname, str(e)))
# Create the devel branch if necessary
if not os.path.exists(os.path.join(GITDIR,
'%s.git' % pkgname)):
self._invoke(SETUP_PACKAGE, [pkgname])
# Create all the required branches for the package
# Use the translated branch name until pkgdb falls inline
for branch in branches:
if branch == 'devel':
continue
if not branch in GITBRANCHES.keys():
print 'Skipping unknown branch %s' % branch
continue
self._create_branch(pkgname, GITBRANCHES[branch])
def mass_branch(self, branchName):
'''Make sure all packages listed for a specific branch in the PackageDB
have a CVS branch.
Args:
:branchName: The branch to ensure.
'''
# Retrieve all the packages in this branch
pkglist = self.client.get_package_list(branchName)
pkglist.sort()
for pkg in pkglist:
# Create a branch for this release for each of them
# Use the translated branch name until pkgdb falls inline
self._create_branch(pkg, GITBRANCHES[branchName])
def parse_commands():
parser = optparse.OptionParser(version=__version__, usage='''pkgdb2branch.py [options] PACKAGENAME [packagename, ...] [-]
pkgdb2branch.py [options] --branchfor BRANCH
pkgdb2branch reads package information from the packagedb and creates branches
on the git server based on what branches are listed there. pkgdb2branch can
read the list of packages from stdin if you specify '-' as an argument.
pkgdb2branch has two modes of operation. In the first mode, you specify which
packages you want to branch. This mode is more efficient for a small number
of packages.
In the second mode, pkgdb2branch will find every package lacking a BRANCH and
will create one if the pkgdb says it's needed. This mode is very efficient for
mass branching. This implies --cache-branches.
For those with a moderate number of packages, using a list of packages and
--cache-branches may be fastest.''')
parser.add_option('-b', '--branch-for',
dest='branchFor',
action='store',
help='Make sure all the packages have been branched for BRANCHFOR. Implies -c.')
parser.add_option('-c', '--cache-branches',
dest='enableCache',
action='store_true',
help='Download a complete cache of packages')
parser.add_option('--verbose',
dest='verbose',
action='store_true',
help='Enable verbose output')
(opts, args) = parser.parse_args()
if opts.branchFor:
if args:
raise ArgumentsError('Cannot specify packages with --branchFor')
opts.enableCache = True
if '-' in args:
opts.fromStdin = True
del (args[args.index('-')])
else:
opts.fromStdin = False
if not (args or opts.fromStdin or opts.branchFor):
raise ArgumentsError('You must list packages to operate on')
return opts, args
if __name__ == '__main__':
try:
options, packages = parse_commands()
except ArgumentsError, e:
print e
sys.exit(1)
unbranchedPackages = []
brancher = Brancher(BASEURL, options.enableCache, options.verbose)
if options.branchFor:
try:
unbranchedPackages = \
brancher.mass_branch(BRANCHBYGIT[options.branchFor])
except PackageDBError, e:
print 'Unable contact the PackageDB. Error: %s' % str(e)
sys.exit(1)
else:
# Process packages specified on the cmdline
for pkgname in packages:
try:
brancher.branch_package(pkgname)
except InternalError, e:
print str(e)
unbranchedPackages.append(pkgname)
# Process packages from stdin
if options.fromStdin:
for pkgname in sys.stdin:
pkgname = pkgname.strip()
try:
brancher.branch_package(pkgname)
except InternalError, e:
print str(e)
unbranchedPackages.append(pkgname)
if unbranchedPackages:
print 'The following packages were unbranched:'
print '\n'.join(unbranchedPackages)
sys.exit(100)
sys.exit(0)

117
scripts/distgit/setup_git_package Executable file
View file

@ -0,0 +1,117 @@
#!/bin/bash
#
# Create a new repo.
# THIS HAS TO BE RUN ON THE GIT SERVER!
# WARNING:
# This file is maintained within puppet?
# All local changes will be lost.
# Figure out the environment we're running in
GITROOT=/srv/git/rpms
# check if a moron is driving me
if [ ! -d $GITROOT ] ; then
# we're not on the git server (this check is fragile)
echo "ERROR: This script has to be run on the git server."
echo "ERROR: Homer sez 'Duh'."
exit -9
fi
# Local variables
VERBOSE=0
TEST=
IGNORE=
AUTHOR="Fedora Release Engineering <rel-eng@lists.fedoraproject.org>"
GIT_SSH_URL="ssh://localhost"
Usage() {
cat <<EOF
Usage:
$0 <package_name>
Creates a new repo for <package_name>
Options:
-h,--help This help message
EOF
}
if [ $# -gt 2 ]; then
Usage
exit -1
fi
# parse the arguments
while [ -n "$1" ] ; do
case "$1" in
-h | --help )
Usage
exit 0
;;
* )
PACKAGE="$1"
;;
esac
shift
done
# I hate shell scripting. I'm sure the above is totally wrong
# check the arguments
if [ -z "$PACKAGE" ] ; then
Usage
exit -1
fi
# Sanity checks before we start doing damage
[ $VERBOSE -gt 1 ] && echo "Checking package $PACKAGE..."
if [ -d $GITROOT/$PACKAGE.git ] ; then
echo "ERROR: Package module $PACKAGE already exists!" >&2
exit -1
fi
# Just don't run as root, mmkey?
if [ "$(id -un)" = "root" ] ; then
echo "Please run this script as yourself"
exit -3
fi
# "global" permissions check
if [ ! -w $GITROOT ] ; then
echo "ERROR: You can not write to $GITROOT"
echo "ERROR: You can not create repos"
exit -1
fi
# Now start working on creating those branches
# Create a tmpdir to do some git work in
TMPDIR=$(mktemp -d /tmp/tmpXXXXXX)
# First create the master repo
mkdir $GITROOT/$PACKAGE.git
pushd $GITROOT/$PACKAGE.git >/dev/null
git init -q --shared --bare
echo "$PACKAGE" > description # This is used to figure out who to send mail to.
git config --add hooks.mailinglist "$PACKAGE-owner@fedoraproject.org,scm-commits@lists.fedoraproject.org"
git config --add hooks.maildomain fedoraproject.org
popd >/dev/null
# Now clone that repo and create the .gitignore and sources file
git clone -q /srv/git/rpms/$PACKAGE.git $TMPDIR/$PACKAGE
pushd $TMPDIR/$PACKAGE >/dev/null
touch .gitignore sources
git add .
git commit -q -m 'Initial setup of the repo' --author "$AUTHOR"
git push -q origin master
popd >/dev/null
# Put our special update hooks in place
ln -s /usr/share/gitolite/hooks/common/update $GITROOT/$PACKAGE.git/hooks/
ln -s /usr/share/git-core/mail-hooks/gnome-post-receive-email \
$GITROOT/$PACKAGE.git/hooks/post-receive
rm -rf $TMPDIR
echo "Done."