Infrastructure/ansible
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

Add a new distgit role

Browse source 
This is the meat of Dist Git, but it requires quite a few of the roles
previously created.
This commit is contained in:
Mathieu Bridon 2014-07-17 18:35:17 +02:00 • committed by Kevin Fenzi
parent 4392ef9f6c
commit 30ab4bd528
18 changed files with 2332 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

71
roles/distgit/files/cgitrc Normal file
View file

@ -0,0 +1,71 @@
#
# See cgitrc(5) or /usr/share/doc/cgit-*/cgitrc.5.html for details
#
# Enable caching of up to 1000 output entries
cache-size=1000
# Specify some default clone prefixes
clone-prefix=git://pkgs.fedoraproject.org ssh://pkgs.fedoraproject.org http://pkgs.fedoraproject.org/git
# Specify the css url
css=/cgit-data/cgit.css
# Show extra links for each repository on the index page
enable-index-links=1
# Enable ASCII art commit history graph on the log pages
enable-commit-graph=1
# Show number of affected files per commit on the log pages
enable-log-filecount=1
# Show number of added/removed lines per commit on the log pages
enable-log-linecount=1
# Add a cgit favicon
#favicon=/favicon.ico
# Use a custom logo
logo=/cgit-data/cgit.png
# Enable statistics per week, month and quarter
max-stats=quarter
# Set the title and heading of the repository index page
root-title=Fedora Project Packages GIT repositories
# Set a subheading for the repository index page
#root-desc=tracking the foobar development
# Include some more info about this site on the index page
#root-readme=/var/www/html/about.html
# Allow download of tar.gz, tar.bz2 and zip-files
snapshots=tar.gz tar.xz zip
##
## List of common mimetypes
##
mimetype.gif=image/gif
mimetype.html=text/html
mimetype.jpg=image/jpeg
mimetype.jpeg=image/jpeg
mimetype.pdf=application/pdf
mimetype.png=image/png
mimetype.svg=image/svg+xml
# Enable syntax highlighting (requires the highlight package)
#source-filter=/usr/libexec/cgit/filters/syntax-highlighting.sh
email-filter=lua:/usr/libexec/cgit/filters/email-libravatar-korg.lua
##
## List of repositories.
## PS: Any repositories listed when section is unset will not be
## displayed under a section heading
## PPS: This list could be kept in a different file (e.g. '/etc/cgitrepos')
## and included like this:
project-list=/srv/git/pkgs-git-repos-list
scan-path=/srv/git/rpms/

4
roles/distgit/files/fedmsg-genacls-config.py Normal file
View file

@ -0,0 +1,4 @@
config = {
'genacls.consumer.enabled': True,
'genacls.consumer.delay': 5, # 5 seconds
}

118
roles/distgit/files/genacls.pkgdb Normal file
View file

@ -0,0 +1,118 @@
#!/usr/bin/python -t
#
# Create an /etc/gitolog/conf/getolog.conf file with acls for dist-git
#
# Takes no arguments!
#
import grp
import sys
import requests
if __name__ == '__main__':
# Get the users in various groups
TRUSTED = grp.getgrnam('cvsadmin')[3]
ARM = grp.getgrnam('fedora-arm')[3]
SPARC = grp.getgrnam('fedora-sparc')[3]
IA64 = grp.getgrnam('fedora-ia64')[3]
S390 = grp.getgrnam('fedora-s390')[3]
PPC = grp.getgrnam('fedora-ppc')[3]
PROVEN = grp.getgrnam('provenpackager')[3]
# Set the active branches to create ACLs for
# Give them the git branch eqiv until pkgdb follows suite
ACTIVE = {'OLPC-2': 'olpc2', 'OLPC-3': 'olpc3', 'EL-4': 'el4',
'EL-5': 'el5', 'el5': 'el5', 'el6': 'el6', 'EL-6': 'el6',
'epel7': 'epel7',
'F-11': 'f11', 'F-12': 'f12', 'F-13': 'f13', 'f14': 'f14', 'f15':
'f15', 'f16': 'f16', 'f17': 'f17', 'f18': 'f18', 'f19': 'f19',
'f20': 'f20', 'devel': 'master', 'master': 'master'}
# Create a "regex"ish list 0f the reserved branches
RESERVED = ['f[0-9][0-9]', 'epel[0-9]', 'epel[0-9][0-9]', 'el[0-9]', 'olpc[0-9]']
# Read the ACL information from the packageDB
data = requests.get('https://admin.fedoraproject.org/pkgdb/api/vcs?format=json').json()
# Get a list of all the packages
acls = data['packageAcls']
pkglist = data['packageAcls'].keys()
pkglist.sort()
# sanity check
if len(pkglist) < 2500:
sys.exit(1)
# print out our user groups
print '@admins = %s' % ' '.join(TRUSTED)
print '@provenpackager = %s' % ' '.join(PROVEN)
print '@fedora-arm = %s' % ' '.join(ARM)
print '@fedora-s390 = %s' % ' '.join(S390)
print '@fedora-ppc = %s' % ' '.join(PPC)
# print our default permissions
print 'repo @all'
print ' RWC = @admins @fedora-arm @fedora-s390 @fedora-ppc'
print ' R = @all'
#print ' RW private- = @all'
# dont' enable the above until we prevent building for real from private-
for pkg in pkglist:
branchAcls = {} # Check whether we need to set separate per branch acls
buffer = [] # Buffer the output per package
masters = [] # Folks that have commit to master
writers = [] # Anybody that has write access
# Examine each branch in the package
branches = acls[pkg].keys()
branches.sort()
for branch in branches:
if not branch in ACTIVE.keys():
continue
if 'packager' in acls[pkg][branch]['commit']['groups']:
# If the packager group is defined, everyone has access
buffer.append(' RWC %s = @all' % (ACTIVE[branch]))
branchAcls.setdefault('@all', []).append((pkg,
ACTIVE[branch]))
if branch == 'master':
masters.append('@all')
if '@all' not in writers:
writers.append('@all')
else:
# Extract the owners
committers = []
owners = acls[pkg][branch]['commit']['people']
owners.sort()
for owner in owners:
committers.append(owner)
if 'provenpackager' in acls[pkg][branch]['commit']['groups']:
committers.append('@provenpackager')
if branch == 'master':
masters.extend(committers)
# add all the committers to the top writers list
for committer in committers:
if not committer in writers:
writers.append(committer)
# Print the committers to the acl for this package-branch
committers = ' '.join(committers)
buffer.append(' RWC %s = %s' %
(ACTIVE[branch], committers))
branchAcls.setdefault(committers, []).append((pkg,
ACTIVE[branch]))
print
print 'repo %s' % pkg
#if len(branchAcls.keys()) == 1:
# acl = branchAcls.keys()[0]
# print ' RW = %s' % acl
#else:
print '\n'.join(buffer)
for reserved in RESERVED:
print ' - %s = @all' % reserved
print ' RWC refs/tags/ = %s' % ' '.join(writers)
if masters:
print ' RWC = %s' % ' '.join(masters)
sys.exit(0)

14
roles/distgit/files/genacls.sh Normal file
View file

@ -0,0 +1,14 @@
#!/bin/sh
TEMPDIR=`mktemp -d -p /var/tmp genacls.XXXXX`
export GL_RC=/etc/gitolite/gitolite.rc
export GL_BINDIR=/usr/bin
cd $TEMPDIR
# Only replace the acls if genacls completes successfully
if /usr/local/bin/genacls.pkgdb > gitolite.conf ; then
mv gitolite.conf /etc/gitolite/conf/
/usr/bin/gl-compile-conf
fi
cd /
rm -rf $TEMPDIR

3
roles/distgit/files/git-smart-http.conf Normal file
View file

@ -0,0 +1,3 @@
SetEnv GIT_PROJECT_ROOT /srv/git/rpms
SetEnv GIT_HTTP_EXPORT_ALL
ScriptAlias /git/ /usr/libexec/git-core/git-http-backend/

233
roles/distgit/files/gitolite.rc Normal file
View file

@ -0,0 +1,233 @@
# paths and configuration variables for gitolite
# please read comments before editing
# this file is meant to be pulled into a perl program using "do" or "require".
# You do NOT need to know perl to edit the paths; it should be fairly
# self-explanatory and easy to maintain perl syntax :-)
# --------------------------------------
# Do not uncomment these values unless you know what you're doing
# $GL_PACKAGE_CONF = "";
# $GL_PACKAGE_HOOKS = "";
# --------------------------------------
# --------------------------------------
# this is where the repos go. If you provide a relative path (not starting
# with "/"), it's relative to your $HOME. You may want to put in something
# like "/bigdisk" or whatever if your $HOME is too small for the repos, for
# example
$REPO_BASE="/srv/git/rpms/";
# the default umask for repositories is 0077; change this if you run stuff
# like gitweb and find it can't read the repos. Please note the syntax; the
# leading 0 is required
$REPO_UMASK = 0002;
# $REPO_UMASK = 0027; # gets you 'rwxr-x---'
# $REPO_UMASK = 0022; # gets you 'rwxr-xr-x'
# part of the setup of gitweb is a variable called $projects_list (please see
# gitweb documentation for more on this). Set this to the same value:
$PROJECTS_LIST = $ENV{HOME} . "/projects.list";
# --------------------------------------
# I see no reason anyone may want to change the gitolite admin directory, but
# feel free to do so. However, please note that it *must* be an *absolute*
# path (i.e., starting with a "/" character)
# gitolite admin directory, files, etc
$GL_ADMINDIR="/etc/gitolite";
# --------------------------------------
# templates for location of the log files and format of their names
# I prefer this template (note the %y and %m placeholders)
# it produces files like `~/.gitolite/logs/gitolite-2009-09.log`
$GL_LOGT="/var/log/gitolite/gitolite-%y-%m.log";
# other choices are below, or you can make your own -- but PLEASE MAKE SURE
# the directory exists and is writable; gitolite won't do that for you (unless
# it is the default, which is "$GL_ADMINDIR/logs")
# $GL_LOGT="$GL_ADMINDIR/logs/gitolite-%y-%m-%d.log";
# $GL_LOGT="$GL_ADMINDIR/logs/gitolite-%y.log";
# --------------------------------------
# Please DO NOT change these three paths
$GL_CONF="$GL_ADMINDIR/conf/gitolite.conf";
$GL_KEYDIR="$GL_ADMINDIR/keydir";
$GL_CONF_COMPILED="$GL_ADMINDIR/conf/gitolite.conf-compiled.pm";
# --------------------------------------
# if git on your server is on a standard path (that is
# ssh git@server git --version
# works), leave this setting as is. Otherwise, choose one of the
# alternatives, or write your own
$GIT_PATH="";
# $GIT_PATH="/opt/bin/";
# --------------------------------------
# ----------------------------------------------------------------------
# BIG CONFIG SETTINGS
# Please read doc/big-config.mkd for details
$GL_BIG_CONFIG = 1;
$GL_NO_DAEMON_NO_GITWEB = 1;
$GL_NO_CREATE_REPOS = 1;
$GL_NO_SETUP_AUTHKEYS = 1;
# ----------------------------------------------------------------------
# SECURITY SENSITIVE SETTINGS
#
# Settings below this point may have security implications. That
# usually means that I have not thought hard enough about all the
# possible ways to crack security if these settings are enabled.
# Please see details on each setting for specifics, if any.
# ----------------------------------------------------------------------
# --------------------------------------
# ALLOW REPO ADMIN TO SET GITCONFIG KEYS
#
# Gitolite allows you to set git repo options using the "config" keyword; see
# conf/example.conf for details and syntax.
#
# However, if you are in an installation where the repo admin does not (and
# should not) have shell access to the server, then allowing him to set
# arbitrary repo config options *may* be a security risk -- some config
# settings may allow executing arbitrary commands.
#
# You have 3 choices. By default $GL_GITCONFIG_KEYS is left empty, which
# completely disables this feature (meaning you cannot set git configs from
# the repo config).
$GL_GITCONFIG_KEYS = "";
# The second choice is to give it a space separated list of settings you
# consider safe. (These are actually treated as a set of regular expression
# patterns, and any one of them must match). For example:
# $GL_GITCONFIG_KEYS = "core\.logAllRefUpdates core\..*compression";
# allows repo admins to set one of those 3 config keys (yes, that second
# pattern matches two settings from "man git-config", if you look)
#
# The third choice (which you may have guessed already if you're familiar with
# regular expressions) is to allow anything and everything:
# $GL_GITCONFIG_KEYS = ".*";
# --------------------------------------
# EXTERNAL COMMAND HELPER -- HTPASSWD
# security note: runs an external command (htpasswd) with specific arguments,
# including a user-chosen "password".
# if you want to enable the "htpasswd" command, give this the absolute path to
# whatever file apache (etc) expect to find the passwords in.
$HTPASSWD_FILE = "";
# Look in doc/3 ("easier to link gitweb authorisation with gitolite" section)
# for more details on using this feature.
# --------------------------------------
# EXTERNAL COMMAND HELPER -- RSYNC
# security note: runs an external command (rsync) with specific arguments, all
# presumably filled in correctly by the client-side rsync.
# base path of all the files that are accessible via rsync. Must be an
# absolute path. Leave it undefined or set to the empty string to disable the
# rsync helper.
$RSYNC_BASE = "";
# $RSYNC_BASE = "/home/git/up-down";
# $RSYNC_BASE = "/tmp/up-down";
# --------------------------------------
# EXTERNAL COMMAND HELPER -- SVNSERVE
# security note: runs an external command (svnserve) with specific arguments,
# as specified below. %u is substituted with the username.
# This setting allows launching svnserve when requested by the ssh client.
# This allows using the same SSH setup (hostname/username/public key) for both
# SVN and git access. Leave it undefined or set to the empty string to disable
# svnserve access.
$SVNSERVE = "";
# $SVNSERVE = "/usr/bin/svnserve -r /var/svn/ -t --tunnel-user=%u";
# --------------------------------------
# ALLOW REPO CONFIG TO USE WILDCARDS
# security note: this used to in a separate "wildrepos" branch. You can
# create repositories based on wild cards, give "ownership" to the specific
# user who created it, allow him/her to hand out R and RW permissions to other
# users to collaborate, etc. This is powerful stuff, and I've made it as
# secure as I can, but it hasn't had the kind of rigorous line-by-line
# analysis that the old "master" branch had.
# This has now been rolled into master, with all the functionality gated by
# this variable. Set this to 1 if you want to enable the wildrepos features.
# Please see doc/4-wildcard-repositories.mkd for details.
$GL_WILDREPOS = 0;
# --------------------------------------
# DEFAULT WILDCARD PERMISSIONS
# If set, this value will be used as the default user-level permission rule of
# new wildcard repositories. The user can change this value with the setperms command
# as desired after repository creation; it is only a default. Note that @all can be
# used here but is special; no other groups can be used in user-level permissions.
# $GL_WILDREPOS_DEFPERMS = 'R = @all';
# --------------------------------------
# HOOK CHAINING
# by default, the update hook in every repo chains to "update.secondary".
# Similarly, the post-update hook in the admin repo chains to
# "post-update.secondary". If you're fine with the defaults, there's no need
# to do anything here. However, if you want to use different names or paths,
# change these variables
# $UPDATE_CHAINS_TO = "hooks/update.secondary";
# $ADMIN_POST_UPDATE_CHAINS_TO = "hooks/post-update.secondary";
# --------------------------------------
# ADMIN DEFINED COMMANDS
# WARNING: Use this feature only if (a) you really really know what you're
# doing or (b) you really don't care too much about security. Please read
# doc/admin-defined-commands.mkd for details.
# $GL_ADC_PATH = "";
# --------------------------------------
# per perl rules, this should be the last line in such a file:
1;
# Local variables:
# mode: perl
# End:
# vim: set syn=perl:

69
roles/distgit/files/lookaside-upload.conf Normal file
View file

@ -0,0 +1,69 @@
Alias /repo/ /srv/cache/lookaside/
# default SSL configuration...
Listen 443
SSLSessionCache shmcb:/var/cache/mod_ssl/scache(512000)
SSLSessionCacheTimeout 300
SSLMutex default
SSLRandomSeed startup file:/dev/urandom 256
SSLRandomSeed connect builtin
SSLCryptoDevice builtin
# SSL host
<VirtualHost _default_:443>
# This alias must come before the /repo/ one to avoid being overridden.
ScriptAlias /repo/pkgs/upload.cgi /srv/web/upload.cgi
Alias /repo/ /srv/cache/lookaside/
ServerName pkgs.fedoraproject.org
ServerAdmin webmaster@fedoraproject.org
SSLEngine on
SSLCertificateFile conf/pkgs.fedoraproject.org_key_and_cert.pem
SSLCertificateKeyFile conf/pkgs.fedoraproject.org_key_and_cert.pem
SSLCACertificateFile conf/cacert.pem
SSLCARevocationFile /etc/pki/tls/crl.pem
SSLCipherSuite RSA:!EXPORT:!DH:!LOW:!NULL:+MEDIUM:+HIGH
# Must be 'optional' everywhere in order to have POST operations work to upload.cgi
SSLVerifyClient optional
# Must be here for POST operations to upload.cgi
SSLOptions +OptRenegotiate
ErrorLog logs/ssl_error_log
CustomLog logs/ssl_access_log \
"%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%{SSL_CLIENT_S_DN_OU}x\" %{SSL_CLIENT_S_DN_CN}x %{SSL_CLIENT_S_DN_emailAddress}x \"%r\" %b"
<Directory /repo/pkgs/>
SSLVerifyClient optional
SSLVerifyDepth 1
SSLOptions +StrictRequire +StdEnvVars +OptRenegotiate
# require that the client auth cert was created by us and signed by us
SSLRequire ( %{SSL_CIPHER} !~ m/^(EXP|NULL)/ \
and %{SSL_CLIENT_S_DN_O} eq "Fedora Project" \
and %{SSL_CLIENT_S_DN_OU} eq "Fedora User Cert" \
and %{SSL_CLIENT_I_DN_O} eq "Fedora Project" \
and %{SSL_CLIENT_I_DN_OU} eq "Fedora Project CA" )
</Directory>
<Location /repo/pkgs/upload.cgi>
SSLRequireSSL
SSLVerifyClient optional
SSLVerifyDepth 1
SSLOptions +StrictRequire +StdEnvVars +OptRenegotiate
# require that the access comes from internal or that
# the client auth cert was created by us and signed by us
SSLRequire ( %{SSL_CIPHER} !~ m/^(EXP|NULL)/ \
and %{SSL_CLIENT_S_DN_O} eq "Fedora Project" \
and %{SSL_CLIENT_S_DN_OU} eq "Fedora User Cert" \
and %{SSL_CLIENT_I_DN_O} eq "Fedora Project" \
and %{SSL_CLIENT_I_DN_OU} eq "Fedora Project CA" )
</Location>
</VirtualHost>

6
roles/distgit/files/lookaside.conf Normal file
View file

@ -0,0 +1,6 @@
Alias /lookaside /srv/cache/lookaside
<Directory /srv/cache/lookaside>
Options Indexes FollowSymLinks
AllowOverride None
</Directory>

181
roles/distgit/files/mkbranch Normal file
View file

@ -0,0 +1,181 @@
#!/bin/bash
#
# Create a new development branch for a module.
# THIS HAS TO BE RUN ON THE GIT SERVER!
# WARNING:
# This file is maintained within puppet?
# All local changes will be lost.
# Figure out the environment we're running in
RUNDIR=$(cd $(dirname $0) && pwd)
GITROOT=/srv/git/rpms
# check if a moron is driving me
if [ ! -d $GITROOT ] ; then
# we're not on the git server (this check is fragile)
echo "ERROR: This script has to be run on the git server."
echo "ERROR: Homer sez 'Duh'."
exit -9
fi
# where are the packages kept
TOPLEVEL=rpms
# Local variables
VERBOSE=0
TEST=
IGNORE=
BRANCH=""
PACKAGES=""
SRC_BRANCH="master"
AUTHOR="Fedora Release Engineering <rel-eng@lists.fedoraproject.org>"
Usage() {
cat <<EOF
Usage:
$0 [ -s <src_branch>] <branch> <package_name>...
Creates a new branch <branch> for the list of <package_name>s.
The /master suffix on branch names is assumed.
Options:
-s,--source=<src_branch> Use <src_branch> as the source branch.
Defaults is master
/master suffix on other branches assumed
-n,--test Don't do nothing, only test
-i,--ignore Ignore erroneous modules
-h,--help This help message
-v,--verbose Increase verbosity
EOF
}
# parse the arguments
while [ -n "$1" ] ; do
case "$1" in
-h | --help )
Usage
exit 0
;;
-v | --verbose )
VERBOSE=$(($VERBOSE + 1))
;;
-i | --ignore )
IGNORE="yes"
;;
-n | --test )
TEST="yes"
;;
-s | --source )
shift
SRC_BRANCH=$1
;;
-b | --branch )
shift
BRANCH=$1/master
;;
* )
if [ -z "$BRANCH" ] ; then
BRANCH="$1"
else
PACKAGES="$PACKAGES $1"
fi
;;
esac
shift
done
# check the arguments
if [ -z "$BRANCH" -o -z "$PACKAGES" ] ; then
Usage
exit -1
fi
# Sanity checks before we start doing damage
NEWP=
for p in $PACKAGES ; do
[ $VERBOSE -gt 1 ] && echo "Checking package $p..."
if [ ! -d $GITROOT/$p.git ] ; then
echo "ERROR: Package module $p is invalid" >&2
[ "$IGNORE" = "yes" ] && continue || exit -1
fi
if [ -z "$(GIT_DIR=$GITROOT/$p.git git rev-parse -q --verify $SRC_BRANCH)" ] ; then \
echo "ERROR: Invalid source branch '$SRC_BRANCH' for package $p" >&2; \
if [ $SRC_BRANCH == 'master' ]; then
[ "$IGNORE" = "yes" ] && continue
else
SRC_BRANCH=master
fi
fi
$(GIT_DIR=$GITROOT/$p.git git rev-parse -q --verify \
$BRANCH >/dev/null) && \
(echo "IGNORING: Package module $p already has a branch $BRANCH" >&2; \
[ "$IGNORE" = "yes" ] && continue || exit -1)
NEWP="$NEWP $p"
done
PACKAGES="$(echo $NEWP)"
if [ -z "$PACKAGES" ] ; then
echo "NOOP: no valid packages found to process"
exit -1
fi
if [ -n "$TEST" ] ; then
echo "Branch $BRANCH valid for $PACKAGES"
exit 0
fi
# This account must have the proper permissions as to not screw up the
# repository work.
if [ "$(id -un)" = "root" ] ; then
echo "Please run this script as yourself"
exit -3
fi
#### Change this to check for proper git-admin rights
# "global" permissions check
if [ ! -w $GITROOT ] ; then
echo "ERROR: You can not write to $GITROOT"
echo "ERROR: You can not perform branching operations"
exit -1
fi
# Now start working on creating those branches
# For every module, "create" the branch
for NAME in $PACKAGES ; do
echo
echo "Creating new module branch '$BRANCH' for '$NAME' from branch '$SRC_BRANCH'..."
# permissions checks for this particular module
if [ ! -w $GITROOT/$NAME.git/refs/heads/ ] ; then
echo "ERROR: You can not write to $d"
echo "ERROR: $NAME can not be branched by you"
continue
fi
#### Replace the above with a gitolite permission check
#[ $VERBOSE -gt 0 ] && echo "Creating $BRANCH-split tag for $NAME/$SRC_BRANCH..."
# Is the above needed?
#cvs -Q rtag -f "$BRANCH-split" $TOPLEVEL/$NAME/$SRC_BRANCH || {
#echo "ERROR: Branch split tag for $NAME/$SRC_BRANCH could not be created" >&2
#exit -2
#}
[ $VERBOSE -gt 0 ] && echo "Creating $NAME $BRANCH from $NAME $SRC_BRANCH..."
$(pushd $GITROOT/$NAME.git >/dev/null && \
git branch --no-track $BRANCH $SRC_BRANCH && \
popd >/dev/null) || {
echo "ERROR: Branch $NAME $BRANCH could not be created" >&2
popd >/dev/null
exit -2
}
done
echo
echo "Done."

152
roles/distgit/files/pkgdb2-clone Normal file
View file

@ -0,0 +1,152 @@
#!/usr/bin/env python
import re
import requests
import sys
import getpass
import pkgdb2client
import subprocess
#PAGE_URL = 'https://fedoraproject.org/w/api.php?format=json&action=query&rvprop=content&prop=revisions&titles=User:Codeblock/RequestsSANDBOX'
PAGE_URL = 'https://fedoraproject.org/w/api.php?format=json&action=query&rvprop=content&prop=revisions&titles=EPEL/epel7/Requests'
NEW_EPEL_VERSION = '7'
NEW_EPEL_SOURCE_BRANCH = 'f19'
RHEL_PKGS_PATH = '/var/lib/rhel/rhel' + NEW_EPEL_VERSION
# parse_page :: String -> IO (Map String String)
# This returns a dictionary of {"pkg_name": "branch"}
def parse_page(url):
r = requests.get(url).json()
text = r['query']['pages'][r['query']['pages'].keys()[0]]['revisions'][0]['*']
lines = text.split("\n")
pkgs = filter(lambda y: y.startswith('| '), lines)
__pkgs_list__ = map(lambda y: ''.join(y.split())[1:].split('||'), pkgs)
pkgs_list = filter(lambda y: y[0] != 'foo', __pkgs_list__)
pkgs_dict = dict(pkgs_list)
return pkgs_dict
# is_in_rhel :: String -> IO Bool
def is_in_rhel(pkg):
with open(RHEL_PKGS_PATH) as f:
pkgs = map(lambda x: x.strip(), f.readlines())
return (pkg in pkgs)
# These tuples will be used to substitute one pattern for another.
# Every transform will be run on every branch name so be sure the
# pattern cannot match if you don't want it to be triggered.
transforms = (
(re.compile(r'^devel$'), 'master'),
(re.compile(r'-'), ''),
(re.compile(r'^fc([0-9])'), r'f\1'),
(re.compile(r'^epel([456])$'), r'el\1'),
(re.compile(r'^el([789]|[1-9][0-9])'), r'epel\1'),
)
branch_replacements = {'devel': (re.compile(r'^devel$'), 'master'),}
# generate_collection_cache :: PkgDB -> IO [String]
def generate_collection_cache(pkgdb):
raw_collections = pkgdb.get_collections(clt_status=(
'Active',
'Under Development'))
collection_cache = frozenset(map(lambda y: y['branchname'],
raw_collections['collections']))
return collection_cache
# normalize_branch :: [String] -> String -> IO (Option String)
def normalize_branch(collection_cache, branch):
# I originally had this implemented as a foldRight (which it really is).
# But Python doesn't eliminate tail calls. It probably would have been fine
# because "transforms" above is only 5 elements, but instead I will deal
# with the local mutation and wish that I had a type system to reason with.
# -rbe
norm_branch = branch.lower()
for transform in transforms:
norm_branch = re.sub(transform[0], transform[1], norm_branch)
# Ugh, here we break purity. Where is the option type when you need it?
if not (norm_branch in collection_cache):
print('Unknown collection specified: {0}'.format(branch))
return None
return norm_branch
# process_package :: PkgDB -> String -> String -> IO Bool
def process_package(pkgdb, pkg, src, dest):
data = pkgdb.get_package(pkg)
pkg_list = data['packages']
maybe_source = filter(lambda y: y['collection']['branchname'] == src,
pkg_list)
maybe_dest = filter(lambda y: y['collection']['branchname'] == dest,
pkg_list)
if len(maybe_source) == 0:
print "Source branch `" + src + "' not found. Please "\
"branch" + pkg + "manually."
return False
if len(maybe_dest) != 0:
print "Package `" + pkg + "' was already branched for `" + dest + "'."\
" Not overwriting branch."
return False
acls = filter(lambda y: y['fas_name'] != 'group::provenpackager',
maybe_source[0]['acls'])
map(lambda acl: pkgdb.update_acl(pkg, dest, acl['acl'], acl['status'],
acl['fas_name']), acls)
return True
# main :: [String] -> IO Unit
def main(args):
new_epel_requests = "epel" + NEW_EPEL_VERSION + "-requests"
if len(args) < 1 or (len(args) < 3 and args[0] != new_epel_requests) or\
len(args) > 3 or (len(args) > 1 and args[0] == new_epel_requests):
print "Usage: pkgdb2-clone " + new_epel_requests
print " - OR -"
print " pkgdb2-clone <source branch> <dest branch> <pkgs ...>"
sys.exit(1)
pkgdb = pkgdb2client.PkgDB()
username = raw_input('Username: ')
password = getpass.getpass()
pkgdb.login(username, password, True)
collection_cache = generate_collection_cache(pkgdb)
if args[0] == new_epel_requests:
pkgs = parse_page(PAGE_URL)
for key in pkgs:
if is_in_rhel(key):
continue
src_branchname = normalize_branch(collection_cache, pkgs[key])
dest_branchname = normalize_branch(collection_cache,
'epel' + NEW_EPEL_VERSION)
if not src_branchname or not dest_branchname:
print "[" + key + "] Invalid source or destination branch "\
"name, " + src_branchname + " -> " + dest_branchname
else:
if process_package(pkgdb, key, src_branchname, dest_branchname):
subprocess.call(["mkbranch",
"-s",
NEW_EPEL_SOURCE_BRANCH,
"epel" + NEW_EPEL_VERSION,
key])
print "[" + key + "] Success"
else:
print "[" + key + "] Error"
print "Done."
else:
src_branchname = normalize_branch(collection_cache, args[0])
dest_branchname = normalize_branch(collection_cache, args[1])
if not src_branchname or not dest_branchname:
print "[" + key + "] Invalid source or destination branch "\
"name, " + src_branchname + " -> " + dest_branchname
for pkg in args[2:]:
if process_package(pkgdb, key, src_branchname, dest_branchname):
print "[" + key + "] Success"
else:
print "[" + key + "] Error"
if __name__ == '__main__':
main(sys.argv[1:])

362
roles/distgit/files/pkgdb2branch.py Normal file
View file

@ -0,0 +1,362 @@
#!/usr/bin/python -t
# Author: Toshio Kuratomi
# Copyright: 2007-2008 Red Hat Software
# License: GPLv2
# This needs a proper license and copyright here
__version__ = '0.3'
import sys
import os
import optparse
import subprocess
import fedmsg
# Do some off-the-bat configuration of fedmsg.
# 1) since this is a one-off script and not a daemon, it needs to connect to
# the fedmsg-relay process running on another node (or noone will hear it)
# 2) its going to use the 'shell' certificate which only 'sysadmin' has read
# access to. Contrast that with the 'scm' certificate which everyone in
# the 'packager' group has access to.
config = fedmsg.config.load_config([], None)
config['active'] = True
config['endpoints']['relay_inbound'] = config['relay_inbound']
fedmsg.init(name='relay_inbound', cert_prefix='shell', **config)
from fedora.client import FedoraServiceError
from pkgdb2client import PkgDB
GITDIR='/srv/git/rpms'
BASEURL = os.environ.get('PACKAGEDBURL') or 'https://admin.fedoraproject.org/pkgdb/'
MKBRANCH='/usr/local/bin/mkbranch'
SETUP_PACKAGE='/usr/local/bin/setup_git_package'
BRANCHES = {'el4': 'master', 'el5': 'master', 'el6': 'master', 'epel7': 'f19',
'olpc2': 'f7',
'olpc3': 'f11',
'master': None,
'fc6': 'master',
'f7': 'master',
'f8': 'master',
'f9': 'master',
'f10': 'master',
'f11': 'master',
'f12': 'master',
'f13': 'master', 'f14': 'master',
'f15': 'master', 'f16': 'master',
'f17': 'master', 'f18': 'master',
'f19': 'master', 'f20': 'master'
}
# The branch names we get out of pkgdb have to be translated to git
GITBRANCHES = {'el4': 'el4', 'el5': 'el5', 'el6': 'el6', 'epel7': 'epel7',
'OLPC-2': 'olpc2',
'FC-6': 'fc6', 'F-7': 'f7', 'F-8': 'f8', 'F-9': 'f9',
'F-10': 'f10', 'OLPC-3': 'olpc3',
'F-11': 'f11', 'F-12': 'f12', 'F-13': 'f13', 'f14': 'f14', 'f15': 'f15', 'f16': 'f16', 'f17': 'f17',
'f18': 'f18', 'f19': 'f19', 'f20': 'f20',
'devel': 'master'}
# The branch options we get from the CLI have to be translated to pkgdb
BRANCHBYGIT = dict([(v, k) for (k, v) in GITBRANCHES.iteritems()])
class InternalError(Exception):
pass
class PackageDBError(InternalError):
pass
class ProcessError(InternalError):
pass
class ArgumentsError(InternalError):
pass
class InvalidBranchError(PackageDBError):
pass
class PackageDBClient(PkgDB):
def __init__(self, baseURL):
'''Initialize the connection.
Args:
:baseURL: URL from which the packageDB is accessed
'''
# We're only performing read operations so we don't need a username
super(PackageDBClient, self).__init__(baseURL)
def get_package_branches(self, pkgname):
'''Return the branches to which a package belongs.
Args:
:pkgname: The package to retrieve branch information about
'''
data = self.get_package(pkgname)
return map(lambda x: x['collection']['branchname'], data['packages'])
def get_package_list(self, branchName):
'''Retrieve all the packages in a specific branch.
Args:
:branchName: to return the packages for
'''
pkgs = map(lambda l: l['name'], self.get_packages('*', branchName, page=0)['packages'])
return pkgs
class Brancher(object):
''' Make branches in the GIT Repository.'''
def __init__(self, pkgdburl, cache, verbose):
# Connect to the package database
self.verbose = verbose
self.client = PackageDBClient(BASEURL)
def _invoke(self, program, args):
'''Run a command and raise an exception if an error occurred.
Args:
:program: The program to invoke
:args: List of arguments to pass to the program
raises ProcessError if there's a problem.
'''
cmdLine = [program]
cmdLine.extend(args)
print ' '.join(cmdLine)
stdoutfd = subprocess.PIPE
if self.verbose:
program = subprocess.Popen(cmdLine, stderr=subprocess.STDOUT)
else:
program = subprocess.Popen(cmdLine, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
retCode = program.wait()
if retCode != 0:
e = ProcessError()
e.returnCode = retCode
e.cmd = ' '.join(cmdLine)
if self.verbose:
output = program.stdout.read()
e.message = 'Error, "%s" returned %s: %s' % (e.cmd, e.returnCode, output)
else:
e.message = 'Error, "%s" returned %s' % (e.cmd, e.returnCode)
raise e
def _create_branch(self, pkgname, branch):
'''Create a specific branch for a package.
Args:
:pkgname: Name of the package to branch
:branch: Name of the branch to create
raises InvalidBranchError if a branchname is unknown.
Will ignore a branch which is EOL.
'''
try:
branchFrom = '%s/master' % BRANCHES[branch]
except KeyError:
raise InvalidBranchError(
'PackageDB returned an invalid branch %s for %s' %
(branch, pkgname))
# Add the master to the branch
# No longer add this after the new branching setup.
#branch = '%s/master' % branch
# If branchFrom is None, this is an EOL release
# If the directory already exists, no need to invoke mkbranch
if branchFrom:
# Fall back to branching from master.
frombranchpath = os.path.join(GITDIR, '%s.git' % pkgname,
'refs/heads', branchFrom)
if not os.path.exists(frombranchpath):
branchFrom = 'master'
branchpath = os.path.join(GITDIR, '%s.git' % pkgname,
'refs/heads', branch)
if not os.path.exists(branchpath):
try:
self._invoke(MKBRANCH, ['-s', branchFrom, branch, pkgname])
except ProcessError, e:
if e.returnCode == 255:
# This is a warning, not an error
return
raise
finally:
fedmsg.publish(
topic='branch',
modname='git',
msg=dict(
agent=os.getlogin(),
name=pkgname,
branch=branch,
),
)
def branch_package(self, pkgname):
'''Create all the branches that are listed in the pkgdb for a package.
Args:
:pkgname: The package to create branches for
Note: this will ignore branches which are EOL.
raises PackageDBError if the package is not present in the Package
Database.
'''
# Retrieve branch information
try:
branches = self.client.get_package_branches(pkgname)
except FedoraServiceError, e:
raise PackageDBError(
'Unable to retrieve information about %s: %s' %
(pkgname, str(e)))
# Create the devel branch if necessary
if not os.path.exists(os.path.join(GITDIR,
'%s.git' % pkgname)):
self._invoke(SETUP_PACKAGE, [pkgname])
# Create all the required branches for the package
# Use the translated branch name until pkgdb falls inline
for branch in branches:
if branch == 'devel':
continue
if not branch in GITBRANCHES.keys():
print 'Skipping unknown branch %s' % branch
continue
self._create_branch(pkgname, GITBRANCHES[branch])
def mass_branch(self, branchName):
'''Make sure all packages listed for a specific branch in the PackageDB
have a CVS branch.
Args:
:branchName: The branch to ensure.
'''
fedmsg.publish(
topic='mass_branch.start',
modname='git',
msg=dict(agent=os.getlogin()),
)
# Retrieve all the packages in this branch
pkglist = self.client.get_package_list(branchName)
pkglist.sort()
for pkg in pkglist:
# Create a branch for this release for each of them
# Use the translated branch name until pkgdb falls inline
self._create_branch(pkg, GITBRANCHES[branchName])
fedmsg.publish(
topic='mass_branch.complete',
modname='git',
msg=dict(agent=os.getlogin()),
)
def parse_commands():
parser = optparse.OptionParser(version=__version__, usage='''pkgdb2branch.py [options] PACKAGENAME [packagename, ...] [-]
pkgdb2branch.py [options] --branchfor BRANCH
pkgdb2branch reads package information from the packagedb and creates branches
on the git server based on what branches are listed there. pkgdb2branch can
read the list of packages from stdin if you specify '-' as an argument.
pkgdb2branch has two modes of operation. In the first mode, you specify which
packages you want to branch. This mode is more efficient for a small number
of packages.
In the second mode, pkgdb2branch will find every package lacking a BRANCH and
will create one if the pkgdb says it's needed. This mode is very efficient for
mass branching. This implies --cache-branches.
For those with a moderate number of packages, using a list of packages and
--cache-branches may be fastest.''')
parser.add_option('-b', '--branch-for',
dest='branchFor',
action='store',
help='Make sure all the packages have been branched for BRANCHFOR. Implies -c.')
parser.add_option('-c', '--cache-branches',
dest='enableCache',
action='store_true',
help='Download a complete cache of packages')
parser.add_option('--verbose',
dest='verbose',
action='store_true',
help='Enable verbose output')
(opts, args) = parser.parse_args()
if opts.branchFor:
if args:
raise ArgumentsError('Cannot specify packages with --branchFor')
opts.enableCache = True
if '-' in args:
opts.fromStdin = True
del (args[args.index('-')])
else:
opts.fromStdin = False
if not (args or opts.fromStdin or opts.branchFor):
raise ArgumentsError('You must list packages to operate on')
return opts, args
if __name__ == '__main__':
try:
options, packages = parse_commands()
except ArgumentsError, e:
print e
sys.exit(1)
branchedPackages, unbranchedPackages = [], []
brancher = Brancher(BASEURL, options.enableCache, options.verbose)
fedmsg.publish(
topic='pkgdb2branch.start',
modname='git',
msg=dict(agent=os.getlogin()),
)
if options.branchFor:
try:
unbranchedPackages = \
brancher.mass_branch(BRANCHBYGIT[options.branchFor])
except PackageDBError, e:
print 'Unable contact the PackageDB. Error: %s' % str(e)
sys.exit(1)
else:
# Process packages specified on the cmdline
for pkgname in packages:
try:
brancher.branch_package(pkgname)
branchedPackages.append(pkgname)
except InternalError, e:
print str(e)
unbranchedPackages.append(pkgname)
# Process packages from stdin
if options.fromStdin:
for pkgname in sys.stdin:
pkgname = pkgname.strip()
try:
brancher.branch_package(pkgname)
branchedPackages.append(pkgname)
except InternalError, e:
print str(e)
unbranchedPackages.append(pkgname)
fedmsg.publish(
topic='pkgdb2branch.complete',
modname='git',
msg=dict(
agent=os.getlogin(),
branchedPackages=branchedPackages,
unbranchedPackages=unbranchedPackages,
),
)
if unbranchedPackages:
print 'The following packages were unbranched:'
print '\n'.join(unbranchedPackages)
sys.exit(100)
sys.exit(0)

1
roles/distgit/files/pkgs.fedoraproject.org.conf Normal file
View file

@ -0,0 +1 @@
include "conf.d/pkgs.fedoraproject.org/*.conf"

654
roles/distgit/files/process-git-requests Normal file
View file

@ -0,0 +1,654 @@
#!/usr/bin/python -t
VERSION = "1.0"
# $Id: process-cvs-requests.py,v 1.25 2010/07/01 04:51:23 tibbs Exp $
# TODO:
# Extract fedora-review flag setter if possible.
# Display last linked spec file.
# Download (and process?) last linked srpm
# Checks to add:
# Package already exists in pkgdb.
# fedora-review flag isn't set (especially if it's still set to '?'
# Catch common misspellings?
# Any owner contains '@' or other invalid character
# Maybe verify owners in pkgdb/FAS.
# SSH into cvs.fedoraproject.org to run pkg2branch.py directly
# or just run on cvs.fedoraproject.org
# Try to do some checking on the ~/.bugzillacookies file and suggest "bugzilla login"
import bugzilla
import codecs
import datetime
import getpass
import glob
import operator
import os
import re
import readline
import sys
import subprocess
import tempfile
import time
import xmlrpclib
from configobj import ConfigObj, flatten_errors
from fedora.client import AuthError, AppError, PackageDB
from optparse import OptionParser
from validate import Validator
# Red Hat's bugzilla
url = 'https://bugzilla.redhat.com/xmlrpc.cgi'
# Users who indicated that they're OK with EPEL branches. Some request that
# they be made comaintainers.
# Taken from http://fedoraproject.org/wiki/EPEL/ContributorStatusNo
epel_ok = ['abompart', 'athimm', 'corsepiu', 'ecik', 'faucamp', 'konradm',
'monnerat', 'mtasaka', 'nim', 'rafalzaq', 'rineau', 'rstrode',
'sgrubb', 'shishz', 'terjeros', 'zkota']
epel_ok_comaint = ['alexlan', 'guidograzioli', 'jwrdegoede', 'kkofler',
'mebourne', 'overholt', 'pgordon', 'rishi', 'snirkel']
PAGER = os.environ.get('PAGER') or '/usr/bin/less'
EDITOR = os.environ.get('EDITOR') or '/usr/bin/vim'
# Override a method in xmlrpclib so it doesn't blow up when getting crap data
# from Red Hat's bugzilla.
# Bugfixes seem to have rendered this unnecessary
#def _decode(data, encoding, is8bit=re.compile("[\x80-\xff]").search):
# # decode non-ascii string (if possible)
# if unicode and encoding and is8bit(data):
# data = unicode(data, encoding, 'replace')
# return data
#xmlrpclib._decode = _decode
def parse_commandline():
usage = 'usage: %prog [options]'
parser = OptionParser(usage)
parser.add_option('--url', dest='url',
help='bugzilla URL to query',
default=url)
parser.add_option('-u', '--user',
help='Username for PackageDB connection',
dest='user',
default=getpass.getuser())
parser.add_option('--debug',
action='store_true',
dest='debug',
default=False,
help='Turn on some debugging statements')
(options, args) = parser.parse_args()
return options
def parse_pkgdb_config():
vldtr = Validator()
# configspec to validate types and set defaults
configspec = '''
[global]
pkgdb.url = string(default = 'https://admin.fedoraproject.org/pkgdb')
pkgdb.retries = integer(default = 5)
pkgdb.knowngroups = list(default = list())
'''.splitlines()
cfg = ConfigObj('/etc/pkgdb-client.cfg', configspec=configspec)
user = ConfigObj(os.path.expanduser('~/.fedora/pkgdb-client.cfg'),
configspec=configspec)
cfg.merge(user)
res = cfg.validate(vldtr, preserve_errors=True)
for entry in flatten_errors(cfg, res):
section_list, key, error = entry
section_list.append(key)
section_string = ','.join(section_list)
if error == False:
error = 'Missing value or section.'
print ','.join(section_list), '=', error
sys.exit(1)
cfg['global']['pkgdb.url'] = os.environ.get('PACKAGEDBURL') or cfg['global']['pkgdb.url']
return cfg['global']
def encode_utf8(object, encoding='utf8', errors='replace'):
if isinstance(object, basestring):
if isinstance(object, str):
return unicode(object, encoding, errors)
else:
return object
return u''
def add_package(pkgdb, request):
for retry in range(1, config['pkgdb.retries'] + 1):
try:
pkgdb.add_package(pkg=request['pkg'],
owner=request['owner'],
description=request['description'],
branches=request['branches'],
cc_list=request['cc_list'],
comaintainers=request['comaintainers'])
except AuthError, e:
if sys.stdin.isatty():
if retry >= config['pkgdb.retries']:
break
pkgdb.password = getpass.getpass('PackageDB Password: ')
else:
# Don't retry if we're reading the password from stdin
break
else:
break
def edit_package(pkgdb, request):
for retry in range(1, config['pkgdb.retries'] + 1):
try:
pkgdb.edit_package(pkg=request['pkg'],
owner=request['owner'],
branches=request['newbranches'],
cc_list=request['cc_list'],
comaintainers=request['comaintainers'])
except AuthError, e:
if retry >= config['pkgdb.retries']:
break
pkgdb.password = getpass.getpass('PackageDB Password: ')
else:
break
def run_query(bz):
querydata = {}
querydata['column_list'] = ['opendate', 'changeddate', 'bug_severity',
'alias', 'assigned_to', 'reporter', 'bug_status', 'resolution',
'component', 'blockedby', 'dependson', 'short_desc',
'status_whiteboard', 'flag_types']
querydata['product'] = ['Fedora']
querydata['field0-0-0'] = 'flagtypes.name'
querydata['type0-0-0'] = 'equals'
querydata['value0-0-0'] = 'fedora-cvs?'
bugs = bz.query(querydata)
bugs.sort(key=operator.attrgetter('bug_id'))
ids = map(lambda x: x.bug_id, bugs)
comments = bz._proxy.Bug.comments({"ids": ids})
return [bugs, comments]
def display_bug(bug, comments):
'''Show the complete ticket in a pager.'''
comment = 0
b = []
b.append('https://bugzilla.redhat.com/%d' % bug.bug_id)
b.append('Bug %d - %s' % (bug.bug_id, bug.short_desc))
b.append('Reported by: %s at %s' % (bug.reporter, bug.opendate))
b.append('Assigned to: %s' % (bug.assigned_to))
for i in comments:
b.append('-'*40)
#b.append('Comment %d by %s at %s\n' % (comment, i['author'], time.strftime('%F %T',i['time'].timetuple())))
#b.append('Comment %d by %s at %04d-%02d-%02d %02d:%02d%02d\n' % (
b.append('Comment %d by %s at %s\n' % (
comment, i['author'], i['time']))
b.append(i['text'])
b.append('')
comment += 1
p = subprocess.Popen(PAGER, stdin=subprocess.PIPE)
p.communicate('\n'.join(b).encode('utf8'))
def edit_string(s):
'''Edit the contents of a string in the user's preferred editor.'''
(fd, f) = tempfile.mkstemp()
fh=os.fdopen(fd, 'w+')
fh.write(s)
fh.close()
p = subprocess.Popen([EDITOR, f]);
sts = os.waitpid(p.pid, 0)[1]
if not sts:
try:
fh = open(f, 'r')
s = fh.read()
finally:
fh.close()
return s
def parse_prefixed_lines(s):
lastitem = ''
items = {}
items['Branches'] = ''
lines = s.splitlines()
# Skip until the Request line
while 1:
if (lines[0].find('New Package CVS Request') == 0
or lines[0].find('Package Change Request') == 0):
break
lines.pop(0)
# Skip until a line containing a colon
while 1:
if lines[0].find(':') >= 0:
break
lines.pop(0)
# Now parse
while 1:
if not len(lines):
break
line = lines.pop(0)
line.strip()
if len(line) == 0:
break
pos = line.find(':')
# Line-wrapped?
if pos < 0:
items[lastitem] += " " + line.strip()
continue
lastitem = line[:pos]
items[lastitem] = line[pos+1:].strip()
return items
def clean_request(items):
'''Clean up various bits that can be passed in a CVS request.'''
request = {}
if not 'InitialCC' in items:
items['InitialCC'] = ''
if not 'Owners' in items:
items['Owners'] = ''
if not 'Short Description' in items:
items['Short Description'] = ''
branches = items['Branches'].strip()
branches = re.sub(r',', ' ', branches)
branches = re.sub(r'f', 'F', branches)
branches = re.sub(r'devel', ' ', branches)
branches = re.sub(r'F([1-9][0-9])', r'F-\1', branches)
branches = re.sub(r'EL([1-9])', r'EL-\1', branches)
branches = re.sub(r'F-14', r'f14', branches)
branches = re.sub(r' +', ' ', branches)
branches = branches.strip()
branches += ' devel'
items['Branches'] = branches
request['branches'] = branches.split()
if 'New Branches' in items:
branches = items['New Branches'].strip()
branches = re.sub(r',', ' ', branches)
branches = re.sub(r'f', 'F', branches)
branches = re.sub(r'F([1-9][0-9])', r'F-\1', branches)
branches = re.sub(r'F-14', r'f14', branches)
branches = re.sub(r' +', ' ', branches)
branches = branches.strip()
items['New Branches'] = branches
request['newbranches'] = branches.split()
owners = items['Owners'].strip()
owners = re.sub(r',', ' ', owners)
if len(owners):
request['owner'] = owners.split()[0]
request['comaintainers'] = owners.split()[1:]
request['cc_list'] = items['InitialCC'].split()
request['pkg'] = items['Package Name']
request['description'] = items['Short Description']
return request
def new_request_string(items, bug):
r = []
r.append("Bug URL: http://bugzilla.redhat.com/%d " % bug.bug_id)
r.append("Bug summary: " + bug.short_desc)
r.append('')
r.append("New Package CVS Request")
r.append("=======================")
r.append("Package Name: " + items['Package Name'])
r.append("Short Description: " + items['Short Description'])
r.append("Owners: " + items['Owners'])
r.append("Branches: " + items['Branches'])
r.append("InitialCC: " + items['InitialCC'])
r.append('')
return '\n'.join(r)
def change_request_string(items, bug):
r = []
r.append("Bug URL: http://bugzilla.redhat.com/%d" % bug.bug_id)
r.append("Bug summary: " + bug.short_desc)
r.append('')
r.append("Package Change Request")
r.append("======================")
r.append("Package Name: " + items['Package Name'])
r.append("Owners: " + items['Owners'])
r.append("New Branches: " + items['New Branches'])
r.append("InitialCC: " + items['InitialCC'])
r.append('')
return '\n'.join(r)
def get_pkgdb_owners(pkgdb, pkg):
owners = {}
o = ''
for i in pkgdb.get_owners(pkg)['packageListings']:
branch = i['collection']['branchname']
if branch not in branches:
continue
owners[branch] = {}
owners[branch]['primary'] = i['owner']
owners[branch]['comaint'] = []
for j in i['people']:
#if 'commit' in j['aclOrder']:
if j['aclOrder']['commit'] != None and j['username'] != owners[branch]:
owners[branch]['comaint'].append(j['username'])
for i in sorted(branches, reverse=True):
if i in owners:
o += "%s: %s" % (i, owners[i]['primary'])
if len(owners[i]['comaint']):
o += ' - %s' % ','.join(sorted(owners[i]['comaint']))
o += '\n'
return (owners, o)
def process_no_request(bug, allcomments):
'''Deal with a ticket where no request was found.'''
while 1:
os.system('clear')
print "No CVS request found in bug %d\nhttp://bugzilla.redhat.com/%d." % (bug.bug_id, bug.bug_id)
ok = raw_input('\nWhat do? (n=Next, s=Show ticket, c=Comment, q=Quit):')
if ok == 'c':
bug_comment = edit_string('')
print bug_comment
ok = raw_input("\nPost this comment to the ticket (y/n)?")
if ok == 'y':
print "Updating bugzilla..."
bug.addcomment(bug_comment)
ok = raw_input("\nClear the fedora-cvs flag (y/n)?")
if ok == 'y':
print "Clearing the flag..."
bug.updateflags({'fedora-cvs':'X', 'nomail':1})
break
elif ok == 'n':
return True
elif ok == 'q':
return False
elif ok == 's':
print
display_bug(bug, allcomments)
return True
def process_new_request(bug, comment, allcomments, firstfound, pkgdb, branches):
'''Parse a new package request, try to repair line wrapping, and do some
basic validity checks.'''
warned = False
warnings = []
items = parse_prefixed_lines(comment['text'])
request = clean_request(items)
if not firstfound:
warnings.append("WARNING: CVS request was not the last comment.")
warned = True
if not 'Package Name' in items:
warnings.append("WARNING: No package name supplied.")
warned = True
if not 'Owners' in items:
warnings.append("WARNING: No owners provided.")
warned = True
if not 'Short Description' in items:
warnings.append("WARNING: No description provided.")
warned = True
for i in request['branches']:
if i not in branches:
warnings.append("WARNING: Invalid branch %s requested" % i)
warned = True
short_desc = bug.short_desc
m=re.search('Review Request:\s([a-zA-Z0-9_+.-]+)\s+', short_desc, re.I)
if not m:
warnings.append("WARNING: Couldn't parse package name out of bug summary.")
warned = True
elif m.group(1) != items['Package Name']:
warnings.append("WARNING: Requested package name %s doesn't match bug summary %s" % (items['Package Name'], m.group(1)))
warned = True
req_string = new_request_string(items, bug)
bug_comment = 'GIT done (by process-git-requests).\n'
okprompt = 'Do it (yes=Yes, n=No, e=Edit request, s=Show ticket, c=Comment, q=Quit)?'
if warned:
prompt = 'Warnings present!\nDo it (a=Accept warnings, n=No, e=Edit request, s=Show ticket, c=Comment, q=Quit)?'
else:
prompt = okprompt
# We have to loop until the user accepts the request
while 1:
# We have to loop until the user enters something that works
while 1:
os.system('clear')
if len(warnings):
print '\n'.join(warnings), "\n"
print "Currently assigned to: %s" % bug.assigned_to
print req_string
ok = raw_input(prompt)
if ok == 'a':
prompt = okprompt
warned = False
if ok == 'c':
bug_comment = edit_string('')
print bug_comment
ok = raw_input("\nPost this comment to the ticket (y/n)?")
if ok == 'y':
print "Updating bugzilla..."
bug.addcomment(bug_comment)
ok = raw_input("\nClear the fedora-cvs flag (y/n)?")
if ok == 'y':
print "Clearing the flag..."
bug.updateflags({'fedora-cvs':'X', 'nomail':1})
return (False, True)
elif ok == 'e':
req_string = edit_string(req_string)
items=parse_prefixed_lines(req_string)
request = clean_request(items)
req_string = new_request_string(items, bug)
break
elif ok == 'n':
return (False, True)
elif ok == 'q':
return (False, False)
elif ok == 's':
print
display_bug(bug, allcomments)
elif ok == 'yes' and not warned:
bug_comment = edit_string(bug_comment)
print '\n', bug_comment
ok = raw_input('Go ahead (y/n)?')
if ok != 'y':
break
print 'Calling pkgdb...'
try:
add_package(pkgdb, request)
except Exception, e:
print "Pkgdb call failed:"
print e
raw_input('\nPress enter to continue to the next ticket.')
return (False, True)
print 'Updating bugzilla...'
# XXX Need to handle errors here
bug.updateflags({'fedora-cvs':'+', 'nomail':1})
bug.addcomment(bug_comment)
return (request['pkg'], True)
else:
pass
def process_change_request(bug, comment, allcomments, firstfound, pkgdb, branches):
'''Parse a change request, try to repair line wrapping, and do some
basic validity checks.'''
owned = False
warned = False
warnings = []
items = parse_prefixed_lines(comment['text'])
request = clean_request(items)
print "Looking up owners in pkgdb..."
(owners, owner_string) = get_pkgdb_owners(pkgdb, items['Package Name'])
# Try to enforce EPEL branch rules
for i in owners.keys():
if request['owner'] == owners[i]['primary'] or request['owner'] in owners[i]['comaint']:
owned = True
if not owned and items['New Branches'].find('EL') >= 0 and owners['devel']['primary'] in epel_ok:
warnings.append("NOTE: new branch owner not owner of other branches,\n but primary devel owner is OK with EPEL branches.")
elif not owned and items['New Branches'].find('EL') >= 0 and owners['devel']['primary'] in epel_ok_comaint:
warnings.append("NOTE: new branch owner not owner of other branches,\n but primary devel owner is OK with EPEL branches\n as long as they comaintain.")
elif not owned:
warnings.append("WARNING: new branch owner not owner of other branches.")
warned = True
if not firstfound:
warnings.append("WARNING: GIT request was not the last comment.")
warned = True
if not 'Package Name' in items:
warnings.append("WARNING: No package name supplied.")
warned = True
if not 'Owners' in items:
warnings.append("WARNING: No owners provided.")
warned = True
if not 'New Branches' in items:
warnings.append("WARNING: No new branches requested.")
for i in request['branches']:
if i not in branches:
warnings.append("WARNING: Invalid branch %s requested" % i)
warned = True
short_desc = bug.short_desc
req_string = change_request_string(items, bug)
bug_comment = 'GIT done (by process-git-requests).\n'
okprompt = 'Do it (yes=Yes, n=No, e=Edit request, s=Show ticket, c=Comment, q=Quit)?'
if warned:
prompt = 'Warnings present!\nDo it (a=Accept warnings, n=No, e=Edit request, s=Show ticket, c=Comment, q=Quit)?'
else:
prompt = okprompt
# We have to loop until the user accepts the request
while 1:
# We have to loop until the user enters something that works
while 1:
os.system('clear')
if len(warnings):
print '\n'.join(warnings), "\n"
print req_string + "\nCurrent branch owners - comaintainers:\n" + owner_string
ok = raw_input(prompt)
if ok == 'a':
prompt = okprompt
warned = False
if ok == 'c':
bug_comment = edit_string('')
print bug_comment
ok = raw_input("\nPost this comment to the ticket (y/n)?")
if ok == 'y':
print "Updating bugzilla..."
bug.addcomment(bug_comment)
ok = raw_input("\nClear the fedora-cvs flag (y/n)?")
if ok == 'y':
print "Clearing the flag..."
bug.updateflags({'fedora-cvs':'X', 'nomail':1})
return (False, True)
elif ok == 'e':
req_string = edit_string(req_string)
items=parse_prefixed_lines(req_string)
request = clean_request(items)
req_string = change_request_string(items, bug)
break
elif ok == 'n':
return (False, True)
elif ok == 'q':
return (False, False)
elif ok == 's':
print
display_bug(bug, allcomments)
elif ok == 'yes' and not warned:
bug_comment = edit_string(bug_comment)
print '\n', bug_comment
ok = raw_input('Go ahead (y/n)?')
if ok != 'y':
break
print 'Calling pkgdb...'
try:
edit_package(pkgdb, request)
except Exception, e:
print "Pkgdb call failed:"
print e
raw_input('\nPress enter to continue to the next ticket.')
return (False, True)
print 'Updating bugzilla...'
# XXX Need to handle errors here
bug.updateflags({'fedora-cvs':'+', 'nomail':1})
bug.addcomment(bug_comment)
return (request['pkg'], True)
else:
pass
if __name__ == '__main__':
branches = {}
processed = []
options = parse_commandline()
print "Connecting to bugzilla..."
bz = bugzilla.Bugzilla(url=options.url)
print "Querying bugzilla..."
(bugs, comments) = run_query(bz)
print "Done; got %d." % len(bugs)
if not len(bugs):
print "No requests to process!"
exit(0)
print "Connecting to pkgdb..."
config = parse_pkgdb_config()
pkgdb = PackageDB(config['pkgdb.url'], username=options.user,
debug=options.debug)
print "Getting valid branches...."
for i in pkgdb.get_collection_list(eol=False):
branches[i[0]['branchname']] = 1
print "Done."
print
# Iterate over bugs
for i in bugs:
firstfound = True
type = ''
print "Parsing bug %d - https://bugzilla.redhat.com/%d" % (i.bug_id, i.bug_id)
for j in reversed(comments['bugs'][str(i.bug_id)]['comments']):
if 'New Package CVS Request' in j['text']:
type = 'new'
break
if 'Package Change Request' in j['text']:
type = 'change'
break
firstfound = False
else:
if not process_no_request(i, comments['bugs'][str(i.bug_id)]['comments']):
break
if type == 'new':
(package, more) = process_new_request(i, j, comments['bugs'][str(i.bug_id)]['comments'], firstfound, pkgdb, branches)
if package:
processed.append(package)
if not more:
break
elif type == 'change':
(package, more) = process_change_request(i, j, comments['bugs'][str(i.bug_id)]['comments'], firstfound, pkgdb, branches)
if package:
processed.append(package)
if not more:
break
if len(processed):
print '\nYou must now run this on the git server\nto set up the git repository:'
print '/usr/local/bin/pkgdb2branch.py ' + ' '.join(processed)
sys.exit(0)

160
roles/distgit/files/redirect.conf Normal file
View file

@ -0,0 +1,160 @@
RedirectMatch permanent ^/$ http://pkgs.fedoraproject.org/cgit/
RewriteEngine on
# rewrite log
# set level to 0 to turn off and speed things up
# when debugging - set to 3 and you can see what the hell is going on
RewriteLogLevel 0
RewriteLog "/var/log/httpd/git-rewrite.log"
# Use cgit and redirect (some) old gitweb-caching things
RewriteRule ^/cgit-data/(.*)$ /cgit-data/$1 [L,PT]
RewriteRule ^/cgit/(.*)$ /cgit/$1 [L,PT]
# blob
RewriteCond %{REQUEST_URI} /(.+)(\.git)/blob/(.+)/(.+):/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=blob;h=(.+);hb=(.+);f=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tree/%5?id=%3;id2=%4 [R,L,NE]
RewriteCond %{REQUEST_URI} /(.+)(\.git)/blob/(.+):/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=blob;hb=(.+);f=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tree/%4?id=%3 [R,L,NE]
RewriteCond %{query_string} p=(.+)(\.git);a=blob;f=(.+);h=(.+);hb=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tree/%3?id=%4;id2=%5 [R,L,NE]
RewriteCond %{query_string} p=(.+)(\.git);a=blob;f=(.+);h=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tree/%3?id=%4 [R,L,NE]
# tree
RewriteCond %{REQUEST_URI} /(.+)(\.git)/tree/(.+)/(.+):/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=tree;h=(.+);hb=(.+);f=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tree/%5?id=%4 [R,L,NE]
RewriteCond %{REQUEST_URI} /(.+)(\.git)/tree/(.+):/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=tree;hb=(.+);f=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tree/%4?id=%3 [R,L,NE]
RewriteCond %{REQUEST_URI} /(.+)(\.git)/tree/(.+)/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=tree;h=(.+);hb=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tree/?id=%4 [R,L,NE]
RewriteCond %{REQUEST_URI} /(.+)(\.git)/tree/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=tree;hb=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tree/?id=%3 [R,L,NE]
RewriteCond %{REQUEST_URI} /(.+)(\.git)/tree [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=tree
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tree/? [R,L,NE]
# commitdiff
RewriteCond %{REQUEST_URI} /(.+)(\.git)/commitdiff/(.+)/(.+):/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=blobdiff;h=(.+);hp=(.+);hb=(.+);f=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/diff/%6?id2=%4;id=%3;id3=%5 [R,L,NE]
RewriteCond %{REQUEST_URI} /(.+)(\.git)/commitdiff/(.+)/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=commitdiff;h=(.+);hp=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/diff/?id=%4;id2=%3 [R,L,NE]
RewriteCond %{REQUEST_URI} /(.+)(\.git)/commitdiff/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=commitdiff;h=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/diff/?id=%3 [R,L,NE]
# commit
RewriteCond %{REQUEST_URI} /(.+)(\.git)/commit/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=commit;h=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/commit/?id=%3 [R,L,NE]
# summary
RewriteCond %{REQUEST_URI} /(.+)(\.git)/summary [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=summary
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/? [R,L,NE]
# shortlog
RewriteCond %{REQUEST_URI} /(.+)(\.git)/shortlog/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=shortlog;h=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/?id=%3 [R,L,NE]
RewriteCond %{REQUEST_URI} /(.+)(\.git)/shortlog [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=shortlog
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/? [R,L,NE]
# log
RewriteCond %{REQUEST_URI} /(.+)(\.git)/log/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=log;h=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/?id=%3 [R,L,NE]
RewriteCond %{REQUEST_URI} /(.+)(\.git)/log [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=log
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log? [R,L,NE]
# history
RewriteCond %{REQUEST_URI} /(.+)(\.git)/history/(.+)/(.+):/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=history;h=(.+);hb=(.+);f=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/%5?id=%4 [R,L,NE]
RewriteCond %{query_string} p=(.+)(\.git);a=history;f=(.+);h=(.+);hb=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/%3?id=%4;id2=%5 [R,L,NE]
RewriteCond %{REQUEST_URI} /(.+)(\.git)/history/(.+):/(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/%4?id=%3 [R,L,NE]
RewriteCond %{query_string} p=(.+)(\.git);a=history;f=(.+);h=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/%3?id=%4 [R,L,NE]
RewriteCond %{REQUEST_URI} /(.+)(\.git)/history/(.+)/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=history;h=(.+);hb=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/?id=%4 [R,L,NE]
RewriteCond %{REQUEST_URI} /(.+)(\.git)/history/(.+):/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=history;hb=(.+);f=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/%4?id=%3 [R,L,NE]
RewriteCond %{REQUEST_URI} /(.+)(\.git)/history/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=history;hb=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/?id=%3 [R,L,NE]
# tag
RewriteCond %{REQUEST_URI} /(.+)(\.git)/tag/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=tag;h=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tag/?id=%3 [R,L,NE]
# blob_plain
RewriteCond %{REQUEST_URI} /(.+)(\.git)/blob_plain/(.+):/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=blob_plain;h=(.+);f=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/plain/%4?id=%3 [R,L,NE]
RewriteCond %{query_string} p=(.+)(\.git);a=blob_plain;f=(.+);hb=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/plain/%3?id2=%4 [R,L,NE]
RewriteCond %{REQUEST_URI} /(.+)(\.git)/blob_plain/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=blob_plain;f=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/plain/%3 [R,L,NE]
# rss|atom
RewriteCond %{REQUEST_URI} /(.+)(\.git)/(rss|atom)/refs/heads/(.+) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=(rss|atom);h=refsheads/(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/atom?h=%4 [R,L,NE]
RewriteCond %{REQUEST_URI} /(.+)(\.git)/(rss|atom) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=(rss|atom)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/atom? [R,L,NE]
# snapshot
RewriteCond %{REQUEST_URI} /(.+)(\.git)/snapshot/(.+)(\.tar\.gz|\.tar\.bz2) [OR]
RewriteCond %{query_string} p=(.+)(\.git);a=snapshot;h=(.+);sf=(.+)
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/snapshot/%3.tar.gz [R,L,NE]
# base old gitweb project
RewriteCond %{REQUEST_URI} /gitweb/(.+)\.git.* [OR]
RewriteCond %{query_string} p=(.+)\.git.*
RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/? [R,L,NE]
# Fail safes incase nothing above matches, try at least to put the person in the project
#RewriteCond %{REQUEST_URI} /(.+)\.git.* [OR]
#RewriteCond %{query_string} p=(.+)\.git.*
#RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/? [R,L,NE]
# Or else in the root of cgit
#RewriteRule ^.* http://pkgs.fedoraproject.org/cgit/ [R,L,NE]

125
roles/distgit/files/setup_git_package Normal file
View file

@ -0,0 +1,125 @@
#!/bin/bash
#
# Create a new repo.
# THIS HAS TO BE RUN ON THE GIT SERVER!
# WARNING:
# This file is maintained within ansible
# All local changes will be lost.
# Figure out the environment we're running in
GITROOT=/srv/git/rpms
# check if a moron is driving me
if [ ! -d $GITROOT ] ; then
# we're not on the git server (this check is fragile)
echo "ERROR: This script has to be run on the git server."
echo "ERROR: Homer sez 'Duh'."
exit -9
fi
# Local variables
VERBOSE=0
TEST=
IGNORE=
AUTHOR="Fedora Release Engineering <rel-eng@lists.fedoraproject.org>"
GIT_SSH_URL="ssh://localhost"
Usage() {
cat <<EOF
Usage:
$0 <package_name>
Creates a new repo for <package_name>
Options:
-h,--help This help message
EOF
}
if [ $# -gt 2 ]; then
Usage
exit -1
fi
# parse the arguments
while [ -n "$1" ] ; do
case "$1" in
-h | --help )
Usage
exit 0
;;
* )
PACKAGE="$1"
;;
esac
shift
done
# I hate shell scripting. I'm sure the above is totally wrong
# check the arguments
if [ -z "$PACKAGE" ] ; then
Usage
exit -1
fi
# Sanity checks before we start doing damage
[ $VERBOSE -gt 1 ] && echo "Checking package $PACKAGE..."
if [ -d $GITROOT/$PACKAGE.git ] ; then
echo "ERROR: Package module $PACKAGE already exists!" >&2
exit -1
fi
# Just don't run as root, mmkey?
if [ "$(id -un)" = "root" ] ; then
echo "Please run this script as yourself"
exit -3
fi
# "global" permissions check
if [ ! -w $GITROOT ] ; then
echo "ERROR: You can not write to $GITROOT"
echo "ERROR: You can not create repos"
exit -1
fi
# Now start working on creating those branches
# Create a tmpdir to do some git work in
TMPDIR=$(mktemp -d /tmp/tmpXXXXXX)
# First create the master repo
mkdir $GITROOT/$PACKAGE.git
pushd $GITROOT/$PACKAGE.git >/dev/null
git init -q --shared --bare
echo "$PACKAGE" > description # This is used to figure out who to send mail to.
git config --add hooks.mailinglist "$PACKAGE-owner@fedoraproject.org,scm-commits@lists.fedoraproject.org"
git config --add hooks.maildomain fedoraproject.org
popd >/dev/null
# Now clone that repo and create the .gitignore and sources file
git clone -q /srv/git/rpms/$PACKAGE.git $TMPDIR/$PACKAGE
pushd $TMPDIR/$PACKAGE >/dev/null
touch .gitignore sources
git add .
git commit -q -m 'Initial setup of the repo' --author "$AUTHOR"
git push -q origin master
popd >/dev/null
# Put our special update hooks in place
ln -s /usr/share/gitolite/hooks/common/update $GITROOT/$PACKAGE.git/hooks/
mkdir -p $GITROOT/$PACKAGE.git/hooks/post-receive-chained.d
ln -s /usr/share/git-core/mail-hooks/gnome-post-receive-email \
$GITROOT/$PACKAGE.git/hooks/post-receive-chained.d/post-receive-email
ln -s /usr/share/git-core/post-receive-fedmsg \
$GITROOT/$PACKAGE.git/hooks/post-receive-chained.d/post-receive-fedmsg
# This one kicks off all the others in post-receive-chained.d
ln -s /usr/share/git-core/post-receive-chained \
$GITROOT/$PACKAGE.git/hooks/post-receive
rm -rf $TMPDIR
echo "Done."

16
roles/distgit/files/ssl.conf Normal file
View file

@ -0,0 +1,16 @@
#
# This is the Apache server configuration file providing SSL support.
# It contains the configuration directives to instruct the server how to
# serve pages over an https connection. For detailing information about these
# directives see <URL:http://httpd.apache.org/docs-2.0/mod/mod_ssl.html>
#
# For the moment, see <URL:http://www.modssl.org/docs/> for this info.
# The documents are still being prepared from material donated by the
# modssl project.
#
# Do NOT simply read the instructions in here without understanding
# what they do. They're here only as hints or reminders. If you are unsure
# consult the online docs. You have been warned.
#
LoadModule ssl_module modules/mod_ssl.so

16
roles/distgit/files/updatecrl.sh Normal file
View file

@ -0,0 +1,16 @@
#!/bin/bash
URL=https://admin.fedoraproject.org/ca/crl.pem
OLD=/etc/pki/tls/crl.pem
NEW=/tmp/crl.pem
wget $URL -O $NEW
OLDUPDATE=`openssl crl -in $OLD -noout -lastupdate`
NEWUPDATE=`openssl crl -in $NEW -noout -lastupdate`
if [ "$OLDUPDATE" != "$NEWUPDATE" ]; then
mv $NEW $OLD
restorecon $OLD
/etc/init.d/httpd graceful
echo "updated to $NEWUPDATE"
fi

147
roles/distgit/tasks/main.yml Normal file
View file

@ -0,0 +1,147 @@
---
# tasklist for setting up Dist Git
#
# This is a bit complex, so I'm dividing it into sections.
# -- Common ----------------------------------------------
# This is very basic stuff that is needed by multiple of the next sections.
- name: install the needed packages
yum: pkg={{item}} state=present
with_items:
- git
- httpd
- mod_ssl
- python-fedmsg-genacls
- name: set some sysctl options
sysctl: name=vm.zone_reclaim_mode value=1 state=present
when: env != "staging"
- name: create the packager group
group: name=packager gid=415 state=present
- name: install the httpd config directory
copy: src=pkgs.fedoraproject.org.conf dest=/etc/httpd/conf.d/pkgs.fedoraproject.org.conf
file: dest=/etc/httpd/conf.d/pkgs.fedoraproject.org state=directory
notify:
- restart httpd
- name: install the mod_ssl configuration
copy: src=ssl.conf dest=/etc/httpd/conf.d/ssl.conf
notify:
- restart httpd
- name: allow httpd to access the files on NFS
seboolean: name=httpd_use_nfs state=yes persistent=yes
# -- Dist Git --------------------------------------------
# This is the Git setup itself: group, root directory, scripts,...
#
# Requires: roles/git/hooks
# Requires: roles/git/make_checkout_seed
# Requires: roles/git/server
- name: create the distgit root directory)
file: dest=/srv/git state=directory mode=0755
file: dest=/srv/git/rpms state=directory mode=2775 group=packager
- name: install the distgit scripts
copy: src={{item}} dest=/usr/local/bin/{{item}} owner=root group=root mode=0755
with_items:
- setup_git_package
- mkbranch
- pkgdb2-clone
- pkgdb2branch.py
- process-git-requests
- name: install the Dist Git-related httpd config
copy: src=git-smart-http.conf dest=/etc/httpd/conf.d/pkgs.fedoraproject.org/git-smart-http.conf
notify:
- restart httpd
# -- Gitolite --------------------------------------------
# This is the permission management for package maintainers, using Gitolite.
#
# Requires: roles/fedmsg/base
# Requires: roles/fedmsg/hub
# Requires: roles/gitolite/base
# Requires: roles/gitolite/check_fedmsg_hooks
- name: mount the lookaside path
mount: >
src=vtap-fedora-nfs01.storage.phx2.redhat.com:/vol/fedora_sourcecache
name=/srv/cache/lookaside
fstype=nfs
opts=rw,hard,bg,intr,noatime,nodev,nosuid,nfsvers=3
state=mounted
when: env != "staging"
- name: create the /var/log/gitolite directory
file: path=/var/log/gitolite owner=root group=packager state=directory mode=2775
- name: create the gen-acls user
group: name=gen-acls gid=417 state=present
user: name=gen-acls comment="dummy system account for the gen-acls fedmsg job" uid=417 group=gen-acls shell=/bin/bash home=/
- name: create the /etc/gitolite/conf directory
file: path=/etc/gitolite/conf owner=gen-acls group=gen-acls state=directory mode=0755
- name: create /etc/gitolite/gitolite.rc
copy: src=gitolite.rc dest=/etc/gitolite/gitolite.rc owner=root group=root mode=0755
- name: install the gitolite scripts
copy: src={{item}} dest=/usr/local/bin/{{item}} mode=0755
with_items:
- genacls.pkgdb
- genacls.sh
- name: install the fedmsg configuration
copy: src=fedmsg-genacls-config.py dest=/etc/fedmsg.d/genacls.py owner=root group=root mode=0644
# -- CGit ------------------------------------------------
# This is the pretty web view of the repositories, using CGit.
#
# Requires: roles/cgit/base
# Requires: roles/cgit/clean_lock_cron
# Requires: roles/cgit/make_pkgs_list
- name: install the cgitrc file
copy: src=cgitrc dest=/etc/cgitrc
- name: install the CGit-related httpd config
copy: src=redirect.conf dest=/etc/httpd/conf.d/pkgs.fedoraproject.org/redirect.conf
notify:
- restart httpd
# -- Lookaside Cache -------------------------------------
# This is the annex to Dist Git, where we host source tarballs.
#
# Requires: clamav
- name: install the Lookaside Cache httpd configs
copy: src={{item}} dest=/etc/httpd/conf.d/pkgs.fedoraproject.org/{{item}}
with_items:
- lookaside.conf
- lookaside-upload.conf
notify:
- restart httpd
- name: create the Lookaside Cache root directory
file: dest=/srv/cache/lookaside/pkgs state=directory
- name: install the certificates
copy: src={{private}}/fedora-ca.cert dest=/etc/httpd/conf/cacert.pem
copy: src={{private}}/pkgs.fedoraproject.org_key_and_cert.pem dest=/etc/httpd/conf/pkgs.fedoraproject.org_key_and_cert.pem owner=apache mode=0400
- name: install, run, and schedule the updatecrl.sh script
copy: src=updatecrl.sh dest=/usr/local/bin/updatecrl.sh owner=root mode=755
command: /usr/local/bin/updatecrl.sh creates=/etc/pki/tls/crl.pem
cron: >
name="updatecrl" cron_file="ansible-updatecrl"
minute=0
user=root
job="/usr/local/bin/updatecrl.sh"
- name: install the upload CGI script
copy: src=dist-git-upload.cgi dest=/srv/web/upload.cgi owner=root group=root mode=0755
notify:
- restart httpd