dist-git: clean up unused files and the main script

This commit is contained in:
clime 2017-07-13 21:36:07 +02:00
parent 5de4b9a026
commit 3fc69696b9
4 changed files with 0 additions and 612 deletions

View file

@ -1,265 +0,0 @@
#!/usr/bin/python
#
# CGI script to handle file updates for the rpms git repository. There
# is nothing really complex here other than tedious checking of our
# every step along the way...
#
# License: GPL
import cgi
import errno
import grp
import hashlib
import os
import sys
import tempfile
import fedmsg
import fedmsg.config
# Reading buffer size
BUFFER_SIZE = 4096
# We check modules exist from this dircetory
GITREPO = '/srv/git/repositories'
# Lookaside cache directory
CACHE_DIR = '/srv/cache/lookaside/pkgs'
# Fedora Packager Group
PACKAGER_GROUP = 'packager'
def send_error(text, status='500 Internal Server Error'):
"""Send an error back to the client
This ensures that the client will get a proper error, including the HTTP
status code, so that it can handle problems appropriately.
Args:
text (str): The error message to send the client
status (str, optional): The HTTP status code to return to the client.
"""
print 'Status: %s' % status
print 'Content-type: text/plain'
print
print text
sys.exit(0)
def check_form(form, var):
ret = form.getvalue(var, None)
if ret is None:
send_error('Required field "%s" is not present.' % var,
status='400 Bad Request')
if isinstance(ret, list):
send_error('Multiple values given for "%s". Aborting.' % var,
status='400 Bad Request')
return ret
def check_auth(username):
authenticated = False
try:
if username in grp.getgrnam(PACKAGER_GROUP)[3]:
authenticated = True
except KeyError:
pass
return authenticated
def hardlink(src, dest, username):
makedirs(os.path.dirname(dest), username)
try:
os.link(src, dest)
except OSError as e:
if e.errno != errno.EEXIST:
send_error(str(e))
# The file already existed at the dest path, hardlink over it
os.unlink(dest)
os.link(src, dest)
sys.stderr.write("[username=%s] ln %s %s\n" % (username, src, dest))
def makedirs(dir_, username, mode=02755):
try:
os.makedirs(dir_, mode=mode)
sys.stderr.write('[username=%s] mkdir %s\n' % (username, dir_))
except OSError as e:
if e.errno != errno.EEXIST:
send_error(str(e))
def main():
os.umask(002)
username = os.environ.get('SSL_CLIENT_S_DN_CN', None)
gssname = os.environ.get('GSS_NAME', None)
if gssname and '@' in gssname and not username:
username = gssname.partition('@')[0]
if not check_auth(username):
send_error('You must connect with a valid certificate and be in the '
'%s group to upload.' % PACKAGER_GROUP,
status='403 Forbidden')
print 'Content-Type: text/plain'
print
assert os.environ['REQUEST_URI'].split('/')[1] == 'repo'
form = cgi.FieldStorage()
name = check_form(form, 'name')
# Search for the file hash, start with stronger hash functions
if 'sha512sum' in form:
checksum = check_form(form, 'sha512sum')
hash_type = "sha512"
elif 'md5sum' in form:
# Fallback on md5, as it's what we currently use
checksum = check_form(form, 'md5sum')
hash_type = "md5"
else:
send_error('Required checksum is not present.',
status='400 Bad Request')
action = None
upload_file = None
filename = None
# Is this a submission or a test?
# in a test, we don't get a file, just a filename.
# In a submission, we don;t get a filename, just the file.
if 'filename' in form:
action = 'check'
filename = check_form(form, 'filename')
filename = os.path.basename(filename)
sys.stderr.write('[username=%s] Checking file status: NAME=%s '
'FILENAME=%s %sSUM=%s\n' % (username, name, filename,
hash_type.upper(),
checksum))
else:
action = 'upload'
if 'file' in form:
upload_file = form['file']
if not upload_file.file:
send_error('No file given for upload. Aborting.',
status='400 Bad Request')
filename = os.path.basename(upload_file.filename)
else:
send_error('Required field "file" is not present.',
status='400 Bad Request')
sys.stderr.write('[username=%s] Processing upload request: '
'NAME=%s FILENAME=%s %sSUM=%s\n' % (
username, name, filename, hash_type.upper(),
checksum))
module_dir = os.path.join(CACHE_DIR, name)
hash_dir = os.path.join(module_dir, filename, hash_type, checksum)
msgpath = os.path.join(name, filename, hash_type, checksum, filename)
# first test if the module really exists
git_dir = os.path.join(GITREPO, '%s.git' % name)
if not os.path.isdir(git_dir):
sys.stderr.write('[username=%s] Unknown module: %s' % (username, name))
send_error('Module "%s" does not exist!' % name,
status='404 Not Found')
# try to see if we already have this file...
dest_file = os.path.join(hash_dir, filename)
old_dir = os.path.join(module_dir, filename, checksum)
old_path = os.path.join(old_dir, filename)
if os.path.exists(dest_file):
if action == 'check':
print 'Available'
else:
upload_file.file.close()
dest_file_stat = os.stat(dest_file)
print 'File %s already exists' % filename
print 'File: %s Size: %d' % (dest_file, dest_file_stat.st_size)
sys.exit(0)
elif action == 'check':
if os.path.exists(old_path):
# The file had been uploaded at the old path
hardlink(old_path, dest_file, username)
print 'Available'
else:
print 'Missing'
sys.exit(0)
# check that all directories are in place
makedirs(module_dir, username)
# grab a temporary filename and dump our file in there
tempfile.tempdir = module_dir
tmpfile = tempfile.mkstemp(checksum)[1]
tmpfd = open(tmpfile, 'w')
# now read the whole file in
m = getattr(hashlib, hash_type)()
filesize = 0
while True:
data = upload_file.file.read(BUFFER_SIZE)
if not data:
break
tmpfd.write(data)
m.update(data)
filesize += len(data)
# now we're done reading, check the checksum of what we got
tmpfd.close()
check_checksum = m.hexdigest()
if checksum != check_checksum:
os.unlink(tmpfile)
send_error("%s check failed. Received %s instead of %s." %
(hash_type.upper(), check_checksum, checksum),
status='400 Bad Request')
# wow, even the checksum matches. make sure full path is valid now
makedirs(hash_dir, username)
os.rename(tmpfile, dest_file)
os.chmod(dest_file, 0644)
sys.stderr.write('[username=%s] Stored %s (%d bytes)' % (username,
dest_file,
filesize))
print 'File %s size %d %s %s stored OK' % (filename, filesize,
hash_type.upper(), checksum)
# Add the file to the old path, where fedpkg is currently looking for it
if hash_type == "md5":
hardlink(dest_file, old_path, username)
# Emit a fedmsg message. Load the config to talk to the fedmsg-relay.
try:
config = fedmsg.config.load_config([], None)
config['active'] = True
config['endpoints']['relay_inbound'] = config['relay_inbound']
fedmsg.init(name="relay_inbound", cert_prefix="lookaside", **config)
topic = "lookaside.new"
msg = dict(name=name, md5sum=checksum,
filename=filename.split('/')[-1], agent=username,
path=msgpath)
fedmsg.publish(modname="git", topic=topic, msg=msg)
except Exception as e:
print "Error with fedmsg", str(e)
if __name__ == '__main__':
try:
main()
except Exception as e:
import traceback
sys.stderr.write('%s\n' % traceback.format_exc())
send_error(str(e))

View file

@ -1,158 +0,0 @@
#!/bin/bash
#
# Create a new development branch for a module.
# THIS HAS TO BE RUN ON THE GIT SERVER!
# WARNING:
# This file is maintained within puppet?
# All local changes will be lost.
# Figure out the environment we're running in
RUNDIR=$(cd $(dirname $0) && pwd)
GITROOT=/srv/git/repositories
# check if a moron is driving me
if [ ! -d $GITROOT ] ; then
# we're not on the git server (this check is fragile)
echo "ERROR: This script has to be run on the git server."
echo "ERROR: Homer sez 'Duh'."
exit -9
fi
# where are the packages kept
TOPLEVEL=rpms
# Local variables
VERBOSE=0
TEST=
IGNORE=
BRANCH=""
PACKAGES=""
SRC_BRANCH="master"
AUTHOR="Fedora Release Engineering <rel-eng@lists.fedoraproject.org>"
Usage() {
cat <<EOF
Usage:
$0 [ -s <src_branch>] <branch> <package_name>...
Creates a new branch <branch> for the list of <package_name>s.
The /master suffix on branch names is assumed.
Options:
-n,--test Don't do nothing, only test
-i,--ignore Ignore erroneous modules
-h,--help This help message
-v,--verbose Increase verbosity
EOF
}
# parse the arguments
while [ -n "$1" ] ; do
case "$1" in
-h | --help )
Usage
exit 0
;;
-v | --verbose )
VERBOSE=$(($VERBOSE + 1))
;;
-i | --ignore )
IGNORE="yes"
;;
-n | --test )
TEST="yes"
;;
-b | --branch )
shift
BRANCH=$1/master
;;
* )
if [ -z "$BRANCH" ] ; then
BRANCH="$1"
else
PACKAGES="$PACKAGES $1"
fi
;;
esac
shift
done
# check the arguments
if [ -z "$BRANCH" -o -z "$PACKAGES" ] ; then
Usage
exit -1
fi
# Sanity checks before we start doing damage
NEWP=
for p in $PACKAGES ; do
[ $VERBOSE -gt 1 ] && echo "Checking package $p..."
if [ ! -d $GITROOT/$p.git ] ; then
echo "ERROR: Package module $p is invalid" >&2
[ "$IGNORE" = "yes" ] && continue || exit -1
fi
$(GIT_DIR=$GITROOT/$p.git git rev-parse -q --verify \
$BRANCH >/dev/null) && \
(echo "IGNORING: Package module $p already has a branch $BRANCH" >&2; \
[ "$IGNORE" = "yes" ] && continue || exit -1)
NEWP="$NEWP $p"
done
PACKAGES="$(echo $NEWP)"
if [ -z "$PACKAGES" ] ; then
echo "NOOP: no valid packages found to process"
exit -1
fi
if [ -n "$TEST" ] ; then
echo "Branch $BRANCH valid for $PACKAGES"
exit 0
fi
# "global" permissions check
if [ ! -w $GITROOT ] ; then
echo "ERROR: You can not write to $GITROOT"
echo "ERROR: You can not perform branching operations"
exit -1
fi
# Now start working on creating those branches
# For every module, "create" the branch
for NAME in $PACKAGES ; do
echo
echo "Creating new module branch '$BRANCH' for '$NAME'..."
# permissions checks for this particular module
if [ ! -w $GITROOT/$NAME.git/refs/heads/ ] ; then
echo "ERROR: You can not write to $d"
echo "ERROR: $NAME can not be branched by you"
continue
fi
#### Replace the above with a gitolite permission check
#[ $VERBOSE -gt 0 ] && echo "Creating $BRANCH-split tag for $NAME/$SRC_BRANCH..."
# Is the above needed?
#cvs -Q rtag -f "$BRANCH-split" $TOPLEVEL/$NAME/$SRC_BRANCH || {
#echo "ERROR: Branch split tag for $NAME/$SRC_BRANCH could not be created" >&2
#exit -2
#}
[ $VERBOSE -gt 0 ] && echo "Creating $NAME $BRANCH from $NAME ..."
$(pushd $GITROOT/$NAME.git >/dev/null && \
git branch --no-track $BRANCH `git rev-list --max-parents=0 master | head -1` && \
popd >/dev/null) || {
echo "ERROR: Branch $NAME $BRANCH could not be created" >&2
popd >/dev/null
exit -2
}
done
echo
echo "Done."

View file

@ -1,158 +0,0 @@
#!/bin/bash
#
# Create a new development branch for a module.
# THIS HAS TO BE RUN ON THE GIT SERVER!
# WARNING:
# This file is maintained within puppet?
# All local changes will be lost.
# Figure out the environment we're running in
RUNDIR=$(cd $(dirname $0) && pwd)
GITROOT=/srv/git/repositories
# check if a moron is driving me
if [ ! -d $GITROOT ] ; then
# we're not on the git server (this check is fragile)
echo "ERROR: This script has to be run on the git server."
echo "ERROR: Homer sez 'Duh'."
exit -9
fi
# where are the packages kept
TOPLEVEL=rpms
# Local variables
VERBOSE=0
TEST=
IGNORE=
BRANCH=""
PACKAGES=""
SRC_BRANCH="master"
AUTHOR="Fedora Release Engineering <rel-eng@lists.fedoraproject.org>"
Usage() {
cat <<EOF
Usage:
$0 [ -s <src_branch>] <branch> <package_name>...
Creates a new branch <branch> for the list of <package_name>s.
The /master suffix on branch names is assumed.
Options:
-n,--test Don't do nothing, only test
-i,--ignore Ignore erroneous modules
-h,--help This help message
-v,--verbose Increase verbosity
EOF
}
# parse the arguments
while [ -n "$1" ] ; do
case "$1" in
-h | --help )
Usage
exit 0
;;
-v | --verbose )
VERBOSE=$(($VERBOSE + 1))
;;
-i | --ignore )
IGNORE="yes"
;;
-n | --test )
TEST="yes"
;;
-b | --branch )
shift
BRANCH=$1/master
;;
* )
if [ -z "$BRANCH" ] ; then
BRANCH="$1"
else
PACKAGES="$PACKAGES $1"
fi
;;
esac
shift
done
# check the arguments
if [ -z "$BRANCH" -o -z "$PACKAGES" ] ; then
Usage
exit -1
fi
# Sanity checks before we start doing damage
NEWP=
for p in $PACKAGES ; do
[ $VERBOSE -gt 1 ] && echo "Checking package $p..."
if [ ! -d $GITROOT/$p.git ] ; then
echo "ERROR: Package module $p is invalid" >&2
[ "$IGNORE" = "yes" ] && continue || exit -1
fi
$(GIT_DIR=$GITROOT/$p.git git rev-parse -q --verify \
$BRANCH >/dev/null) && \
(echo "IGNORING: Package module $p already has a branch $BRANCH" >&2; \
[ "$IGNORE" = "yes" ] && continue || exit -1)
NEWP="$NEWP $p"
done
PACKAGES="$(echo $NEWP)"
if [ -z "$PACKAGES" ] ; then
echo "NOOP: no valid packages found to process"
exit -1
fi
if [ -n "$TEST" ] ; then
echo "Branch $BRANCH valid for $PACKAGES"
exit 0
fi
# "global" permissions check
if [ ! -w $GITROOT ] ; then
echo "ERROR: You can not write to $GITROOT"
echo "ERROR: You can not perform branching operations"
exit -1
fi
# Now start working on creating those branches
# For every module, "create" the branch
for NAME in $PACKAGES ; do
echo
echo "Creating new module branch '$BRANCH' for '$NAME'..."
# permissions checks for this particular module
if [ ! -w $GITROOT/$NAME.git/refs/heads/ ] ; then
echo "ERROR: You can not write to $d"
echo "ERROR: $NAME can not be branched by you"
continue
fi
#### Replace the above with a gitolite permission check
#[ $VERBOSE -gt 0 ] && echo "Creating $BRANCH-split tag for $NAME/$SRC_BRANCH..."
# Is the above needed?
#cvs -Q rtag -f "$BRANCH-split" $TOPLEVEL/$NAME/$SRC_BRANCH || {
#echo "ERROR: Branch split tag for $NAME/$SRC_BRANCH could not be created" >&2
#exit -2
#}
[ $VERBOSE -gt 0 ] && echo "Creating $NAME $BRANCH from $NAME ..."
$(pushd $GITROOT/$NAME.git >/dev/null && \
git branch --no-track $BRANCH `git rev-list master | head -1` && \
popd >/dev/null) || {
echo "ERROR: Branch $NAME $BRANCH could not be created" >&2
popd >/dev/null
exit -2
}
done
echo
echo "Done."

View file

@ -17,20 +17,6 @@
tags:
- distgit
- name: uninstall the httpd config file of non-packaged dist-git
file: dest=/etc/httpd/conf.d/pkgs.fedoraproject.org.conf state=absent
notify:
- reload httpd
tags:
- distgit
- name: uninstall the httpd config directory of non-packaged dist-git
file: dest=/etc/httpd/conf.d/pkgs.fedoraproject.org state=absent
notify:
- reload httpd
tags:
- distgit
- name: install the mod_ssl configuration
copy: src=ssl.conf dest=/etc/httpd/conf.d/ssl.conf
notify:
@ -123,16 +109,6 @@
tags:
- distgit
- name: uninstall the distgit scripts of non-packaged dist-git
file: dest=/usr/local/bin/{{item}} state=absent
with_items:
- setup_git_package
- mkbranch
- mkbranch_branching
tags:
- config
- distgit
- name: install the DistGit related httpd config
copy: src=git-smart-http.conf dest=/etc/httpd/conf.d/dist-git/git-smart-http.conf
notify:
@ -437,13 +413,6 @@
tags:
- distgit
- name: uninstall the upload CGI script of non-packaged dist-git
file: dest=/srv/web/upload.cgi state=absent
notify:
- reload httpd
tags:
- distgit
# Three tasks for handling our selinux policy for upload.cgi
- name: ensure a directory exists for our SELinux policy
file: dest=/usr/local/share/selinux/ state=directory