From 3fc69696b95254c07d561971750e907cd8ccf7f0 Mon Sep 17 00:00:00 2001 From: clime Date: Thu, 13 Jul 2017 21:36:07 +0200 Subject: [PATCH] dist-git: clean up unused files and the main script --- roles/distgit/files/dist-git-upload.cgi | 265 ------------------------ roles/distgit/files/mkbranch | 158 -------------- roles/distgit/files/mkbranch_branching | 158 -------------- roles/distgit/tasks/main.yml | 31 --- 4 files changed, 612 deletions(-) delete mode 100644 roles/distgit/files/dist-git-upload.cgi delete mode 100644 roles/distgit/files/mkbranch delete mode 100644 roles/distgit/files/mkbranch_branching diff --git a/roles/distgit/files/dist-git-upload.cgi b/roles/distgit/files/dist-git-upload.cgi deleted file mode 100644 index 7f7ba315a2..0000000000 --- a/roles/distgit/files/dist-git-upload.cgi +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/python -# -# CGI script to handle file updates for the rpms git repository. There -# is nothing really complex here other than tedious checking of our -# every step along the way... -# -# License: GPL - -import cgi -import errno -import grp -import hashlib -import os -import sys -import tempfile - -import fedmsg -import fedmsg.config - - -# Reading buffer size -BUFFER_SIZE = 4096 - -# We check modules exist from this dircetory -GITREPO = '/srv/git/repositories' - -# Lookaside cache directory -CACHE_DIR = '/srv/cache/lookaside/pkgs' - -# Fedora Packager Group -PACKAGER_GROUP = 'packager' - - -def send_error(text, status='500 Internal Server Error'): - """Send an error back to the client - - This ensures that the client will get a proper error, including the HTTP - status code, so that it can handle problems appropriately. - - Args: - text (str): The error message to send the client - status (str, optional): The HTTP status code to return to the client. - """ - print 'Status: %s' % status - print 'Content-type: text/plain' - print - print text - sys.exit(0) - - -def check_form(form, var): - ret = form.getvalue(var, None) - if ret is None: - send_error('Required field "%s" is not present.' % var, - status='400 Bad Request') - if isinstance(ret, list): - send_error('Multiple values given for "%s". Aborting.' % var, - status='400 Bad Request') - return ret - - -def check_auth(username): - authenticated = False - try: - if username in grp.getgrnam(PACKAGER_GROUP)[3]: - authenticated = True - except KeyError: - pass - return authenticated - - -def hardlink(src, dest, username): - makedirs(os.path.dirname(dest), username) - - try: - os.link(src, dest) - - except OSError as e: - if e.errno != errno.EEXIST: - send_error(str(e)) - - # The file already existed at the dest path, hardlink over it - os.unlink(dest) - os.link(src, dest) - - sys.stderr.write("[username=%s] ln %s %s\n" % (username, src, dest)) - - -def makedirs(dir_, username, mode=02755): - try: - os.makedirs(dir_, mode=mode) - sys.stderr.write('[username=%s] mkdir %s\n' % (username, dir_)) - - except OSError as e: - if e.errno != errno.EEXIST: - send_error(str(e)) - - -def main(): - os.umask(002) - - username = os.environ.get('SSL_CLIENT_S_DN_CN', None) - gssname = os.environ.get('GSS_NAME', None) - if gssname and '@' in gssname and not username: - username = gssname.partition('@')[0] - if not check_auth(username): - send_error('You must connect with a valid certificate and be in the ' - '%s group to upload.' % PACKAGER_GROUP, - status='403 Forbidden') - - print 'Content-Type: text/plain' - print - - assert os.environ['REQUEST_URI'].split('/')[1] == 'repo' - - form = cgi.FieldStorage() - name = check_form(form, 'name') - - # Search for the file hash, start with stronger hash functions - if 'sha512sum' in form: - checksum = check_form(form, 'sha512sum') - hash_type = "sha512" - - elif 'md5sum' in form: - # Fallback on md5, as it's what we currently use - checksum = check_form(form, 'md5sum') - hash_type = "md5" - - else: - send_error('Required checksum is not present.', - status='400 Bad Request') - - action = None - upload_file = None - filename = None - - # Is this a submission or a test? - # in a test, we don't get a file, just a filename. - # In a submission, we don;t get a filename, just the file. - if 'filename' in form: - action = 'check' - filename = check_form(form, 'filename') - filename = os.path.basename(filename) - sys.stderr.write('[username=%s] Checking file status: NAME=%s ' - 'FILENAME=%s %sSUM=%s\n' % (username, name, filename, - hash_type.upper(), - checksum)) - else: - action = 'upload' - if 'file' in form: - upload_file = form['file'] - if not upload_file.file: - send_error('No file given for upload. Aborting.', - status='400 Bad Request') - filename = os.path.basename(upload_file.filename) - else: - send_error('Required field "file" is not present.', - status='400 Bad Request') - - sys.stderr.write('[username=%s] Processing upload request: ' - 'NAME=%s FILENAME=%s %sSUM=%s\n' % ( - username, name, filename, hash_type.upper(), - checksum)) - - module_dir = os.path.join(CACHE_DIR, name) - hash_dir = os.path.join(module_dir, filename, hash_type, checksum) - msgpath = os.path.join(name, filename, hash_type, checksum, filename) - - # first test if the module really exists - git_dir = os.path.join(GITREPO, '%s.git' % name) - if not os.path.isdir(git_dir): - sys.stderr.write('[username=%s] Unknown module: %s' % (username, name)) - send_error('Module "%s" does not exist!' % name, - status='404 Not Found') - - # try to see if we already have this file... - dest_file = os.path.join(hash_dir, filename) - old_dir = os.path.join(module_dir, filename, checksum) - old_path = os.path.join(old_dir, filename) - - if os.path.exists(dest_file): - if action == 'check': - print 'Available' - else: - upload_file.file.close() - dest_file_stat = os.stat(dest_file) - print 'File %s already exists' % filename - print 'File: %s Size: %d' % (dest_file, dest_file_stat.st_size) - sys.exit(0) - elif action == 'check': - if os.path.exists(old_path): - # The file had been uploaded at the old path - hardlink(old_path, dest_file, username) - print 'Available' - else: - print 'Missing' - - sys.exit(0) - - # check that all directories are in place - makedirs(module_dir, username) - - # grab a temporary filename and dump our file in there - tempfile.tempdir = module_dir - tmpfile = tempfile.mkstemp(checksum)[1] - tmpfd = open(tmpfile, 'w') - - # now read the whole file in - m = getattr(hashlib, hash_type)() - filesize = 0 - while True: - data = upload_file.file.read(BUFFER_SIZE) - if not data: - break - tmpfd.write(data) - m.update(data) - filesize += len(data) - - # now we're done reading, check the checksum of what we got - tmpfd.close() - check_checksum = m.hexdigest() - if checksum != check_checksum: - os.unlink(tmpfile) - send_error("%s check failed. Received %s instead of %s." % - (hash_type.upper(), check_checksum, checksum), - status='400 Bad Request') - - # wow, even the checksum matches. make sure full path is valid now - makedirs(hash_dir, username) - os.rename(tmpfile, dest_file) - os.chmod(dest_file, 0644) - - sys.stderr.write('[username=%s] Stored %s (%d bytes)' % (username, - dest_file, - filesize)) - print 'File %s size %d %s %s stored OK' % (filename, filesize, - hash_type.upper(), checksum) - - # Add the file to the old path, where fedpkg is currently looking for it - if hash_type == "md5": - hardlink(dest_file, old_path, username) - - # Emit a fedmsg message. Load the config to talk to the fedmsg-relay. - try: - config = fedmsg.config.load_config([], None) - config['active'] = True - config['endpoints']['relay_inbound'] = config['relay_inbound'] - fedmsg.init(name="relay_inbound", cert_prefix="lookaside", **config) - - topic = "lookaside.new" - msg = dict(name=name, md5sum=checksum, - filename=filename.split('/')[-1], agent=username, - path=msgpath) - fedmsg.publish(modname="git", topic=topic, msg=msg) - except Exception as e: - print "Error with fedmsg", str(e) - -if __name__ == '__main__': - try: - main() - - except Exception as e: - import traceback - sys.stderr.write('%s\n' % traceback.format_exc()) - send_error(str(e)) diff --git a/roles/distgit/files/mkbranch b/roles/distgit/files/mkbranch deleted file mode 100644 index bde1f1e782..0000000000 --- a/roles/distgit/files/mkbranch +++ /dev/null @@ -1,158 +0,0 @@ -#!/bin/bash -# -# Create a new development branch for a module. -# THIS HAS TO BE RUN ON THE GIT SERVER! - -# WARNING: -# This file is maintained within puppet? -# All local changes will be lost. - - -# Figure out the environment we're running in -RUNDIR=$(cd $(dirname $0) && pwd) -GITROOT=/srv/git/repositories - -# check if a moron is driving me -if [ ! -d $GITROOT ] ; then - # we're not on the git server (this check is fragile) - echo "ERROR: This script has to be run on the git server." - echo "ERROR: Homer sez 'Duh'." - exit -9 -fi - -# where are the packages kept -TOPLEVEL=rpms - -# Local variables -VERBOSE=0 -TEST= -IGNORE= -BRANCH="" -PACKAGES="" -SRC_BRANCH="master" -AUTHOR="Fedora Release Engineering " - -Usage() { - cat <] ... - - Creates a new branch for the list of s. - The /master suffix on branch names is assumed. - -Options: - -n,--test Don't do nothing, only test - -i,--ignore Ignore erroneous modules - -h,--help This help message - -v,--verbose Increase verbosity -EOF -} - -# parse the arguments -while [ -n "$1" ] ; do - case "$1" in - -h | --help ) - Usage - exit 0 - ;; - - -v | --verbose ) - VERBOSE=$(($VERBOSE + 1)) - ;; - - -i | --ignore ) - IGNORE="yes" - ;; - - -n | --test ) - TEST="yes" - ;; - - - -b | --branch ) - shift - BRANCH=$1/master - ;; - - * ) - if [ -z "$BRANCH" ] ; then - BRANCH="$1" - else - PACKAGES="$PACKAGES $1" - fi - ;; - esac - shift -done - -# check the arguments -if [ -z "$BRANCH" -o -z "$PACKAGES" ] ; then - Usage - exit -1 -fi - - -# Sanity checks before we start doing damage -NEWP= -for p in $PACKAGES ; do - [ $VERBOSE -gt 1 ] && echo "Checking package $p..." - if [ ! -d $GITROOT/$p.git ] ; then - echo "ERROR: Package module $p is invalid" >&2 - [ "$IGNORE" = "yes" ] && continue || exit -1 - fi - $(GIT_DIR=$GITROOT/$p.git git rev-parse -q --verify \ - $BRANCH >/dev/null) && \ - (echo "IGNORING: Package module $p already has a branch $BRANCH" >&2; \ - [ "$IGNORE" = "yes" ] && continue || exit -1) - NEWP="$NEWP $p" -done -PACKAGES="$(echo $NEWP)" -if [ -z "$PACKAGES" ] ; then - echo "NOOP: no valid packages found to process" - exit -1 -fi - -if [ -n "$TEST" ] ; then - echo "Branch $BRANCH valid for $PACKAGES" - exit 0 -fi - -# "global" permissions check -if [ ! -w $GITROOT ] ; then - echo "ERROR: You can not write to $GITROOT" - echo "ERROR: You can not perform branching operations" - exit -1 -fi - -# Now start working on creating those branches - -# For every module, "create" the branch -for NAME in $PACKAGES ; do - echo - echo "Creating new module branch '$BRANCH' for '$NAME'..." - - # permissions checks for this particular module - if [ ! -w $GITROOT/$NAME.git/refs/heads/ ] ; then - echo "ERROR: You can not write to $d" - echo "ERROR: $NAME can not be branched by you" - continue - fi - #### Replace the above with a gitolite permission check - #[ $VERBOSE -gt 0 ] && echo "Creating $BRANCH-split tag for $NAME/$SRC_BRANCH..." - # Is the above needed? - #cvs -Q rtag -f "$BRANCH-split" $TOPLEVEL/$NAME/$SRC_BRANCH || { - #echo "ERROR: Branch split tag for $NAME/$SRC_BRANCH could not be created" >&2 - #exit -2 - #} - [ $VERBOSE -gt 0 ] && echo "Creating $NAME $BRANCH from $NAME ..." - $(pushd $GITROOT/$NAME.git >/dev/null && \ - git branch --no-track $BRANCH `git rev-list --max-parents=0 master | head -1` && \ - popd >/dev/null) || { - echo "ERROR: Branch $NAME $BRANCH could not be created" >&2 - popd >/dev/null - exit -2 - } -done - -echo -echo "Done." diff --git a/roles/distgit/files/mkbranch_branching b/roles/distgit/files/mkbranch_branching deleted file mode 100644 index 14b0c21aeb..0000000000 --- a/roles/distgit/files/mkbranch_branching +++ /dev/null @@ -1,158 +0,0 @@ -#!/bin/bash -# -# Create a new development branch for a module. -# THIS HAS TO BE RUN ON THE GIT SERVER! - -# WARNING: -# This file is maintained within puppet? -# All local changes will be lost. - - -# Figure out the environment we're running in -RUNDIR=$(cd $(dirname $0) && pwd) -GITROOT=/srv/git/repositories - -# check if a moron is driving me -if [ ! -d $GITROOT ] ; then - # we're not on the git server (this check is fragile) - echo "ERROR: This script has to be run on the git server." - echo "ERROR: Homer sez 'Duh'." - exit -9 -fi - -# where are the packages kept -TOPLEVEL=rpms - -# Local variables -VERBOSE=0 -TEST= -IGNORE= -BRANCH="" -PACKAGES="" -SRC_BRANCH="master" -AUTHOR="Fedora Release Engineering " - -Usage() { - cat <] ... - - Creates a new branch for the list of s. - The /master suffix on branch names is assumed. - -Options: - -n,--test Don't do nothing, only test - -i,--ignore Ignore erroneous modules - -h,--help This help message - -v,--verbose Increase verbosity -EOF -} - -# parse the arguments -while [ -n "$1" ] ; do - case "$1" in - -h | --help ) - Usage - exit 0 - ;; - - -v | --verbose ) - VERBOSE=$(($VERBOSE + 1)) - ;; - - -i | --ignore ) - IGNORE="yes" - ;; - - -n | --test ) - TEST="yes" - ;; - - - -b | --branch ) - shift - BRANCH=$1/master - ;; - - * ) - if [ -z "$BRANCH" ] ; then - BRANCH="$1" - else - PACKAGES="$PACKAGES $1" - fi - ;; - esac - shift -done - -# check the arguments -if [ -z "$BRANCH" -o -z "$PACKAGES" ] ; then - Usage - exit -1 -fi - - -# Sanity checks before we start doing damage -NEWP= -for p in $PACKAGES ; do - [ $VERBOSE -gt 1 ] && echo "Checking package $p..." - if [ ! -d $GITROOT/$p.git ] ; then - echo "ERROR: Package module $p is invalid" >&2 - [ "$IGNORE" = "yes" ] && continue || exit -1 - fi - $(GIT_DIR=$GITROOT/$p.git git rev-parse -q --verify \ - $BRANCH >/dev/null) && \ - (echo "IGNORING: Package module $p already has a branch $BRANCH" >&2; \ - [ "$IGNORE" = "yes" ] && continue || exit -1) - NEWP="$NEWP $p" -done -PACKAGES="$(echo $NEWP)" -if [ -z "$PACKAGES" ] ; then - echo "NOOP: no valid packages found to process" - exit -1 -fi - -if [ -n "$TEST" ] ; then - echo "Branch $BRANCH valid for $PACKAGES" - exit 0 -fi - -# "global" permissions check -if [ ! -w $GITROOT ] ; then - echo "ERROR: You can not write to $GITROOT" - echo "ERROR: You can not perform branching operations" - exit -1 -fi - -# Now start working on creating those branches - -# For every module, "create" the branch -for NAME in $PACKAGES ; do - echo - echo "Creating new module branch '$BRANCH' for '$NAME'..." - - # permissions checks for this particular module - if [ ! -w $GITROOT/$NAME.git/refs/heads/ ] ; then - echo "ERROR: You can not write to $d" - echo "ERROR: $NAME can not be branched by you" - continue - fi - #### Replace the above with a gitolite permission check - #[ $VERBOSE -gt 0 ] && echo "Creating $BRANCH-split tag for $NAME/$SRC_BRANCH..." - # Is the above needed? - #cvs -Q rtag -f "$BRANCH-split" $TOPLEVEL/$NAME/$SRC_BRANCH || { - #echo "ERROR: Branch split tag for $NAME/$SRC_BRANCH could not be created" >&2 - #exit -2 - #} - [ $VERBOSE -gt 0 ] && echo "Creating $NAME $BRANCH from $NAME ..." - $(pushd $GITROOT/$NAME.git >/dev/null && \ - git branch --no-track $BRANCH `git rev-list master | head -1` && \ - popd >/dev/null) || { - echo "ERROR: Branch $NAME $BRANCH could not be created" >&2 - popd >/dev/null - exit -2 - } -done - -echo -echo "Done." diff --git a/roles/distgit/tasks/main.yml b/roles/distgit/tasks/main.yml index 752844a94c..bad9e62ba0 100644 --- a/roles/distgit/tasks/main.yml +++ b/roles/distgit/tasks/main.yml @@ -17,20 +17,6 @@ tags: - distgit -- name: uninstall the httpd config file of non-packaged dist-git - file: dest=/etc/httpd/conf.d/pkgs.fedoraproject.org.conf state=absent - notify: - - reload httpd - tags: - - distgit - -- name: uninstall the httpd config directory of non-packaged dist-git - file: dest=/etc/httpd/conf.d/pkgs.fedoraproject.org state=absent - notify: - - reload httpd - tags: - - distgit - - name: install the mod_ssl configuration copy: src=ssl.conf dest=/etc/httpd/conf.d/ssl.conf notify: @@ -123,16 +109,6 @@ tags: - distgit -- name: uninstall the distgit scripts of non-packaged dist-git - file: dest=/usr/local/bin/{{item}} state=absent - with_items: - - setup_git_package - - mkbranch - - mkbranch_branching - tags: - - config - - distgit - - name: install the DistGit related httpd config copy: src=git-smart-http.conf dest=/etc/httpd/conf.d/dist-git/git-smart-http.conf notify: @@ -437,13 +413,6 @@ tags: - distgit -- name: uninstall the upload CGI script of non-packaged dist-git - file: dest=/srv/web/upload.cgi state=absent - notify: - - reload httpd - tags: - - distgit - # Three tasks for handling our selinux policy for upload.cgi - name: ensure a directory exists for our SELinux policy file: dest=/usr/local/share/selinux/ state=directory