From 382d837d7a88cb554c029826fb60fe82c7c376fa Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Fri, 11 Dec 2015 12:56:15 +0100 Subject: [PATCH] Update pkgdb_sync_git_branches to the new pkgdb API while remaining backward compatible This commit also adds a TEST_ONLY global boolean useful if you want to test the script without modifying anything on the filesystem. --- .../templates/pkgdb_sync_git_branches.py | 207 ++++++++++-------- 1 file changed, 118 insertions(+), 89 deletions(-) diff --git a/roles/distgit/templates/pkgdb_sync_git_branches.py b/roles/distgit/templates/pkgdb_sync_git_branches.py index 70bab3c92c..6199601df7 100644 --- a/roles/distgit/templates/pkgdb_sync_git_branches.py +++ b/roles/distgit/templates/pkgdb_sync_git_branches.py @@ -24,8 +24,21 @@ branches are missing. It even goes one step further but actually adjusting the git repo by adding the missing branches (or even the missing repo) +Here are the different steps of this script: + +1/ Query pkgdb for the information about who is allowed to access which git + repo + +2/ Check the local repo in each namespace + +3/ Create any git repo that could be missing + +4/ For each git repo, verifies if all the branch that should be there are, + and if not, create them. (multi-threaded this part to save time) + """ +import itertools import multiprocessing.pool import os import subprocess @@ -48,18 +61,20 @@ config['active'] = True config['endpoints']['relay_inbound'] = config['relay_inbound'] fedmsg.init(name='relay_inbound', cert_prefix='shell', **config) -{% if env == 'staging' %} +{% if env == 'staging' -%} PKGDB_URL = 'https://admin.stg.fedoraproject.org/pkgdb' -{% else %} +GIT_FOLDER = '/srv/git/repositories/' +{%- else -%} PKGDB_URL = 'https://admin.fedoraproject.org/pkgdb' -{% endif %} - GIT_FOLDER = '/srv/git/rpms/' +{%- endif %} + MKBRANCH = '/usr/local/bin/mkbranch' SETUP_PACKAGE = '/usr/local/bin/setup_git_package' THREADS = 20 VERBOSE = False +TEST_ONLY = False class InternalError(Exception): @@ -121,16 +136,19 @@ def _create_branch(pkgname, branch, existing_branches): return try: - _invoke(MKBRANCH, [branch, pkgname]) - fedmsg.publish( - topic='branch', - modname='git', - msg=dict( - agent='pkgdb', - name=pkgname, - branch=branch, - ), - ) + if VERBOSE: + print 'Creating branch: %s for package: %s' % (branch, pkgname) + if not TEST_ONLY: + _invoke(MKBRANCH, [branch, pkgname]) + fedmsg.publish( + topic='branch', + modname='git', + msg=dict( + agent='pkgdb', + name=pkgname, + branch=branch, + ), + ) except ProcessError, e: if e.returnCode == 255: # This is a warning, not an error @@ -150,24 +168,23 @@ def pkgdb_pkg_branch(): data = req.json() output = {} - key = 'packageAcls' - for pkg in data[key]: - if pkg in output: - if VERBOSE: - print 'Strange package: %s, it is present twice in the ' \ - 'pkgdb output' % pkg - output[pkg].updated(data[key][pkg].keys()) - else: - output[pkg] = set(data[key][pkg].keys()) + for key in data: + if key == 'title': + continue + for pkg in data[key]: + output.setdefault( + key, {}).setdefault( + pkg, set()).update(data[key][pkg].keys()) return output -def get_git_branch(pkg): +def get_git_branch(el): """ For the specified package name, check the local git and return the list of branches found. """ - git_folder = os.path.join(GIT_FOLDER, '%s.git' % pkg) + ns, pkg = el + git_folder = os.path.join(GIT_FOLDER, ns, '%s.git' % pkg) if not os.path.exists(git_folder): if VERBOSE: print 'Could not find %s' % git_folder @@ -180,9 +197,10 @@ def get_git_branch(pkg): return set(branches) -def branch_package(pkgname, requested_branches, existing_branches): +def branch_package(ns, pkgname, requested_branches, existing_branches): '''Create all the branches that are listed in the pkgdb for a package. + :arg ns: The namespace of the package :arg pkgname: The package to create branches for :arg requested_branches: The branches to creates :arg existing_branches: A list of existing local branches @@ -192,20 +210,22 @@ def branch_package(pkgname, requested_branches, existing_branches): print 'Fixing package %s for branches %s' % (pkgname, requested_branches) # Create the devel branch if necessary - exists = os.path.exists(os.path.join(GIT_FOLDER, '%s.git' % pkgname)) + exists = os.path.exists(os.path.join(GIT_FOLDER, ns, '%s.git' % pkgname)) if not exists or 'master' not in existing_branches: - _invoke(SETUP_PACKAGE, [pkgname]) - if 'master' in requested_branches: - requested_branches.remove('master') # SETUP_PACKAGE creates master - fedmsg.publish( - topic='branch', - modname='git', - msg=dict( - agent='pkgdb', - name=pkgname, - branch='master', - ), - ) + if not TEST_ONLY: + _invoke(SETUP_PACKAGE, [pkgname]) + # SETUP_PACKAGE creates master + if 'master' in requested_branches: + requested_branches.remove('master') + fedmsg.publish( + topic='branch', + modname='git', + msg=dict( + agent='pkgdb', + name=pkgname, + branch='master', + ), + ) # Create all the required branches for the package # Use the translated branch name until pkgdb falls inline @@ -218,60 +238,69 @@ def main(): branches and fix inconsistencies. """ - local_pkgs = set(os.listdir(GIT_FOLDER)) - local_pkgs = set([it.replace('.git', '') for it in local_pkgs]) - if VERBOSE: - print "Found %i local packages" % len(local_pkgs) - pkgdb_info = pkgdb_pkg_branch() - pkgdb_pkgs = set(pkgdb_info.keys()) - if VERBOSE: - print "Found %i pkgdb packages" % len(pkgdb_pkgs) + for ns in pkgdb_info: + namespace = ns + if ns == 'packageAcls': + namespace = '' - ## Commented out as we keep the git of retired packages while they won't - ## show up in the information retrieved from pkgdb. - - #if (local_pkgs - pkgdb_pkgs): - #print 'Some packages are present locally but not on pkgdb:' - #print ', '.join(sorted(local_pkgs - pkgdb_pkgs)) - - if (pkgdb_pkgs - local_pkgs): - print 'Some packages are present in pkgdb but not locally:' - print ', '.join(sorted(pkgdb_pkgs - local_pkgs)) - - - if VERBOSE: - print "Finding the lists of local branches for local repos." - start = time.time() - if THREADS == 1: - git_branch_lookup = map(get_git_branch, sorted(pkgdb_info)) - else: - threadpool = multiprocessing.pool.ThreadPool(processes=THREADS) - git_branch_lookup = threadpool.map(get_git_branch, sorted(pkgdb_info)) - - # Zip that list of results up into a lookup dict. - git_branch_lookup = dict(zip(sorted(pkgdb_info), git_branch_lookup)) - - if VERBOSE: - print "Found all local git branches in %0.2fs" % (time.time() - start) - - tofix = set() - for pkg in sorted(pkgdb_info): - pkgdb_branches = pkgdb_info[pkg] - git_branches = git_branch_lookup[pkg] - diff = (pkgdb_branches - git_branches) - if diff: - print '%s missing: %s' % (pkg, ','.join(sorted(diff))) - tofix.add(pkg) - branch_package(pkg, diff, git_branches) - - if tofix: - print 'Packages fixed (%s): %s' % ( - len(tofix), ', '.join(sorted(tofix))) - else: + pkgdb_pkgs = set(pkgdb_info[ns].keys()) if VERBOSE: - print 'Didn\'t find any packages to fix.' + print "Found %i pkgdb packages (namespace: %s)" % ( + len(pkgdb_pkgs), ns) + + local_pkgs = set(os.listdir(os.path.join(GIT_FOLDER, namespace))) + local_pkgs = set([it.replace('.git', '') for it in local_pkgs]) + if VERBOSE: + print "Found %i local packages (namespace: %s)" % ( + len(local_pkgs), ns) + + ## Commented out as we keep the git of retired packages while they won't + ## show up in the information retrieved from pkgdb. + + #if (local_pkgs - pkgdb_pkgs): + #print 'Some packages are present locally but not on pkgdb:' + #print ', '.join(sorted(local_pkgs - pkgdb_pkgs)) + + if (pkgdb_pkgs - local_pkgs): + print 'Some packages are present in pkgdb but not locally:' + print ', '.join(sorted(pkgdb_pkgs - local_pkgs)) + + + if VERBOSE: + print "Finding the lists of local branches for local repos." + start = time.time() + if THREADS == 1: + git_branch_lookup = map(get_git_branch, + itertools.product([namespace], sorted(pkgdb_info[ns]))) + else: + threadpool = multiprocessing.pool.ThreadPool(processes=THREADS) + git_branch_lookup = threadpool.map(get_git_branch, + itertools.product([namespace], sorted(pkgdb_info[ns]))) + + # Zip that list of results up into a lookup dict. + git_branch_lookup = dict(zip(sorted(pkgdb_info[ns]), git_branch_lookup)) + + if VERBOSE: + print "Found all local git branches in %0.2fs" % (time.time() - start) + + tofix = set() + for pkg in sorted(pkgdb_info[ns]): + pkgdb_branches = pkgdb_info[ns][pkg] + git_branches = git_branch_lookup[pkg] + diff = (pkgdb_branches - git_branches) + if diff: + print '%s missing: %s' % (pkg, ','.join(sorted(diff))) + tofix.add(pkg) + branch_package(namespace, pkg, diff, git_branches) + + if tofix: + print 'Packages fixed (%s): %s' % ( + len(tofix), ', '.join(sorted(tofix))) + else: + if VERBOSE: + print 'Didn\'t find any packages to fix.' if __name__ == '__main__':