Update pkgdb_sync_git_branches to the new pkgdb API while remaining backward compatible

This commit also adds a TEST_ONLY global boolean useful if you want to
test the script without modifying anything on the filesystem.
This commit is contained in:
Pierre-Yves Chibon 2015-12-11 12:56:15 +01:00
parent d4307d396e
commit 382d837d7a

View file

@ -24,8 +24,21 @@ branches are missing.
It even goes one step further but actually adjusting the git repo by adding It even goes one step further but actually adjusting the git repo by adding
the missing branches (or even the missing repo) the missing branches (or even the missing repo)
Here are the different steps of this script:
1/ Query pkgdb for the information about who is allowed to access which git
repo
2/ Check the local repo in each namespace
3/ Create any git repo that could be missing
4/ For each git repo, verifies if all the branch that should be there are,
and if not, create them. (multi-threaded this part to save time)
""" """
import itertools
import multiprocessing.pool import multiprocessing.pool
import os import os
import subprocess import subprocess
@ -48,18 +61,20 @@ config['active'] = True
config['endpoints']['relay_inbound'] = config['relay_inbound'] config['endpoints']['relay_inbound'] = config['relay_inbound']
fedmsg.init(name='relay_inbound', cert_prefix='shell', **config) fedmsg.init(name='relay_inbound', cert_prefix='shell', **config)
{% if env == 'staging' %} {% if env == 'staging' -%}
PKGDB_URL = 'https://admin.stg.fedoraproject.org/pkgdb' PKGDB_URL = 'https://admin.stg.fedoraproject.org/pkgdb'
{% else %} GIT_FOLDER = '/srv/git/repositories/'
{%- else -%}
PKGDB_URL = 'https://admin.fedoraproject.org/pkgdb' PKGDB_URL = 'https://admin.fedoraproject.org/pkgdb'
{% endif %}
GIT_FOLDER = '/srv/git/rpms/' GIT_FOLDER = '/srv/git/rpms/'
{%- endif %}
MKBRANCH = '/usr/local/bin/mkbranch' MKBRANCH = '/usr/local/bin/mkbranch'
SETUP_PACKAGE = '/usr/local/bin/setup_git_package' SETUP_PACKAGE = '/usr/local/bin/setup_git_package'
THREADS = 20 THREADS = 20
VERBOSE = False VERBOSE = False
TEST_ONLY = False
class InternalError(Exception): class InternalError(Exception):
@ -121,6 +136,9 @@ def _create_branch(pkgname, branch, existing_branches):
return return
try: try:
if VERBOSE:
print 'Creating branch: %s for package: %s' % (branch, pkgname)
if not TEST_ONLY:
_invoke(MKBRANCH, [branch, pkgname]) _invoke(MKBRANCH, [branch, pkgname])
fedmsg.publish( fedmsg.publish(
topic='branch', topic='branch',
@ -150,24 +168,23 @@ def pkgdb_pkg_branch():
data = req.json() data = req.json()
output = {} output = {}
key = 'packageAcls' for key in data:
if key == 'title':
continue
for pkg in data[key]: for pkg in data[key]:
if pkg in output: output.setdefault(
if VERBOSE: key, {}).setdefault(
print 'Strange package: %s, it is present twice in the ' \ pkg, set()).update(data[key][pkg].keys())
'pkgdb output' % pkg
output[pkg].updated(data[key][pkg].keys())
else:
output[pkg] = set(data[key][pkg].keys())
return output return output
def get_git_branch(pkg): def get_git_branch(el):
""" For the specified package name, check the local git and return the """ For the specified package name, check the local git and return the
list of branches found. list of branches found.
""" """
git_folder = os.path.join(GIT_FOLDER, '%s.git' % pkg) ns, pkg = el
git_folder = os.path.join(GIT_FOLDER, ns, '%s.git' % pkg)
if not os.path.exists(git_folder): if not os.path.exists(git_folder):
if VERBOSE: if VERBOSE:
print 'Could not find %s' % git_folder print 'Could not find %s' % git_folder
@ -180,9 +197,10 @@ def get_git_branch(pkg):
return set(branches) return set(branches)
def branch_package(pkgname, requested_branches, existing_branches): def branch_package(ns, pkgname, requested_branches, existing_branches):
'''Create all the branches that are listed in the pkgdb for a package. '''Create all the branches that are listed in the pkgdb for a package.
:arg ns: The namespace of the package
:arg pkgname: The package to create branches for :arg pkgname: The package to create branches for
:arg requested_branches: The branches to creates :arg requested_branches: The branches to creates
:arg existing_branches: A list of existing local branches :arg existing_branches: A list of existing local branches
@ -192,11 +210,13 @@ def branch_package(pkgname, requested_branches, existing_branches):
print 'Fixing package %s for branches %s' % (pkgname, requested_branches) print 'Fixing package %s for branches %s' % (pkgname, requested_branches)
# Create the devel branch if necessary # Create the devel branch if necessary
exists = os.path.exists(os.path.join(GIT_FOLDER, '%s.git' % pkgname)) exists = os.path.exists(os.path.join(GIT_FOLDER, ns, '%s.git' % pkgname))
if not exists or 'master' not in existing_branches: if not exists or 'master' not in existing_branches:
if not TEST_ONLY:
_invoke(SETUP_PACKAGE, [pkgname]) _invoke(SETUP_PACKAGE, [pkgname])
# SETUP_PACKAGE creates master
if 'master' in requested_branches: if 'master' in requested_branches:
requested_branches.remove('master') # SETUP_PACKAGE creates master requested_branches.remove('master')
fedmsg.publish( fedmsg.publish(
topic='branch', topic='branch',
modname='git', modname='git',
@ -218,16 +238,23 @@ def main():
branches and fix inconsistencies. branches and fix inconsistencies.
""" """
local_pkgs = set(os.listdir(GIT_FOLDER))
local_pkgs = set([it.replace('.git', '') for it in local_pkgs])
if VERBOSE:
print "Found %i local packages" % len(local_pkgs)
pkgdb_info = pkgdb_pkg_branch() pkgdb_info = pkgdb_pkg_branch()
pkgdb_pkgs = set(pkgdb_info.keys()) for ns in pkgdb_info:
namespace = ns
if ns == 'packageAcls':
namespace = ''
pkgdb_pkgs = set(pkgdb_info[ns].keys())
if VERBOSE: if VERBOSE:
print "Found %i pkgdb packages" % len(pkgdb_pkgs) print "Found %i pkgdb packages (namespace: %s)" % (
len(pkgdb_pkgs), ns)
local_pkgs = set(os.listdir(os.path.join(GIT_FOLDER, namespace)))
local_pkgs = set([it.replace('.git', '') for it in local_pkgs])
if VERBOSE:
print "Found %i local packages (namespace: %s)" % (
len(local_pkgs), ns)
## Commented out as we keep the git of retired packages while they won't ## Commented out as we keep the git of retired packages while they won't
## show up in the information retrieved from pkgdb. ## show up in the information retrieved from pkgdb.
@ -245,26 +272,28 @@ def main():
print "Finding the lists of local branches for local repos." print "Finding the lists of local branches for local repos."
start = time.time() start = time.time()
if THREADS == 1: if THREADS == 1:
git_branch_lookup = map(get_git_branch, sorted(pkgdb_info)) git_branch_lookup = map(get_git_branch,
itertools.product([namespace], sorted(pkgdb_info[ns])))
else: else:
threadpool = multiprocessing.pool.ThreadPool(processes=THREADS) threadpool = multiprocessing.pool.ThreadPool(processes=THREADS)
git_branch_lookup = threadpool.map(get_git_branch, sorted(pkgdb_info)) git_branch_lookup = threadpool.map(get_git_branch,
itertools.product([namespace], sorted(pkgdb_info[ns])))
# Zip that list of results up into a lookup dict. # Zip that list of results up into a lookup dict.
git_branch_lookup = dict(zip(sorted(pkgdb_info), git_branch_lookup)) git_branch_lookup = dict(zip(sorted(pkgdb_info[ns]), git_branch_lookup))
if VERBOSE: if VERBOSE:
print "Found all local git branches in %0.2fs" % (time.time() - start) print "Found all local git branches in %0.2fs" % (time.time() - start)
tofix = set() tofix = set()
for pkg in sorted(pkgdb_info): for pkg in sorted(pkgdb_info[ns]):
pkgdb_branches = pkgdb_info[pkg] pkgdb_branches = pkgdb_info[ns][pkg]
git_branches = git_branch_lookup[pkg] git_branches = git_branch_lookup[pkg]
diff = (pkgdb_branches - git_branches) diff = (pkgdb_branches - git_branches)
if diff: if diff:
print '%s missing: %s' % (pkg, ','.join(sorted(diff))) print '%s missing: %s' % (pkg, ','.join(sorted(diff)))
tofix.add(pkg) tofix.add(pkg)
branch_package(pkg, diff, git_branches) branch_package(namespace, pkg, diff, git_branches)
if tofix: if tofix:
print 'Packages fixed (%s): %s' % ( print 'Packages fixed (%s): %s' % (