Update pkgdb_sync_git_branches to the new pkgdb API while remaining backward compatible
This commit also adds a TEST_ONLY global boolean useful if you want to test the script without modifying anything on the filesystem.
This commit is contained in:
Pierre-Yves Chibon
parent
d4307d396e
commit
382d837d7a
1 changed files with 118 additions and 89 deletions
|
|
@ -24,8 +24,21 @@ branches are missing.
|
|||
It even goes one step further but actually adjusting the git repo by adding
|
||||
the missing branches (or even the missing repo)
|
||||
|
||||
Here are the different steps of this script:
|
||||
|
||||
1/ Query pkgdb for the information about who is allowed to access which git
|
||||
repo
|
||||
|
||||
2/ Check the local repo in each namespace
|
||||
|
||||
3/ Create any git repo that could be missing
|
||||
|
||||
4/ For each git repo, verifies if all the branch that should be there are,
|
||||
and if not, create them. (multi-threaded this part to save time)
|
||||
|
||||
"""
|
||||
|
||||
import itertools
|
||||
import multiprocessing.pool
|
||||
import os
|
||||
import subprocess
|
||||
|
|
@ -48,18 +61,20 @@ config['active'] = True
|
|||
config['endpoints']['relay_inbound'] = config['relay_inbound']
|
||||
fedmsg.init(name='relay_inbound', cert_prefix='shell', **config)
|
||||
|
||||
{% if env == 'staging' %}
|
||||
{% if env == 'staging' -%}
|
||||
PKGDB_URL = 'https://admin.stg.fedoraproject.org/pkgdb'
|
||||
{% else %}
|
||||
GIT_FOLDER = '/srv/git/repositories/'
|
||||
{%- else -%}
|
||||
PKGDB_URL = 'https://admin.fedoraproject.org/pkgdb'
|
||||
{% endif %}
|
||||
|
||||
GIT_FOLDER = '/srv/git/rpms/'
|
||||
{%- endif %}
|
||||
|
||||
MKBRANCH = '/usr/local/bin/mkbranch'
|
||||
SETUP_PACKAGE = '/usr/local/bin/setup_git_package'
|
||||
|
||||
THREADS = 20
|
||||
VERBOSE = False
|
||||
TEST_ONLY = False
|
||||
|
||||
|
||||
class InternalError(Exception):
|
||||
|
|
@ -121,16 +136,19 @@ def _create_branch(pkgname, branch, existing_branches):
|
|||
return
|
||||
|
||||
try:
|
||||
_invoke(MKBRANCH, [branch, pkgname])
|
||||
fedmsg.publish(
|
||||
topic='branch',
|
||||
modname='git',
|
||||
msg=dict(
|
||||
agent='pkgdb',
|
||||
name=pkgname,
|
||||
branch=branch,
|
||||
),
|
||||
)
|
||||
if VERBOSE:
|
||||
print 'Creating branch: %s for package: %s' % (branch, pkgname)
|
||||
if not TEST_ONLY:
|
||||
_invoke(MKBRANCH, [branch, pkgname])
|
||||
fedmsg.publish(
|
||||
topic='branch',
|
||||
modname='git',
|
||||
msg=dict(
|
||||
agent='pkgdb',
|
||||
name=pkgname,
|
||||
branch=branch,
|
||||
),
|
||||
)
|
||||
except ProcessError, e:
|
||||
if e.returnCode == 255:
|
||||
# This is a warning, not an error
|
||||
|
|
@ -150,24 +168,23 @@ def pkgdb_pkg_branch():
|
|||
data = req.json()
|
||||
|
||||
output = {}
|
||||
key = 'packageAcls'
|
||||
for pkg in data[key]:
|
||||
if pkg in output:
|
||||
if VERBOSE:
|
||||
print 'Strange package: %s, it is present twice in the ' \
|
||||
'pkgdb output' % pkg
|
||||
output[pkg].updated(data[key][pkg].keys())
|
||||
else:
|
||||
output[pkg] = set(data[key][pkg].keys())
|
||||
for key in data:
|
||||
if key == 'title':
|
||||
continue
|
||||
for pkg in data[key]:
|
||||
output.setdefault(
|
||||
key, {}).setdefault(
|
||||
pkg, set()).update(data[key][pkg].keys())
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def get_git_branch(pkg):
|
||||
def get_git_branch(el):
|
||||
""" For the specified package name, check the local git and return the
|
||||
list of branches found.
|
||||
"""
|
||||
git_folder = os.path.join(GIT_FOLDER, '%s.git' % pkg)
|
||||
ns, pkg = el
|
||||
git_folder = os.path.join(GIT_FOLDER, ns, '%s.git' % pkg)
|
||||
if not os.path.exists(git_folder):
|
||||
if VERBOSE:
|
||||
print 'Could not find %s' % git_folder
|
||||
|
|
@ -180,9 +197,10 @@ def get_git_branch(pkg):
|
|||
return set(branches)
|
||||
|
||||
|
||||
def branch_package(pkgname, requested_branches, existing_branches):
|
||||
def branch_package(ns, pkgname, requested_branches, existing_branches):
|
||||
'''Create all the branches that are listed in the pkgdb for a package.
|
||||
|
||||
:arg ns: The namespace of the package
|
||||
:arg pkgname: The package to create branches for
|
||||
:arg requested_branches: The branches to creates
|
||||
:arg existing_branches: A list of existing local branches
|
||||
|
|
@ -192,20 +210,22 @@ def branch_package(pkgname, requested_branches, existing_branches):
|
|||
print 'Fixing package %s for branches %s' % (pkgname, requested_branches)
|
||||
|
||||
# Create the devel branch if necessary
|
||||
exists = os.path.exists(os.path.join(GIT_FOLDER, '%s.git' % pkgname))
|
||||
exists = os.path.exists(os.path.join(GIT_FOLDER, ns, '%s.git' % pkgname))
|
||||
if not exists or 'master' not in existing_branches:
|
||||
_invoke(SETUP_PACKAGE, [pkgname])
|
||||
if 'master' in requested_branches:
|
||||
requested_branches.remove('master') # SETUP_PACKAGE creates master
|
||||
fedmsg.publish(
|
||||
topic='branch',
|
||||
modname='git',
|
||||
msg=dict(
|
||||
agent='pkgdb',
|
||||
name=pkgname,
|
||||
branch='master',
|
||||
),
|
||||
)
|
||||
if not TEST_ONLY:
|
||||
_invoke(SETUP_PACKAGE, [pkgname])
|
||||
# SETUP_PACKAGE creates master
|
||||
if 'master' in requested_branches:
|
||||
requested_branches.remove('master')
|
||||
fedmsg.publish(
|
||||
topic='branch',
|
||||
modname='git',
|
||||
msg=dict(
|
||||
agent='pkgdb',
|
||||
name=pkgname,
|
||||
branch='master',
|
||||
),
|
||||
)
|
||||
|
||||
# Create all the required branches for the package
|
||||
# Use the translated branch name until pkgdb falls inline
|
||||
|
|
@ -218,60 +238,69 @@ def main():
|
|||
branches and fix inconsistencies.
|
||||
"""
|
||||
|
||||
local_pkgs = set(os.listdir(GIT_FOLDER))
|
||||
local_pkgs = set([it.replace('.git', '') for it in local_pkgs])
|
||||
if VERBOSE:
|
||||
print "Found %i local packages" % len(local_pkgs)
|
||||
|
||||
pkgdb_info = pkgdb_pkg_branch()
|
||||
|
||||
pkgdb_pkgs = set(pkgdb_info.keys())
|
||||
if VERBOSE:
|
||||
print "Found %i pkgdb packages" % len(pkgdb_pkgs)
|
||||
for ns in pkgdb_info:
|
||||
namespace = ns
|
||||
if ns == 'packageAcls':
|
||||
namespace = ''
|
||||
|
||||
## Commented out as we keep the git of retired packages while they won't
|
||||
## show up in the information retrieved from pkgdb.
|
||||
|
||||
#if (local_pkgs - pkgdb_pkgs):
|
||||
#print 'Some packages are present locally but not on pkgdb:'
|
||||
#print ', '.join(sorted(local_pkgs - pkgdb_pkgs))
|
||||
|
||||
if (pkgdb_pkgs - local_pkgs):
|
||||
print 'Some packages are present in pkgdb but not locally:'
|
||||
print ', '.join(sorted(pkgdb_pkgs - local_pkgs))
|
||||
|
||||
|
||||
if VERBOSE:
|
||||
print "Finding the lists of local branches for local repos."
|
||||
start = time.time()
|
||||
if THREADS == 1:
|
||||
git_branch_lookup = map(get_git_branch, sorted(pkgdb_info))
|
||||
else:
|
||||
threadpool = multiprocessing.pool.ThreadPool(processes=THREADS)
|
||||
git_branch_lookup = threadpool.map(get_git_branch, sorted(pkgdb_info))
|
||||
|
||||
# Zip that list of results up into a lookup dict.
|
||||
git_branch_lookup = dict(zip(sorted(pkgdb_info), git_branch_lookup))
|
||||
|
||||
if VERBOSE:
|
||||
print "Found all local git branches in %0.2fs" % (time.time() - start)
|
||||
|
||||
tofix = set()
|
||||
for pkg in sorted(pkgdb_info):
|
||||
pkgdb_branches = pkgdb_info[pkg]
|
||||
git_branches = git_branch_lookup[pkg]
|
||||
diff = (pkgdb_branches - git_branches)
|
||||
if diff:
|
||||
print '%s missing: %s' % (pkg, ','.join(sorted(diff)))
|
||||
tofix.add(pkg)
|
||||
branch_package(pkg, diff, git_branches)
|
||||
|
||||
if tofix:
|
||||
print 'Packages fixed (%s): %s' % (
|
||||
len(tofix), ', '.join(sorted(tofix)))
|
||||
else:
|
||||
pkgdb_pkgs = set(pkgdb_info[ns].keys())
|
||||
if VERBOSE:
|
||||
print 'Didn\'t find any packages to fix.'
|
||||
print "Found %i pkgdb packages (namespace: %s)" % (
|
||||
len(pkgdb_pkgs), ns)
|
||||
|
||||
local_pkgs = set(os.listdir(os.path.join(GIT_FOLDER, namespace)))
|
||||
local_pkgs = set([it.replace('.git', '') for it in local_pkgs])
|
||||
if VERBOSE:
|
||||
print "Found %i local packages (namespace: %s)" % (
|
||||
len(local_pkgs), ns)
|
||||
|
||||
## Commented out as we keep the git of retired packages while they won't
|
||||
## show up in the information retrieved from pkgdb.
|
||||
|
||||
#if (local_pkgs - pkgdb_pkgs):
|
||||
#print 'Some packages are present locally but not on pkgdb:'
|
||||
#print ', '.join(sorted(local_pkgs - pkgdb_pkgs))
|
||||
|
||||
if (pkgdb_pkgs - local_pkgs):
|
||||
print 'Some packages are present in pkgdb but not locally:'
|
||||
print ', '.join(sorted(pkgdb_pkgs - local_pkgs))
|
||||
|
||||
|
||||
if VERBOSE:
|
||||
print "Finding the lists of local branches for local repos."
|
||||
start = time.time()
|
||||
if THREADS == 1:
|
||||
git_branch_lookup = map(get_git_branch,
|
||||
itertools.product([namespace], sorted(pkgdb_info[ns])))
|
||||
else:
|
||||
threadpool = multiprocessing.pool.ThreadPool(processes=THREADS)
|
||||
git_branch_lookup = threadpool.map(get_git_branch,
|
||||
itertools.product([namespace], sorted(pkgdb_info[ns])))
|
||||
|
||||
# Zip that list of results up into a lookup dict.
|
||||
git_branch_lookup = dict(zip(sorted(pkgdb_info[ns]), git_branch_lookup))
|
||||
|
||||
if VERBOSE:
|
||||
print "Found all local git branches in %0.2fs" % (time.time() - start)
|
||||
|
||||
tofix = set()
|
||||
for pkg in sorted(pkgdb_info[ns]):
|
||||
pkgdb_branches = pkgdb_info[ns][pkg]
|
||||
git_branches = git_branch_lookup[pkg]
|
||||
diff = (pkgdb_branches - git_branches)
|
||||
if diff:
|
||||
print '%s missing: %s' % (pkg, ','.join(sorted(diff)))
|
||||
tofix.add(pkg)
|
||||
branch_package(namespace, pkg, diff, git_branches)
|
||||
|
||||
if tofix:
|
||||
print 'Packages fixed (%s): %s' % (
|
||||
len(tofix), ', '.join(sorted(tofix)))
|
||||
else:
|
||||
if VERBOSE:
|
||||
print 'Didn\'t find any packages to fix.'
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue