Update pkgdb_sync_git_branches to the new pkgdb API while remaining backward compatible
This commit also adds a TEST_ONLY global boolean useful if you want to test the script without modifying anything on the filesystem.
This commit is contained in:
parent
d4307d396e
commit
382d837d7a
1 changed files with 118 additions and 89 deletions
|
@ -24,8 +24,21 @@ branches are missing.
|
||||||
It even goes one step further but actually adjusting the git repo by adding
|
It even goes one step further but actually adjusting the git repo by adding
|
||||||
the missing branches (or even the missing repo)
|
the missing branches (or even the missing repo)
|
||||||
|
|
||||||
|
Here are the different steps of this script:
|
||||||
|
|
||||||
|
1/ Query pkgdb for the information about who is allowed to access which git
|
||||||
|
repo
|
||||||
|
|
||||||
|
2/ Check the local repo in each namespace
|
||||||
|
|
||||||
|
3/ Create any git repo that could be missing
|
||||||
|
|
||||||
|
4/ For each git repo, verifies if all the branch that should be there are,
|
||||||
|
and if not, create them. (multi-threaded this part to save time)
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import itertools
|
||||||
import multiprocessing.pool
|
import multiprocessing.pool
|
||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
|
@ -48,18 +61,20 @@ config['active'] = True
|
||||||
config['endpoints']['relay_inbound'] = config['relay_inbound']
|
config['endpoints']['relay_inbound'] = config['relay_inbound']
|
||||||
fedmsg.init(name='relay_inbound', cert_prefix='shell', **config)
|
fedmsg.init(name='relay_inbound', cert_prefix='shell', **config)
|
||||||
|
|
||||||
{% if env == 'staging' %}
|
{% if env == 'staging' -%}
|
||||||
PKGDB_URL = 'https://admin.stg.fedoraproject.org/pkgdb'
|
PKGDB_URL = 'https://admin.stg.fedoraproject.org/pkgdb'
|
||||||
{% else %}
|
GIT_FOLDER = '/srv/git/repositories/'
|
||||||
|
{%- else -%}
|
||||||
PKGDB_URL = 'https://admin.fedoraproject.org/pkgdb'
|
PKGDB_URL = 'https://admin.fedoraproject.org/pkgdb'
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
GIT_FOLDER = '/srv/git/rpms/'
|
GIT_FOLDER = '/srv/git/rpms/'
|
||||||
|
{%- endif %}
|
||||||
|
|
||||||
MKBRANCH = '/usr/local/bin/mkbranch'
|
MKBRANCH = '/usr/local/bin/mkbranch'
|
||||||
SETUP_PACKAGE = '/usr/local/bin/setup_git_package'
|
SETUP_PACKAGE = '/usr/local/bin/setup_git_package'
|
||||||
|
|
||||||
THREADS = 20
|
THREADS = 20
|
||||||
VERBOSE = False
|
VERBOSE = False
|
||||||
|
TEST_ONLY = False
|
||||||
|
|
||||||
|
|
||||||
class InternalError(Exception):
|
class InternalError(Exception):
|
||||||
|
@ -121,6 +136,9 @@ def _create_branch(pkgname, branch, existing_branches):
|
||||||
return
|
return
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
if VERBOSE:
|
||||||
|
print 'Creating branch: %s for package: %s' % (branch, pkgname)
|
||||||
|
if not TEST_ONLY:
|
||||||
_invoke(MKBRANCH, [branch, pkgname])
|
_invoke(MKBRANCH, [branch, pkgname])
|
||||||
fedmsg.publish(
|
fedmsg.publish(
|
||||||
topic='branch',
|
topic='branch',
|
||||||
|
@ -150,24 +168,23 @@ def pkgdb_pkg_branch():
|
||||||
data = req.json()
|
data = req.json()
|
||||||
|
|
||||||
output = {}
|
output = {}
|
||||||
key = 'packageAcls'
|
for key in data:
|
||||||
|
if key == 'title':
|
||||||
|
continue
|
||||||
for pkg in data[key]:
|
for pkg in data[key]:
|
||||||
if pkg in output:
|
output.setdefault(
|
||||||
if VERBOSE:
|
key, {}).setdefault(
|
||||||
print 'Strange package: %s, it is present twice in the ' \
|
pkg, set()).update(data[key][pkg].keys())
|
||||||
'pkgdb output' % pkg
|
|
||||||
output[pkg].updated(data[key][pkg].keys())
|
|
||||||
else:
|
|
||||||
output[pkg] = set(data[key][pkg].keys())
|
|
||||||
|
|
||||||
return output
|
return output
|
||||||
|
|
||||||
|
|
||||||
def get_git_branch(pkg):
|
def get_git_branch(el):
|
||||||
""" For the specified package name, check the local git and return the
|
""" For the specified package name, check the local git and return the
|
||||||
list of branches found.
|
list of branches found.
|
||||||
"""
|
"""
|
||||||
git_folder = os.path.join(GIT_FOLDER, '%s.git' % pkg)
|
ns, pkg = el
|
||||||
|
git_folder = os.path.join(GIT_FOLDER, ns, '%s.git' % pkg)
|
||||||
if not os.path.exists(git_folder):
|
if not os.path.exists(git_folder):
|
||||||
if VERBOSE:
|
if VERBOSE:
|
||||||
print 'Could not find %s' % git_folder
|
print 'Could not find %s' % git_folder
|
||||||
|
@ -180,9 +197,10 @@ def get_git_branch(pkg):
|
||||||
return set(branches)
|
return set(branches)
|
||||||
|
|
||||||
|
|
||||||
def branch_package(pkgname, requested_branches, existing_branches):
|
def branch_package(ns, pkgname, requested_branches, existing_branches):
|
||||||
'''Create all the branches that are listed in the pkgdb for a package.
|
'''Create all the branches that are listed in the pkgdb for a package.
|
||||||
|
|
||||||
|
:arg ns: The namespace of the package
|
||||||
:arg pkgname: The package to create branches for
|
:arg pkgname: The package to create branches for
|
||||||
:arg requested_branches: The branches to creates
|
:arg requested_branches: The branches to creates
|
||||||
:arg existing_branches: A list of existing local branches
|
:arg existing_branches: A list of existing local branches
|
||||||
|
@ -192,11 +210,13 @@ def branch_package(pkgname, requested_branches, existing_branches):
|
||||||
print 'Fixing package %s for branches %s' % (pkgname, requested_branches)
|
print 'Fixing package %s for branches %s' % (pkgname, requested_branches)
|
||||||
|
|
||||||
# Create the devel branch if necessary
|
# Create the devel branch if necessary
|
||||||
exists = os.path.exists(os.path.join(GIT_FOLDER, '%s.git' % pkgname))
|
exists = os.path.exists(os.path.join(GIT_FOLDER, ns, '%s.git' % pkgname))
|
||||||
if not exists or 'master' not in existing_branches:
|
if not exists or 'master' not in existing_branches:
|
||||||
|
if not TEST_ONLY:
|
||||||
_invoke(SETUP_PACKAGE, [pkgname])
|
_invoke(SETUP_PACKAGE, [pkgname])
|
||||||
|
# SETUP_PACKAGE creates master
|
||||||
if 'master' in requested_branches:
|
if 'master' in requested_branches:
|
||||||
requested_branches.remove('master') # SETUP_PACKAGE creates master
|
requested_branches.remove('master')
|
||||||
fedmsg.publish(
|
fedmsg.publish(
|
||||||
topic='branch',
|
topic='branch',
|
||||||
modname='git',
|
modname='git',
|
||||||
|
@ -218,16 +238,23 @@ def main():
|
||||||
branches and fix inconsistencies.
|
branches and fix inconsistencies.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
local_pkgs = set(os.listdir(GIT_FOLDER))
|
|
||||||
local_pkgs = set([it.replace('.git', '') for it in local_pkgs])
|
|
||||||
if VERBOSE:
|
|
||||||
print "Found %i local packages" % len(local_pkgs)
|
|
||||||
|
|
||||||
pkgdb_info = pkgdb_pkg_branch()
|
pkgdb_info = pkgdb_pkg_branch()
|
||||||
|
|
||||||
pkgdb_pkgs = set(pkgdb_info.keys())
|
for ns in pkgdb_info:
|
||||||
|
namespace = ns
|
||||||
|
if ns == 'packageAcls':
|
||||||
|
namespace = ''
|
||||||
|
|
||||||
|
pkgdb_pkgs = set(pkgdb_info[ns].keys())
|
||||||
if VERBOSE:
|
if VERBOSE:
|
||||||
print "Found %i pkgdb packages" % len(pkgdb_pkgs)
|
print "Found %i pkgdb packages (namespace: %s)" % (
|
||||||
|
len(pkgdb_pkgs), ns)
|
||||||
|
|
||||||
|
local_pkgs = set(os.listdir(os.path.join(GIT_FOLDER, namespace)))
|
||||||
|
local_pkgs = set([it.replace('.git', '') for it in local_pkgs])
|
||||||
|
if VERBOSE:
|
||||||
|
print "Found %i local packages (namespace: %s)" % (
|
||||||
|
len(local_pkgs), ns)
|
||||||
|
|
||||||
## Commented out as we keep the git of retired packages while they won't
|
## Commented out as we keep the git of retired packages while they won't
|
||||||
## show up in the information retrieved from pkgdb.
|
## show up in the information retrieved from pkgdb.
|
||||||
|
@ -245,26 +272,28 @@ def main():
|
||||||
print "Finding the lists of local branches for local repos."
|
print "Finding the lists of local branches for local repos."
|
||||||
start = time.time()
|
start = time.time()
|
||||||
if THREADS == 1:
|
if THREADS == 1:
|
||||||
git_branch_lookup = map(get_git_branch, sorted(pkgdb_info))
|
git_branch_lookup = map(get_git_branch,
|
||||||
|
itertools.product([namespace], sorted(pkgdb_info[ns])))
|
||||||
else:
|
else:
|
||||||
threadpool = multiprocessing.pool.ThreadPool(processes=THREADS)
|
threadpool = multiprocessing.pool.ThreadPool(processes=THREADS)
|
||||||
git_branch_lookup = threadpool.map(get_git_branch, sorted(pkgdb_info))
|
git_branch_lookup = threadpool.map(get_git_branch,
|
||||||
|
itertools.product([namespace], sorted(pkgdb_info[ns])))
|
||||||
|
|
||||||
# Zip that list of results up into a lookup dict.
|
# Zip that list of results up into a lookup dict.
|
||||||
git_branch_lookup = dict(zip(sorted(pkgdb_info), git_branch_lookup))
|
git_branch_lookup = dict(zip(sorted(pkgdb_info[ns]), git_branch_lookup))
|
||||||
|
|
||||||
if VERBOSE:
|
if VERBOSE:
|
||||||
print "Found all local git branches in %0.2fs" % (time.time() - start)
|
print "Found all local git branches in %0.2fs" % (time.time() - start)
|
||||||
|
|
||||||
tofix = set()
|
tofix = set()
|
||||||
for pkg in sorted(pkgdb_info):
|
for pkg in sorted(pkgdb_info[ns]):
|
||||||
pkgdb_branches = pkgdb_info[pkg]
|
pkgdb_branches = pkgdb_info[ns][pkg]
|
||||||
git_branches = git_branch_lookup[pkg]
|
git_branches = git_branch_lookup[pkg]
|
||||||
diff = (pkgdb_branches - git_branches)
|
diff = (pkgdb_branches - git_branches)
|
||||||
if diff:
|
if diff:
|
||||||
print '%s missing: %s' % (pkg, ','.join(sorted(diff)))
|
print '%s missing: %s' % (pkg, ','.join(sorted(diff)))
|
||||||
tofix.add(pkg)
|
tofix.add(pkg)
|
||||||
branch_package(pkg, diff, git_branches)
|
branch_package(namespace, pkg, diff, git_branches)
|
||||||
|
|
||||||
if tofix:
|
if tofix:
|
||||||
print 'Packages fixed (%s): %s' % (
|
print 'Packages fixed (%s): %s' % (
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue