Update pkgdb_sync_git_branches to the new pkgdb API while remaining backward compatible

This commit also adds a TEST_ONLY global boolean useful if you want to
test the script without modifying anything on the filesystem.

roles/distgit/templates/pkgdb_sync_git_branches.py

@@ -24,8 +24,21 @@ branches are missing.
 It even goes one step further but actually adjusting the git repo by adding
 the missing branches (or even the missing repo)
 
+Here are the different steps of this script:
+
+1/ Query pkgdb for the information about who is allowed to access which git
+   repo
+
+2/ Check the local repo in each namespace
+
+3/ Create any git repo that could be missing
+
+4/ For each git repo, verifies if all the branch that should be there are,
+   and if not, create them. (multi-threaded this part to save time)
+
 """
 
+import itertools
 import multiprocessing.pool
 import os
 import subprocess
@@ -48,18 +61,20 @@ config['active'] = True
 config['endpoints']['relay_inbound'] = config['relay_inbound']
 fedmsg.init(name='relay_inbound', cert_prefix='shell', **config)
 
-{% if env == 'staging' %}
+{% if env == 'staging' -%}
 PKGDB_URL = 'https://admin.stg.fedoraproject.org/pkgdb'
-{% else %}
+GIT_FOLDER = '/srv/git/repositories/'
+{%- else -%}
 PKGDB_URL = 'https://admin.fedoraproject.org/pkgdb'
-{% endif %}
-
 GIT_FOLDER = '/srv/git/rpms/'
+{%- endif %}
+
 MKBRANCH = '/usr/local/bin/mkbranch'
 SETUP_PACKAGE = '/usr/local/bin/setup_git_package'
 
 THREADS = 20
 VERBOSE = False
+TEST_ONLY = False
 
 
 class InternalError(Exception):
@@ -121,16 +136,19 @@ def _create_branch(pkgname, branch, existing_branches):
        return
 
     try:
-        _invoke(MKBRANCH, [branch, pkgname])
+        if VERBOSE:
-        fedmsg.publish(
+            print 'Creating branch: %s for package: %s' % (branch, pkgname)
-            topic='branch',
+        if not TEST_ONLY:
-            modname='git',
+            _invoke(MKBRANCH, [branch, pkgname])
-            msg=dict(
+            fedmsg.publish(
-                agent='pkgdb',
+                topic='branch',
-                name=pkgname,
+                modname='git',
-                branch=branch,
+                msg=dict(
-            ),
+                    agent='pkgdb',
-        )
+                    name=pkgname,
+                    branch=branch,
+                ),
+            )
     except ProcessError, e:
         if e.returnCode == 255:
             # This is a warning, not an error
@@ -150,24 +168,23 @@ def pkgdb_pkg_branch():
     data = req.json()
 
     output = {}
-    key = 'packageAcls'
+    for key in data:
-    for pkg in data[key]:
+        if key == 'title':
-        if pkg in output:
+            continue
-            if VERBOSE:
+        for pkg in data[key]:
-                print 'Strange package: %s, it is present twice in the ' \
+            output.setdefault(
-                    'pkgdb output' % pkg
+                key, {}).setdefault(
-            output[pkg].updated(data[key][pkg].keys())
+                    pkg, set()).update(data[key][pkg].keys())
-        else:
-            output[pkg] = set(data[key][pkg].keys())
 
     return output
 
 
-def get_git_branch(pkg):
+def get_git_branch(el):
     """ For the specified package name, check the local git and return the
     list of branches found.
     """
-    git_folder = os.path.join(GIT_FOLDER, '%s.git' % pkg)
+    ns, pkg = el
+    git_folder = os.path.join(GIT_FOLDER, ns, '%s.git' % pkg)
     if not os.path.exists(git_folder):
         if VERBOSE:
             print 'Could not find %s' % git_folder
@@ -180,9 +197,10 @@ def get_git_branch(pkg):
     return set(branches)
 
 
-def branch_package(pkgname, requested_branches, existing_branches):
+def branch_package(ns, pkgname, requested_branches, existing_branches):
     '''Create all the branches that are listed in the pkgdb for a package.
 
+    :arg ns: The namespace of the package
     :arg pkgname: The package to create branches for
     :arg requested_branches: The branches to creates
     :arg existing_branches: A list of existing local branches
@@ -192,20 +210,22 @@ def branch_package(pkgname, requested_branches, existing_branches):
         print 'Fixing package %s for branches %s' % (pkgname, requested_branches)
 
     # Create the devel branch if necessary
-    exists = os.path.exists(os.path.join(GIT_FOLDER, '%s.git' % pkgname))
+    exists = os.path.exists(os.path.join(GIT_FOLDER, ns, '%s.git' % pkgname))
     if not exists or 'master' not in existing_branches:
-        _invoke(SETUP_PACKAGE, [pkgname])
+        if not TEST_ONLY:
-        if 'master' in requested_branches:
+            _invoke(SETUP_PACKAGE, [pkgname])
-            requested_branches.remove('master')  # SETUP_PACKAGE creates master
+            # SETUP_PACKAGE creates master
-        fedmsg.publish(
+            if 'master' in requested_branches:
-            topic='branch',
+                requested_branches.remove('master')
-            modname='git',
+            fedmsg.publish(
-            msg=dict(
+                topic='branch',
-                agent='pkgdb',
+                modname='git',
-                name=pkgname,
+                msg=dict(
-                branch='master',
+                    agent='pkgdb',
-            ),
+                    name=pkgname,
-        )
+                    branch='master',
+                ),
+            )
 
     # Create all the required branches for the package
     # Use the translated branch name until pkgdb falls inline
@@ -218,60 +238,69 @@ def main():
     branches and fix inconsistencies.
     """
 
-    local_pkgs = set(os.listdir(GIT_FOLDER))
-    local_pkgs = set([it.replace('.git', '') for it in local_pkgs])
-    if VERBOSE:
-        print "Found %i local packages" % len(local_pkgs)
-
     pkgdb_info = pkgdb_pkg_branch()
 
-    pkgdb_pkgs = set(pkgdb_info.keys())
+    for ns in pkgdb_info:
-    if VERBOSE:
+        namespace = ns
-        print "Found %i pkgdb packages" % len(pkgdb_pkgs)
+        if ns == 'packageAcls':
+            namespace = ''
 
-    ## Commented out as we keep the git of retired packages while they won't
+        pkgdb_pkgs = set(pkgdb_info[ns].keys())
-    ## show up in the information retrieved from pkgdb.
-
-    #if (local_pkgs - pkgdb_pkgs):
-        #print 'Some packages are present locally but not on pkgdb:'
-        #print ', '.join(sorted(local_pkgs - pkgdb_pkgs))
-
-    if (pkgdb_pkgs - local_pkgs):
-        print 'Some packages are present in pkgdb but not locally:'
-        print ', '.join(sorted(pkgdb_pkgs - local_pkgs))
-
-
-    if VERBOSE:
-        print "Finding the lists of local branches for local repos."
-    start = time.time()
-    if THREADS == 1:
-        git_branch_lookup = map(get_git_branch, sorted(pkgdb_info))
-    else:
-        threadpool = multiprocessing.pool.ThreadPool(processes=THREADS)
-        git_branch_lookup = threadpool.map(get_git_branch, sorted(pkgdb_info))
-
-    # Zip that list of results up into a lookup dict.
-    git_branch_lookup = dict(zip(sorted(pkgdb_info), git_branch_lookup))
-
-    if VERBOSE:
-        print "Found all local git branches in %0.2fs" % (time.time() - start)
-
-    tofix = set()
-    for pkg in sorted(pkgdb_info):
-        pkgdb_branches = pkgdb_info[pkg]
-        git_branches = git_branch_lookup[pkg]
-        diff = (pkgdb_branches - git_branches)
-        if diff:
-            print '%s missing: %s' % (pkg, ','.join(sorted(diff)))
-            tofix.add(pkg)
-            branch_package(pkg, diff, git_branches)
-
-    if tofix:
-        print 'Packages fixed (%s): %s' % (
-            len(tofix), ', '.join(sorted(tofix)))
-    else:
         if VERBOSE:
-            print 'Didn\'t find any packages to fix.'
+            print "Found %i pkgdb packages (namespace: %s)" % (
+                len(pkgdb_pkgs), ns)
+
+        local_pkgs = set(os.listdir(os.path.join(GIT_FOLDER, namespace)))
+        local_pkgs = set([it.replace('.git', '') for it in local_pkgs])
+        if VERBOSE:
+            print "Found %i local packages (namespace: %s)" % (
+                len(local_pkgs), ns)
+
+        ## Commented out as we keep the git of retired packages while they won't
+        ## show up in the information retrieved from pkgdb.
+
+        #if (local_pkgs - pkgdb_pkgs):
+            #print 'Some packages are present locally but not on pkgdb:'
+            #print ', '.join(sorted(local_pkgs - pkgdb_pkgs))
+
+        if (pkgdb_pkgs - local_pkgs):
+            print 'Some packages are present in pkgdb but not locally:'
+            print ', '.join(sorted(pkgdb_pkgs - local_pkgs))
+
+
+        if VERBOSE:
+            print "Finding the lists of local branches for local repos."
+        start = time.time()
+        if THREADS == 1:
+            git_branch_lookup = map(get_git_branch,
+                itertools.product([namespace], sorted(pkgdb_info[ns])))
+        else:
+            threadpool = multiprocessing.pool.ThreadPool(processes=THREADS)
+            git_branch_lookup = threadpool.map(get_git_branch,
+                itertools.product([namespace], sorted(pkgdb_info[ns])))
+
+        # Zip that list of results up into a lookup dict.
+        git_branch_lookup = dict(zip(sorted(pkgdb_info[ns]), git_branch_lookup))
+
+        if VERBOSE:
+            print "Found all local git branches in %0.2fs" % (time.time() - start)
+
+        tofix = set()
+        for pkg in sorted(pkgdb_info[ns]):
+            pkgdb_branches = pkgdb_info[ns][pkg]
+            git_branches = git_branch_lookup[pkg]
+            diff = (pkgdb_branches - git_branches)
+            if diff:
+                print '%s missing: %s' % (pkg, ','.join(sorted(diff)))
+                tofix.add(pkg)
+                branch_package(namespace, pkg, diff, git_branches)
+
+        if tofix:
+            print 'Packages fixed (%s): %s' % (
+                len(tofix), ', '.join(sorted(tofix)))
+        else:
+            if VERBOSE:
+                print 'Didn\'t find any packages to fix.'
 
 
 if __name__ == '__main__':