diff --git a/roles/distgit/files/pkgdb_sync_git_branches.py b/roles/distgit/files/pkgdb_sync_git_branches.py new file mode 100644 index 0000000000..42833c2881 --- /dev/null +++ b/roles/distgit/files/pkgdb_sync_git_branches.py @@ -0,0 +1,246 @@ +#!/usr/bin/python -tt +# -*- coding: utf-8 -*- + +""" +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +This script is able to query pkgdb and retrieve for all packages which active +branches should be there, browse all the git repos and find out which active +branches are missing. + +It even goes one step further but actually adjusting the git repo by adding +the missing branches (or even the missing repo) + +""" + +import os +import subprocess + +import requests + +import fedmsg + +# Do some off-the-bat configuration of fedmsg. +# 1) since this is a one-off script and not a daemon, it needs to connect +# to the fedmsg-relay process running on another node (or noone will +# hear it) +# 2) its going to use the 'shell' certificate which only 'sysadmin' has +# read access to. Contrast that with the 'scm' certificate which +# everyone in the 'packager' group has access to. + +config = fedmsg.config.load_config([], None) +config['active'] = True +config['endpoints']['relay_inbound'] = config['relay_inbound'] +fedmsg.init(name='relay_inbound', cert_prefix='shell', **config) + +PKGDB_URL = 'https://admin.fedoraproject.org/pkgdb' +GIT_FOLDER = '/srv/git/rpms/' +MKBRANCH = '/usr/local/bin/mkbranch' +SETUP_PACKAGE = '/usr/local/bin/setup_git_package' + +BRANCHES_FROM = { + 'epel7': 'f19', + 'olpc2': 'f7', + 'olpc3': 'f11', + 'master': None, +} +VERBOSE = False + + +class InternalError(Exception): + pass + + +class ProcessError(InternalError): + pass + + +def _invoke(program, args): + '''Run a command and raise an exception if an error occurred. + + :arg program: The program to invoke + :args: List of arguments to pass to the program + + raises ProcessError if there's a problem. + ''' + cmdLine = [program] + cmdLine.extend(args) + if VERBOSE: + print ' '.join(cmdLine) + + if VERBOSE: + program = subprocess.Popen(cmdLine, stderr=subprocess.STDOUT) + else: + program = subprocess.Popen( + cmdLine, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + retCode = program.wait() + if retCode != 0: + e = ProcessError() + e.returnCode = retCode + e.cmd = ' '.join(cmdLine) + if VERBOSE: + output = program.stdout.read() + e.message = 'Error, "%s" returned %s: %s' % ( + e.cmd, e.returnCode, output) + else: + e.message = 'Error, "%s" returned %s' % (e.cmd, e.returnCode) + raise e + + +def _create_branch(pkgname, branch): + '''Create a specific branch for a package. + + :arg pkgname: Name of the package to branch + :arg branch: Name of the branch to create + + ''' + + if branch in BRANCHES_FROM: + branch_from = BRANCHES_FROM[branch] + else: + branch_from = 'master' + + # Fall back to branching from master. + frombranchpath = os.path.join( + GIT_FOLDER, '%s.git' % pkgname, 'refs/heads', branch_from) + if not os.path.exists(frombranchpath): + branch_from = 'master' + + branchpath = os.path.join( + GIT_FOLDER, '%s.git' % pkgname, 'refs/heads', branch) + if not os.path.exists(branchpath): + try: + _invoke(MKBRANCH, ['-s', branch_from, branch, pkgname]) + except ProcessError, e: + if e.returnCode == 255: + # This is a warning, not an error + return + raise + finally: + fedmsg.publish( + topic='branch', + modname='git', + msg=dict( + agent=os.getlogin(), + name=pkgname, + branch=branch, + ), + ) + elif VERBOSE: + print 'Was asked to create branch %s of package %s, but it '\ + 'already exists' % (pkgname, branch) + + +def pkgdb_pkg_branch(): + """ Queries pkgdb information about VCS and return a dictionnary of + which branches are available for which packages. + + :return: a dict[pkg_name] = [pkg_branches] + :rtype: dict + """ + url = '%s/api/vcs' % PKGDB_URL + req = requests.get(url, params={'format': 'json'}) + data = req.json() + + output = {} + for pkg in data['packageAcls']: + if pkg in output: + if VERBOSE: + print 'Strange package: %s, it is present twice in the ' \ + 'pkgdb output' % pkg + output[pkg].updated(data['packageAcls'][pkg].keys()) + else: + output[pkg] = set(data['packageAcls'][pkg].keys()) + + return output + + +def get_git_branch(pkg): + """ For the specified package name, check the local git and return the + list of branches found. + """ + git_folder = os.path.join(GIT_FOLDER, '%s.git' % pkg) + if not os.path.exists(git_folder): + print 'Could not find %s' % git_folder + return set() + + head_folder = os.path.join(git_folder, 'refs', 'heads') + return set(os.listdir(head_folder)) + + +def branch_package(pkgname, branches): + '''Create all the branches that are listed in the pkgdb for a package. + + :arg pkgname: The package to create branches for + :arg branches: The branches to creates + + ''' + if VERBOSE: + print 'Fixing package %s for branches %s' % (pkgname, branches) + + # Create the devel branch if necessary + if not os.path.exists( + os.path.join(GIT_FOLDER, '%s.git' % pkgname)): + _invoke(SETUP_PACKAGE, [pkgname]) + + # Create all the required branches for the package + # Use the translated branch name until pkgdb falls inline + for branch in branches: + _create_branch(pkgname, branch) + + +def main(): + """ For each package found via pkgdb, check the local git for its + branches and fix inconsistencies. + """ + + local_pkgs = set(os.listdir(GIT_FOLDER)) + local_pkgs = set([it.replace('.git', '') for it in local_pkgs]) + + pkgdb_info = pkgdb_pkg_branch() + + pkgdb_pkgs = set(pkgdb_info.keys()) + + ## Commented out as we keep the git of retired packages while they won't + ## show up in the information retrieved from pkgdb. + + #if (local_pkgs - pkgdb_pkgs): + #print 'Some packages are present locally but not on pkgdb:' + #print ', '.join(sorted(local_pkgs - pkgdb_pkgs)) + + if (pkgdb_pkgs - local_pkgs): + print 'Some packages are present in pkgdb but not locally:' + print ', '.join(sorted(pkgdb_pkgs - local_pkgs)) + + tofix = set() + for pkg in sorted(pkgdb_info): + pkgdb_branches = pkgdb_info[pkg] + git_branches = get_git_branch(pkg) + diff = (pkgdb_branches - git_branches) + if diff: + print '%s missing: %s' % (pkg, ','.join(sorted(diff))) + tofix.add(pkg) + branch_package(pkg, diff) + + if tofix: + print 'Packages fixed (%s): %s' % ( + len(tofix), ', '.join(sorted(tofix))) + + +if __name__ == '__main__': + import sys + sys.exit(main())