Add the pkgdb_sync_git_branches.py script
This commit is contained in:
Pierre-Yves Chibon
parent
e2a513f270
commit
49e0ca88ea
1 changed files with 246 additions and 0 deletions
246
roles/distgit/files/pkgdb_sync_git_branches.py
Normal file
246
roles/distgit/files/pkgdb_sync_git_branches.py
Normal file
|
|
@ -0,0 +1,246 @@
|
||||||
|
#!/usr/bin/python -tt
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
"""
|
||||||
|
This program is free software; you can redistribute it and/or modify
|
||||||
|
it under the terms of the GNU General Public License as published by
|
||||||
|
the Free Software Foundation; either version 2 of the License, or
|
||||||
|
(at your option) any later version.
|
||||||
|
|
||||||
|
This program is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU General Public License for more details.
|
||||||
|
|
||||||
|
You should have received a copy of the GNU General Public License along
|
||||||
|
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||||
|
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||||
|
|
||||||
|
|
||||||
|
This script is able to query pkgdb and retrieve for all packages which active
|
||||||
|
branches should be there, browse all the git repos and find out which active
|
||||||
|
branches are missing.
|
||||||
|
|
||||||
|
It even goes one step further but actually adjusting the git repo by adding
|
||||||
|
the missing branches (or even the missing repo)
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
import fedmsg
|
||||||
|
|
||||||
|
# Do some off-the-bat configuration of fedmsg.
|
||||||
|
# 1) since this is a one-off script and not a daemon, it needs to connect
|
||||||
|
# to the fedmsg-relay process running on another node (or noone will
|
||||||
|
# hear it)
|
||||||
|
# 2) its going to use the 'shell' certificate which only 'sysadmin' has
|
||||||
|
# read access to. Contrast that with the 'scm' certificate which
|
||||||
|
# everyone in the 'packager' group has access to.
|
||||||
|
|
||||||
|
config = fedmsg.config.load_config([], None)
|
||||||
|
config['active'] = True
|
||||||
|
config['endpoints']['relay_inbound'] = config['relay_inbound']
|
||||||
|
fedmsg.init(name='relay_inbound', cert_prefix='shell', **config)
|
||||||
|
|
||||||
|
PKGDB_URL = 'https://admin.fedoraproject.org/pkgdb'
|
||||||
|
GIT_FOLDER = '/srv/git/rpms/'
|
||||||
|
MKBRANCH = '/usr/local/bin/mkbranch'
|
||||||
|
SETUP_PACKAGE = '/usr/local/bin/setup_git_package'
|
||||||
|
|
||||||
|
BRANCHES_FROM = {
|
||||||
|
'epel7': 'f19',
|
||||||
|
'olpc2': 'f7',
|
||||||
|
'olpc3': 'f11',
|
||||||
|
'master': None,
|
||||||
|
}
|
||||||
|
VERBOSE = False
|
||||||
|
|
||||||
|
|
||||||
|
class InternalError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class ProcessError(InternalError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _invoke(program, args):
|
||||||
|
'''Run a command and raise an exception if an error occurred.
|
||||||
|
|
||||||
|
:arg program: The program to invoke
|
||||||
|
:args: List of arguments to pass to the program
|
||||||
|
|
||||||
|
raises ProcessError if there's a problem.
|
||||||
|
'''
|
||||||
|
cmdLine = [program]
|
||||||
|
cmdLine.extend(args)
|
||||||
|
if VERBOSE:
|
||||||
|
print ' '.join(cmdLine)
|
||||||
|
|
||||||
|
if VERBOSE:
|
||||||
|
program = subprocess.Popen(cmdLine, stderr=subprocess.STDOUT)
|
||||||
|
else:
|
||||||
|
program = subprocess.Popen(
|
||||||
|
cmdLine, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||||
|
|
||||||
|
retCode = program.wait()
|
||||||
|
if retCode != 0:
|
||||||
|
e = ProcessError()
|
||||||
|
e.returnCode = retCode
|
||||||
|
e.cmd = ' '.join(cmdLine)
|
||||||
|
if VERBOSE:
|
||||||
|
output = program.stdout.read()
|
||||||
|
e.message = 'Error, "%s" returned %s: %s' % (
|
||||||
|
e.cmd, e.returnCode, output)
|
||||||
|
else:
|
||||||
|
e.message = 'Error, "%s" returned %s' % (e.cmd, e.returnCode)
|
||||||
|
raise e
|
||||||
|
|
||||||
|
|
||||||
|
def _create_branch(pkgname, branch):
|
||||||
|
'''Create a specific branch for a package.
|
||||||
|
|
||||||
|
:arg pkgname: Name of the package to branch
|
||||||
|
:arg branch: Name of the branch to create
|
||||||
|
|
||||||
|
'''
|
||||||
|
|
||||||
|
if branch in BRANCHES_FROM:
|
||||||
|
branch_from = BRANCHES_FROM[branch]
|
||||||
|
else:
|
||||||
|
branch_from = 'master'
|
||||||
|
|
||||||
|
# Fall back to branching from master.
|
||||||
|
frombranchpath = os.path.join(
|
||||||
|
GIT_FOLDER, '%s.git' % pkgname, 'refs/heads', branch_from)
|
||||||
|
if not os.path.exists(frombranchpath):
|
||||||
|
branch_from = 'master'
|
||||||
|
|
||||||
|
branchpath = os.path.join(
|
||||||
|
GIT_FOLDER, '%s.git' % pkgname, 'refs/heads', branch)
|
||||||
|
if not os.path.exists(branchpath):
|
||||||
|
try:
|
||||||
|
_invoke(MKBRANCH, ['-s', branch_from, branch, pkgname])
|
||||||
|
except ProcessError, e:
|
||||||
|
if e.returnCode == 255:
|
||||||
|
# This is a warning, not an error
|
||||||
|
return
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
fedmsg.publish(
|
||||||
|
topic='branch',
|
||||||
|
modname='git',
|
||||||
|
msg=dict(
|
||||||
|
agent=os.getlogin(),
|
||||||
|
name=pkgname,
|
||||||
|
branch=branch,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
elif VERBOSE:
|
||||||
|
print 'Was asked to create branch %s of package %s, but it '\
|
||||||
|
'already exists' % (pkgname, branch)
|
||||||
|
|
||||||
|
|
||||||
|
def pkgdb_pkg_branch():
|
||||||
|
""" Queries pkgdb information about VCS and return a dictionnary of
|
||||||
|
which branches are available for which packages.
|
||||||
|
|
||||||
|
:return: a dict[pkg_name] = [pkg_branches]
|
||||||
|
:rtype: dict
|
||||||
|
"""
|
||||||
|
url = '%s/api/vcs' % PKGDB_URL
|
||||||
|
req = requests.get(url, params={'format': 'json'})
|
||||||
|
data = req.json()
|
||||||
|
|
||||||
|
output = {}
|
||||||
|
for pkg in data['packageAcls']:
|
||||||
|
if pkg in output:
|
||||||
|
if VERBOSE:
|
||||||
|
print 'Strange package: %s, it is present twice in the ' \
|
||||||
|
'pkgdb output' % pkg
|
||||||
|
output[pkg].updated(data['packageAcls'][pkg].keys())
|
||||||
|
else:
|
||||||
|
output[pkg] = set(data['packageAcls'][pkg].keys())
|
||||||
|
|
||||||
|
return output
|
||||||
|
|
||||||
|
|
||||||
|
def get_git_branch(pkg):
|
||||||
|
""" For the specified package name, check the local git and return the
|
||||||
|
list of branches found.
|
||||||
|
"""
|
||||||
|
git_folder = os.path.join(GIT_FOLDER, '%s.git' % pkg)
|
||||||
|
if not os.path.exists(git_folder):
|
||||||
|
print 'Could not find %s' % git_folder
|
||||||
|
return set()
|
||||||
|
|
||||||
|
head_folder = os.path.join(git_folder, 'refs', 'heads')
|
||||||
|
return set(os.listdir(head_folder))
|
||||||
|
|
||||||
|
|
||||||
|
def branch_package(pkgname, branches):
|
||||||
|
'''Create all the branches that are listed in the pkgdb for a package.
|
||||||
|
|
||||||
|
:arg pkgname: The package to create branches for
|
||||||
|
:arg branches: The branches to creates
|
||||||
|
|
||||||
|
'''
|
||||||
|
if VERBOSE:
|
||||||
|
print 'Fixing package %s for branches %s' % (pkgname, branches)
|
||||||
|
|
||||||
|
# Create the devel branch if necessary
|
||||||
|
if not os.path.exists(
|
||||||
|
os.path.join(GIT_FOLDER, '%s.git' % pkgname)):
|
||||||
|
_invoke(SETUP_PACKAGE, [pkgname])
|
||||||
|
|
||||||
|
# Create all the required branches for the package
|
||||||
|
# Use the translated branch name until pkgdb falls inline
|
||||||
|
for branch in branches:
|
||||||
|
_create_branch(pkgname, branch)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
""" For each package found via pkgdb, check the local git for its
|
||||||
|
branches and fix inconsistencies.
|
||||||
|
"""
|
||||||
|
|
||||||
|
local_pkgs = set(os.listdir(GIT_FOLDER))
|
||||||
|
local_pkgs = set([it.replace('.git', '') for it in local_pkgs])
|
||||||
|
|
||||||
|
pkgdb_info = pkgdb_pkg_branch()
|
||||||
|
|
||||||
|
pkgdb_pkgs = set(pkgdb_info.keys())
|
||||||
|
|
||||||
|
## Commented out as we keep the git of retired packages while they won't
|
||||||
|
## show up in the information retrieved from pkgdb.
|
||||||
|
|
||||||
|
#if (local_pkgs - pkgdb_pkgs):
|
||||||
|
#print 'Some packages are present locally but not on pkgdb:'
|
||||||
|
#print ', '.join(sorted(local_pkgs - pkgdb_pkgs))
|
||||||
|
|
||||||
|
if (pkgdb_pkgs - local_pkgs):
|
||||||
|
print 'Some packages are present in pkgdb but not locally:'
|
||||||
|
print ', '.join(sorted(pkgdb_pkgs - local_pkgs))
|
||||||
|
|
||||||
|
tofix = set()
|
||||||
|
for pkg in sorted(pkgdb_info):
|
||||||
|
pkgdb_branches = pkgdb_info[pkg]
|
||||||
|
git_branches = get_git_branch(pkg)
|
||||||
|
diff = (pkgdb_branches - git_branches)
|
||||||
|
if diff:
|
||||||
|
print '%s missing: %s' % (pkg, ','.join(sorted(diff)))
|
||||||
|
tofix.add(pkg)
|
||||||
|
branch_package(pkg, diff)
|
||||||
|
|
||||||
|
if tofix:
|
||||||
|
print 'Packages fixed (%s): %s' % (
|
||||||
|
len(tofix), ', '.join(sorted(tofix)))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
import sys
|
||||||
|
sys.exit(main())
|
||||||
Loading…
Add table
Add a link
Reference in a new issue