Infrastructure/fedora-infrastructure
1
0
Fork 0
Code Issues 70 Pull requests Projects Releases Packages Wiki Activity Actions
fedora-infrastructure/scripts/process-git-requests/process-git-requests
Jason Tibbitts 22ce64372a SSH in to create branches. 
If we're not on pkgs01, SSH in to run pkgdb2branch.py.
If we are, just run it directly.
If there are errors (ssh failed, maybe) just save the package name and
  proceed as we always did.
2010-08-06 10:23:24 -05:00

694 lines
25 KiB
Python
Executable file
Raw Blame History

#!/usr/bin/python -t
VERSION = "1.0"
# TODO:
# Extract fedora-review flag setter if possible.
# Display last linked spec file.
# Download (and process?) last linked srpm
# Checks to add:
# Package already exists in pkgdb.
# fedora-review flag isn't set (especially if it's still set to '?'
# Catch common misspellings?
# Any owner contains '@' or other invalid character
# Maybe verify owners in pkgdb/FAS.
# SSH into pkgs.fedoraproject.org to run pkg2branch.py directly
# or just run on cvs.fedoraproject.org
# Try to do some checking on the ~/.bugzillacookies file and suggest "bugzilla login"
# Verify that the supplied summary has something in it.
import bugzilla
import codecs
import datetime
import getpass
import glob
import operator
import os
import re
import readline
import sys
import subprocess
import tempfile
import time
import xmlrpclib
from configobj import ConfigObj, flatten_errors
from fedora.client import AuthError, AppError, PackageDB
from optparse import OptionParser
from validate import Validator
# Red Hat's bugzilla
url = 'https://bugzilla.redhat.com/xmlrpc.cgi'
# Users who indicated that they're OK with EPEL branches. Some request that
# they be made comaintainers.
# Taken from http://fedoraproject.org/wiki/EPEL/ContributorStatusNo
epel_ok = ['abompart', 'athimm', 'corsepiu', 'ecik', 'faucamp', 'konradm',
'monnerat', 'mtasaka', 'nim', 'rafalzaq', 'rineau', 'rstrode',
'sgrubb', 'shishz', 'terjeros', 'zkota']
epel_ok_comaint = ['alexlan', 'guidograzioli', 'jwrdegoede', 'kkofler',
'mebourne', 'overholt', 'pgordon', 'rishi', 'snirkel']
PAGER = os.environ.get('PAGER') or '/usr/bin/less'
EDITOR = os.environ.get('EDITOR') or '/usr/bin/vim'
# Override a method in xmlrpclib so it doesn't blow up when getting crap data
# from Red Hat's bugzilla.
# Bugfixes seem to have rendered this unnecessary
#def _decode(data, encoding, is8bit=re.compile("[\x80-\xff]").search):
# # decode non-ascii string (if possible)
# if unicode and encoding and is8bit(data):
# data = unicode(data, encoding, 'replace')
# return data
#xmlrpclib._decode = _decode
def clear():
#os.system('clearl)
print "\n============================================================\n"
def parse_commandline():
usage = 'usage: %prog [options]'
parser = OptionParser(usage)
parser.add_option('--url', dest='url',
help='bugzilla URL to query',
default=url)
parser.add_option('--pkghost',
help='Hostname of the machine where branches must be created',
dest='pkghost',
default='pkgs.fedoraproject.org')
parser.add_option('--pkghostlocal',
help='Local hostname of the machine where branches must be created',
dest='pkghostlocal',
default='pkgs01.phx2.fedoraproject.org')
parser.add_option('-u', '--user',
help='Username for PackageDB connection',
dest='user',
default=getpass.getuser())
parser.add_option('--debug',
action='store_true',
dest='debug',
default=False,
help='Turn on some debugging statements')
(options, args) = parser.parse_args()
return options
def parse_pkgdb_config():
vldtr = Validator()
# configspec to validate types and set defaults
configspec = '''
[global]
pkgdb.url = string(default = 'https://admin.fedoraproject.org/pkgdb')
pkgdb.retries = integer(default = 5)
pkgdb.knowngroups = list(default = list())
'''.splitlines()
cfg = ConfigObj('/etc/pkgdb-client.cfg', configspec=configspec)
user = ConfigObj(os.path.expanduser('~/.fedora/pkgdb-client.cfg'),
configspec=configspec)
cfg.merge(user)
res = cfg.validate(vldtr, preserve_errors=True)
for entry in flatten_errors(cfg, res):
section_list, key, error = entry
section_list.append(key)
section_string = ','.join(section_list)
if error == False:
error = 'Missing value or section.'
print ','.join(section_list), '=', error
sys.exit(1)
cfg['global']['pkgdb.url'] = os.environ.get('PACKAGEDBURL') or cfg['global']['pkgdb.url']
return cfg['global']
def encode_utf8(object, encoding='utf8', errors='replace'):
if isinstance(object, basestring):
if isinstance(object, str):
return unicode(object, encoding, errors)
else:
return object
return u''
def add_package(pkgdb, request):
for retry in range(1, config['pkgdb.retries'] + 1):
try:
pkgdb.add_package(pkg=request['pkg'],
owner=request['owner'],
description=request['description'],
branches=request['branches'],
cc_list=request['cc_list'],
comaintainers=request['comaintainers'])
except AuthError, e:
if sys.stdin.isatty():
if retry >= config['pkgdb.retries']:
break
pkgdb.password = getpass.getpass('PackageDB Password: ')
else:
# Don't retry if we're reading the password from stdin
break
else:
break
def edit_package(pkgdb, request):
for retry in range(1, config['pkgdb.retries'] + 1):
try:
pkgdb.edit_package(pkg=request['pkg'],
owner=request['owner'],
branches=request['newbranches'],
cc_list=request['cc_list'],
comaintainers=request['comaintainers'])
except AuthError, e:
if retry >= config['pkgdb.retries']:
break
pkgdb.password = getpass.getpass('PackageDB Password: ')
else:
break
def run_query(bz):
querydata = {}
querydata['column_list'] = ['opendate', 'changeddate', 'bug_severity',
'alias', 'assigned_to', 'reporter', 'bug_status', 'resolution',
'component', 'blockedby', 'dependson', 'short_desc',
'status_whiteboard', 'flag_types']
querydata['field0-0-0'] = 'flagtypes.name'
querydata['type0-0-0'] = 'equals'
querydata['value0-0-0'] = 'fedora-cvs?'
bugs = bz.query(querydata)
bugs.sort(key=operator.attrgetter('bug_id'))
ids = map(lambda x: x.bug_id, bugs)
comments = bz._proxy.Bug.comments({"ids": ids})
return [bugs, comments]
def display_bug(bug, comments):
'''Show the complete ticket in a pager.'''
comment = 0
b = []
b.append('https://bugzilla.redhat.com/%d' % bug.bug_id)
b.append('Bug %d - %s' % (bug.bug_id, bug.short_desc))
b.append('Reported by: %s at %s' % (bug.reporter, bug.opendate))
b.append('Assigned to: %s' % (bug.assigned_to))
for i in comments:
b.append('-'*40)
#b.append('Comment %d by %s at %s\n' % (comment, i['author'], time.strftime('%F %T',i['time'].timetuple())))
#b.append('Comment %d by %s at %04d-%02d-%02d %02d:%02d%02d\n' % (
b.append('Comment %d by %s at %s\n' % (
comment, i['author'], i['time']))
b.append(i['text'])
b.append('')
comment += 1
p = subprocess.Popen(PAGER, stdin=subprocess.PIPE)
p.communicate('\n'.join(b).encode('utf8'))
def edit_string(s):
'''Edit the contents of a string in the user's preferred editor.'''
(fd, f) = tempfile.mkstemp()
fh=os.fdopen(fd, 'w+')
fh.write(s)
fh.close()
p = subprocess.Popen([EDITOR, f]);
sts = os.waitpid(p.pid, 0)[1]
if not sts:
try:
fh = open(f, 'r')
s = fh.read()
finally:
fh.close()
return s
def parse_prefixed_lines(s):
lastitem = ''
items = {}
items['Branches'] = ''
items['New Branches'] = ''
lines = s.splitlines()
# Skip until the Request line
while 1:
if (lines[0].find('New Package CVS Request') == 0
or lines[0].find('New Package GIT Request') == 0
or lines[0].find('New Package SCM Request') == 0
or lines[0].find('Package Change Request') == 0):
break
lines.pop(0)
# Skip until a line containing a colon
while 1:
if lines[0].find(':') >= 0:
break
lines.pop(0)
# Now parse
while 1:
if not len(lines):
break
line = lines.pop(0)
line.strip()
if len(line) == 0:
break
pos = line.find(':')
# Line-wrapped?
if pos < 0:
items[lastitem] += " " + line.strip()
continue
lastitem = line[:pos]
items[lastitem] = line[pos+1:].strip()
return items
def clean_branches(branches):
'''Clean up a list of branches and turn them into what pkgdb expects.'''
branches = re.sub(r',', ' ', branches)
branches = re.sub(r'F', 'f', branches)
branches = re.sub(r'EL', 'el', branches)
branches = re.sub(r'devel', ' ', branches)
branches = re.sub(r'master', ' ', branches)
branches = re.sub(r'f-([1-9][0-9])', r'f\1', branches)
branches = re.sub(r'el-([1-9])', r'el\1', branches)
branches = re.sub(r' +', ' ', branches)
# Now make things nasty to satisfy history
branches = re.sub(r'f12', r'F-12', branches)
branches = re.sub(r'f13', r'F-13', branches)
branches = re.sub(r'el4', r'EL-4', branches)
branches = re.sub(r'el5', r'EL-5', branches)
branches = re.sub(r'el6', r'EL-6', branches)
branches = branches.strip()
return branches
def clean_request(items):
'''Clean up various bits that can be passed in a request.'''
request = {}
if not 'InitialCC' in items:
items['InitialCC'] = ''
if not 'Owners' in items:
items['Owners'] = ''
if not 'Short Description' in items:
items['Short Description'] = ''
branches = clean_branches(items['Branches'].strip())
branches += ' devel'
items['Branches'] = branches
request['branches'] = branches.split()
branches = clean_branches(items['New Branches'].strip())
items['New Branches'] = branches
request['newbranches'] = branches.split()
owners = items['Owners'].strip()
owners = re.sub(r',', ' ', owners)
if len(owners):
request['owner'] = owners.split()[0]
request['comaintainers'] = owners.split()[1:]
request['cc_list'] = items['InitialCC'].split()
request['pkg'] = items['Package Name']
request['description'] = items['Short Description']
return request
def new_request_string(items, bug):
r = []
r.append("Bug URL: http://bugzilla.redhat.com/%d " % bug.bug_id)
r.append("Bug summary: " + bug.short_desc)
r.append('')
r.append("New Package SCM Request")
r.append("=======================")
r.append("Package Name: " + items['Package Name'])
r.append("Short Description: " + items['Short Description'])
r.append("Owners: " + items['Owners'])
r.append("Branches: " + items['Branches'])
r.append("InitialCC: " + items['InitialCC'])
r.append('')
return '\n'.join(r)
def change_request_string(items, bug):
r = []
r.append("Bug URL: http://bugzilla.redhat.com/%d" % bug.bug_id)
r.append("Bug summary: " + bug.short_desc)
r.append('')
r.append("Package Change Request")
r.append("======================")
r.append("Package Name: " + items['Package Name'])
r.append("Owners: " + items['Owners'])
r.append("New Branches: " + items['New Branches'])
r.append("InitialCC: " + items['InitialCC'])
r.append('')
return '\n'.join(r)
def get_pkgdb_owners(pkgdb, pkg):
owners = {}
o = ''
for i in pkgdb.get_owners(pkg)['packageListings']:
branch = i['collection']['branchname']
if branch not in branches:
continue
owners[branch] = {}
owners[branch]['primary'] = i['owner']
owners[branch]['comaint'] = []
for j in i['people']:
#if 'commit' in j['aclOrder']:
if j['aclOrder']['commit'] != None and j['username'] != owners[branch]:
owners[branch]['comaint'].append(j['username'])
for i in sorted(branches, reverse=True):
if i in owners:
o += "%s: %s" % (i, owners[i]['primary'])
if len(owners[i]['comaint']):
o += ' - %s' % ','.join(sorted(owners[i]['comaint']))
o += '\n'
return (owners, o)
def process_no_request(bug, allcomments):
'''Deal with a ticket where no request was found.'''
while 1:
clear()
print "No SCM request found in bug %d\nhttp://bugzilla.redhat.com/%d." % (bug.bug_id, bug.bug_id)
ok = raw_input('\nWhat do? (n=Next, s=Show ticket, c=Comment, q=Quit):')
if ok == 'c':
bug_comment = edit_string('')
print bug_comment
ok = raw_input("\nPost this comment to the ticket (y/n)?")
if ok == 'y':
print "Updating bugzilla..."
bug.addcomment(bug_comment)
ok = raw_input("\nClear the fedora-cvs flag (y/n)?")
if ok == 'y':
print "Clearing the flag..."
bug.updateflags({'fedora-cvs':'X', 'nomail':1})
break
elif ok == 'n':
return True
elif ok == 'q':
return False
elif ok == 's':
print
display_bug(bug, allcomments)
return True
def process_new_request(bug, comment, allcomments, firstfound, pkgdb, branches):
'''Parse a new package request, try to repair line wrapping, and do some
basic validity checks.'''
warned = False
warnings = []
items = parse_prefixed_lines(comment['text'])
request = clean_request(items)
if not firstfound:
warnings.append("WARNING: SCM request was not the last comment.")
warned = True
if not 'Package Name' in items:
warnings.append("WARNING: No package name supplied.")
warned = True
if not 'Owners' in items:
warnings.append("WARNING: No owners provided.")
warned = True
if not 'Short Description' in items:
warnings.append("WARNING: No description provided.")
warned = True
for i in request['branches']:
if i not in branches:
warnings.append("WARNING: Invalid branch %s requested" % i)
warned = True
short_desc = bug.short_desc
m=re.search('Review Request:\s([a-zA-Z0-9_+.-]+)\s+', short_desc, re.I)
if not m:
warnings.append("WARNING: Couldn't parse package name out of bug summary.")
warned = True
elif m.group(1) != items['Package Name']:
warnings.append("WARNING: Requested package name %s doesn't match bug summary %s" % (items['Package Name'], m.group(1)))
warned = True
req_string = new_request_string(items, bug)
bug_comment = 'Git done (by process-git-requests).\n'
okprompt = 'Do it (yes=Yes, n=No, e=Edit request, s=Show ticket, c=Comment, q=Quit)?'
if warned:
prompt = 'Warnings present!\nDo it (a=Accept warnings, n=No, e=Edit request, s=Show ticket, c=Comment, q=Quit)?'
else:
prompt = okprompt
# We have to loop until the user accepts the request
while 1:
# We have to loop until the user enters something that works
while 1:
clear()
if len(warnings):
print '\n'.join(warnings), "\n"
print "Currently assigned to: %s" % bug.assigned_to
print req_string
ok = raw_input(prompt)
if ok == 'a':
prompt = okprompt
warned = False
if ok == 'c':
bug_comment = edit_string('')
print bug_comment
ok = raw_input("\nPost this comment to the ticket (y/n)?")
if ok == 'y':
print "Updating bugzilla..."
bug.addcomment(bug_comment)
ok = raw_input("\nClear the fedora-cvs flag (y/n)?")
if ok == 'y':
print "Clearing the flag..."
bug.updateflags({'fedora-cvs':'X', 'nomail':1})
return (False, True)
elif ok == 'e':
req_string = edit_string(req_string)
items=parse_prefixed_lines(req_string)
request = clean_request(items)
req_string = new_request_string(items, bug)
break
elif ok == 'n':
return (False, True)
elif ok == 'q':
return (False, False)
elif ok == 's':
print
display_bug(bug, allcomments)
elif ok == 'yes' and not warned:
bug_comment = edit_string(bug_comment)
print '\n', bug_comment
ok = raw_input('Go ahead (y/n)?')
if ok != 'y':
break
print 'Calling pkgdb...'
try:
add_package(pkgdb, request)
except Exception, e:
print "Pkgdb call failed:"
print e
raw_input('\nPress enter to continue to the next ticket.')
return (False, True)
print 'Updating bugzilla...'
# XXX Need to handle errors here
bug.updateflags({'fedora-cvs':'+', 'nomail':1})
bug.addcomment(bug_comment)
return (request['pkg'], True)
else:
pass
def process_change_request(bug, comment, allcomments, firstfound, pkgdb, branches):
'''Parse a change request, try to repair line wrapping, and do some
basic validity checks.'''
owned = False
warned = False
warnings = []
items = parse_prefixed_lines(comment['text'])
request = clean_request(items)
print "Looking up owners in pkgdb..."
(owners, owner_string) = get_pkgdb_owners(pkgdb, items['Package Name'])
# Try to enforce EPEL branch rules
for i in owners.keys():
if request['owner'] == owners[i]['primary'] or request['owner'] in owners[i]['comaint']:
owned = True
if not owned and items['New Branches'].find('EL') >= 0 and owners['devel']['primary'] in epel_ok:
warnings.append("NOTE: new branch owner not owner of other branches,\n but primary devel owner is OK with EPEL branches.")
elif not owned and items['New Branches'].find('EL') >= 0 and owners['devel']['primary'] in epel_ok_comaint:
warnings.append("NOTE: new branch owner not owner of other branches,\n but primary devel owner is OK with EPEL branches\n as long as they comaintain.")
elif not owned:
warnings.append("WARNING: new branch owner not owner of other branches.")
warned = True
if not firstfound:
warnings.append("WARNING: SCM request was not the last comment.")
warned = True
if not 'Package Name' in items:
warnings.append("WARNING: No package name supplied.")
warned = True
if not 'Owners' in items:
warnings.append("WARNING: No owners provided.")
warned = True
if not len(items['New Branches']):
warnings.append("WARNING: No new branches requested.")
for i in request['branches']:
if i not in branches:
warnings.append("WARNING: Invalid branch %s requested" % i)
warned = True
short_desc = bug.short_desc
req_string = change_request_string(items, bug)
bug_comment = 'Git done (by process-git-requests).\n'
okprompt = 'Do it (yes=Yes, n=No, e=Edit request, s=Show ticket, c=Comment, q=Quit)?'
if warned:
prompt = 'Warnings present!\nDo it (a=Accept warnings, n=No, e=Edit request, s=Show ticket, c=Comment, q=Quit)?'
else:
prompt = okprompt
# We have to loop until the user accepts the request
while 1:
# We have to loop until the user enters something that works
while 1:
clear()
if len(warnings):
print '\n'.join(warnings), "\n"
print req_string + "\nCurrent branch owners - comaintainers:\n" + owner_string
ok = raw_input(prompt)
if ok == 'a':
prompt = okprompt
warned = False
if ok == 'c':
bug_comment = edit_string('')
print bug_comment
ok = raw_input("\nPost this comment to the ticket (y/n)?")
if ok == 'y':
print "Updating bugzilla..."
bug.addcomment(bug_comment)
ok = raw_input("\nClear the fedora-cvs flag (y/n)?")
if ok == 'y':
print "Clearing the flag..."
bug.updateflags({'fedora-cvs':'X', 'nomail':1})
return (False, True)
elif ok == 'e':
req_string = edit_string(req_string)
items=parse_prefixed_lines(req_string)
request = clean_request(items)
req_string = change_request_string(items, bug)
break
elif ok == 'n':
return (False, True)
elif ok == 'q':
return (False, False)
elif ok == 's':
print
display_bug(bug, allcomments)
elif ok == 'yes' and not warned:
bug_comment = edit_string(bug_comment)
print '\n', bug_comment
ok = raw_input('Go ahead (y/n)?')
if ok != 'y':
break
print 'Calling pkgdb...'
try:
edit_package(pkgdb, request)
except Exception, e:
print "Pkgdb call failed:"
print e
raw_input('\nPress enter to continue to the next ticket.')
return (False, True)
print 'Updating bugzilla...'
# XXX Need to handle errors here
bug.updateflags({'fedora-cvs':'+', 'nomail':1})
bug.addcomment(bug_comment)
return (request['pkg'], True)
else:
pass
def create_branches(package, pkghost, pkghostlocal):
'''If on pkgs01, run /usr/local/bin/pkgdb2branch.py directly. Otherwise,
call ssh.'''
hostname = os.uname()[1]
cmd = []
if hostname != pkghostlocal:
cmd.extend(['ssh', pkghost])
cmd.extend(['/usr/local/bin/pkgdb2branch.py', package])
print "Calling pkgdb2branch.py...."
proc = subprocess.check_call(cmd, stdout=sys.stdout, stderr=sys.stderr)
print
if __name__ == '__main__':
branches = {}
processed = []
options = parse_commandline()
print "Connecting to bugzilla..."
bz = bugzilla.Bugzilla(url=options.url)
print "Querying bugzilla..."
(bugs, comments) = run_query(bz)
print "Done; got %d." % len(bugs)
if not len(bugs):
print "No requests to process!"
exit(0)
print "Connecting to pkgdb..."
config = parse_pkgdb_config()
pkgdb = PackageDB(config['pkgdb.url'], username=options.user,
debug=options.debug)
print "Getting valid branches...."
for i in pkgdb.get_collection_list(eol=False):
branches[i[0]['branchname']] = 1
print "Done."
print
# Iterate over bugs
for i in bugs:
firstfound = True
type = ''
print "Parsing bug %d - https://bugzilla.redhat.com/%d" % (i.bug_id, i.bug_id)
for j in reversed(comments['bugs'][str(i.bug_id)]['comments']):
if ('New Package CVS Request' in j['text']
or 'New Package GIT Request' in j['text']
or 'New Package SCM Request' in j['text']):
type = 'new'
break
if 'Package Change Request' in j['text']:
type = 'change'
break
firstfound = False
else:
if not process_no_request(i, comments['bugs'][str(i.bug_id)]['comments']):
break
if type == 'new':
(package, more) = process_new_request(i, j, comments['bugs'][str(i.bug_id)]['comments'], firstfound, pkgdb, branches)
if package:
try:
create_branches(package, options.pkghost, options.pkghostlocal)
except:
processed.append(package)
if not more:
break
elif type == 'change':
(package, more) = process_change_request(i, j, comments['bugs'][str(i.bug_id)]['comments'], firstfound, pkgdb, branches)
if package:
try:
create_branches(package, options.pkghost, options.pkghostlocal)
except:
processed.append(package)
if not more:
break
if len(processed):
print '\nYou must now run this on the git server\nto set up the git repository:'
print '/usr/local/bin/pkgdb2branch.py ' + ' '.join(processed)
sys.exit(0)
Reference in a new issue View git blame Copy permalink