diff --git a/roles/distgit/files/process-git-requests b/roles/distgit/files/process-git-requests deleted file mode 100644 index c2bb9b2d4b..0000000000 --- a/roles/distgit/files/process-git-requests +++ /dev/null @@ -1,654 +0,0 @@ -#!/usr/bin/python -t -VERSION = "1.0" - -# $Id: process-cvs-requests.py,v 1.25 2010/07/01 04:51:23 tibbs Exp $ - -# TODO: -# Extract fedora-review flag setter if possible. -# Display last linked spec file. -# Download (and process?) last linked srpm - -# Checks to add: -# Package already exists in pkgdb. -# fedora-review flag isn't set (especially if it's still set to '?' -# Catch common misspellings? -# Any owner contains '@' or other invalid character -# Maybe verify owners in pkgdb/FAS. -# SSH into cvs.fedoraproject.org to run pkg2branch.py directly -# or just run on cvs.fedoraproject.org -# Try to do some checking on the ~/.bugzillacookies file and suggest "bugzilla login" - -import bugzilla -import codecs -import datetime -import getpass -import glob -import operator -import os -import re -import readline -import sys -import subprocess -import tempfile -import time -import xmlrpclib -from configobj import ConfigObj, flatten_errors -from fedora.client import AuthError, AppError, PackageDB -from optparse import OptionParser -from validate import Validator - -# Red Hat's bugzilla -url = 'https://bugzilla.redhat.com/xmlrpc.cgi' - -# Users who indicated that they're OK with EPEL branches. Some request that -# they be made comaintainers. -# Taken from http://fedoraproject.org/wiki/EPEL/ContributorStatusNo -epel_ok = ['abompart', 'athimm', 'corsepiu', 'ecik', 'faucamp', 'konradm', - 'monnerat', 'mtasaka', 'nim', 'rafalzaq', 'rineau', 'rstrode', - 'sgrubb', 'shishz', 'terjeros', 'zkota'] -epel_ok_comaint = ['alexlan', 'guidograzioli', 'jwrdegoede', 'kkofler', - 'mebourne', 'overholt', 'pgordon', 'rishi', 'snirkel'] - -PAGER = os.environ.get('PAGER') or '/usr/bin/less' -EDITOR = os.environ.get('EDITOR') or '/usr/bin/vim' - -# Override a method in xmlrpclib so it doesn't blow up when getting crap data -# from Red Hat's bugzilla. -# Bugfixes seem to have rendered this unnecessary -#def _decode(data, encoding, is8bit=re.compile("[\x80-\xff]").search): -# # decode non-ascii string (if possible) -# if unicode and encoding and is8bit(data): -# data = unicode(data, encoding, 'replace') -# return data -#xmlrpclib._decode = _decode - -def parse_commandline(): - usage = 'usage: %prog [options]' - parser = OptionParser(usage) - parser.add_option('--url', dest='url', - help='bugzilla URL to query', - default=url) - parser.add_option('-u', '--user', - help='Username for PackageDB connection', - dest='user', - default=getpass.getuser()) - parser.add_option('--debug', - action='store_true', - dest='debug', - default=False, - help='Turn on some debugging statements') - - (options, args) = parser.parse_args() - return options - -def parse_pkgdb_config(): - vldtr = Validator() - # configspec to validate types and set defaults - configspec = ''' - [global] - pkgdb.url = string(default = 'https://admin.fedoraproject.org/pkgdb') - pkgdb.retries = integer(default = 5) - pkgdb.knowngroups = list(default = list()) - '''.splitlines() - - cfg = ConfigObj('/etc/pkgdb-client.cfg', configspec=configspec) - user = ConfigObj(os.path.expanduser('~/.fedora/pkgdb-client.cfg'), - configspec=configspec) - cfg.merge(user) - res = cfg.validate(vldtr, preserve_errors=True) - - for entry in flatten_errors(cfg, res): - section_list, key, error = entry - section_list.append(key) - section_string = ','.join(section_list) - if error == False: - error = 'Missing value or section.' - print ','.join(section_list), '=', error - sys.exit(1) - - cfg['global']['pkgdb.url'] = os.environ.get('PACKAGEDBURL') or cfg['global']['pkgdb.url'] - return cfg['global'] - -def encode_utf8(object, encoding='utf8', errors='replace'): - if isinstance(object, basestring): - if isinstance(object, str): - return unicode(object, encoding, errors) - else: - return object - return u'' - -def add_package(pkgdb, request): - for retry in range(1, config['pkgdb.retries'] + 1): - try: - pkgdb.add_package(pkg=request['pkg'], - owner=request['owner'], - description=request['description'], - branches=request['branches'], - cc_list=request['cc_list'], - comaintainers=request['comaintainers']) - except AuthError, e: - if sys.stdin.isatty(): - if retry >= config['pkgdb.retries']: - break - pkgdb.password = getpass.getpass('PackageDB Password: ') - else: - # Don't retry if we're reading the password from stdin - break - else: - break - -def edit_package(pkgdb, request): - for retry in range(1, config['pkgdb.retries'] + 1): - try: - pkgdb.edit_package(pkg=request['pkg'], - owner=request['owner'], - branches=request['newbranches'], - cc_list=request['cc_list'], - comaintainers=request['comaintainers']) - except AuthError, e: - if retry >= config['pkgdb.retries']: - break - pkgdb.password = getpass.getpass('PackageDB Password: ') - else: - break - -def run_query(bz): - querydata = {} - querydata['column_list'] = ['opendate', 'changeddate', 'bug_severity', - 'alias', 'assigned_to', 'reporter', 'bug_status', 'resolution', - 'component', 'blockedby', 'dependson', 'short_desc', - 'status_whiteboard', 'flag_types'] - querydata['product'] = ['Fedora'] - - querydata['field0-0-0'] = 'flagtypes.name' - querydata['type0-0-0'] = 'equals' - querydata['value0-0-0'] = 'fedora-cvs?' - - bugs = bz.query(querydata) - bugs.sort(key=operator.attrgetter('bug_id')) - - ids = map(lambda x: x.bug_id, bugs) - comments = bz._proxy.Bug.comments({"ids": ids}) - - return [bugs, comments] - -def display_bug(bug, comments): - '''Show the complete ticket in a pager.''' - comment = 0 - b = [] - b.append('https://bugzilla.redhat.com/%d' % bug.bug_id) - b.append('Bug %d - %s' % (bug.bug_id, bug.short_desc)) - b.append('Reported by: %s at %s' % (bug.reporter, bug.opendate)) - b.append('Assigned to: %s' % (bug.assigned_to)) - for i in comments: - b.append('-'*40) - #b.append('Comment %d by %s at %s\n' % (comment, i['author'], time.strftime('%F %T',i['time'].timetuple()))) - #b.append('Comment %d by %s at %04d-%02d-%02d %02d:%02d%02d\n' % ( - b.append('Comment %d by %s at %s\n' % ( - comment, i['author'], i['time'])) - b.append(i['text']) - b.append('') - comment += 1 - - p = subprocess.Popen(PAGER, stdin=subprocess.PIPE) - p.communicate('\n'.join(b).encode('utf8')) - - -def edit_string(s): - '''Edit the contents of a string in the user's preferred editor.''' - (fd, f) = tempfile.mkstemp() - fh=os.fdopen(fd, 'w+') - fh.write(s) - fh.close() - p = subprocess.Popen([EDITOR, f]); - sts = os.waitpid(p.pid, 0)[1] - if not sts: - try: - fh = open(f, 'r') - s = fh.read() - finally: - fh.close() - - return s - - -def parse_prefixed_lines(s): - lastitem = '' - items = {} - items['Branches'] = '' - lines = s.splitlines() - - # Skip until the Request line - while 1: - if (lines[0].find('New Package CVS Request') == 0 - or lines[0].find('Package Change Request') == 0): - break - lines.pop(0) - - # Skip until a line containing a colon - while 1: - if lines[0].find(':') >= 0: - break - lines.pop(0) - - # Now parse - while 1: - if not len(lines): - break - - line = lines.pop(0) - line.strip() - if len(line) == 0: - break - - pos = line.find(':') - - # Line-wrapped? - if pos < 0: - items[lastitem] += " " + line.strip() - continue - - lastitem = line[:pos] - items[lastitem] = line[pos+1:].strip() - - return items - -def clean_request(items): - '''Clean up various bits that can be passed in a CVS request.''' - request = {} - - if not 'InitialCC' in items: - items['InitialCC'] = '' - if not 'Owners' in items: - items['Owners'] = '' - if not 'Short Description' in items: - items['Short Description'] = '' - - branches = items['Branches'].strip() - branches = re.sub(r',', ' ', branches) - branches = re.sub(r'f', 'F', branches) - branches = re.sub(r'devel', ' ', branches) - branches = re.sub(r'F([1-9][0-9])', r'F-\1', branches) - branches = re.sub(r'EL([1-9])', r'EL-\1', branches) - branches = re.sub(r'F-14', r'f14', branches) - branches = re.sub(r' +', ' ', branches) - branches = branches.strip() - branches += ' devel' - items['Branches'] = branches - request['branches'] = branches.split() - - if 'New Branches' in items: - branches = items['New Branches'].strip() - branches = re.sub(r',', ' ', branches) - branches = re.sub(r'f', 'F', branches) - branches = re.sub(r'F([1-9][0-9])', r'F-\1', branches) - branches = re.sub(r'F-14', r'f14', branches) - branches = re.sub(r' +', ' ', branches) - branches = branches.strip() - items['New Branches'] = branches - request['newbranches'] = branches.split() - - owners = items['Owners'].strip() - owners = re.sub(r',', ' ', owners) - if len(owners): - request['owner'] = owners.split()[0] - request['comaintainers'] = owners.split()[1:] - - request['cc_list'] = items['InitialCC'].split() - request['pkg'] = items['Package Name'] - request['description'] = items['Short Description'] - - return request - -def new_request_string(items, bug): - r = [] - r.append("Bug URL: http://bugzilla.redhat.com/%d " % bug.bug_id) - r.append("Bug summary: " + bug.short_desc) - r.append('') - r.append("New Package CVS Request") - r.append("=======================") - r.append("Package Name: " + items['Package Name']) - r.append("Short Description: " + items['Short Description']) - r.append("Owners: " + items['Owners']) - r.append("Branches: " + items['Branches']) - r.append("InitialCC: " + items['InitialCC']) - r.append('') - return '\n'.join(r) - -def change_request_string(items, bug): - r = [] - r.append("Bug URL: http://bugzilla.redhat.com/%d" % bug.bug_id) - r.append("Bug summary: " + bug.short_desc) - r.append('') - r.append("Package Change Request") - r.append("======================") - r.append("Package Name: " + items['Package Name']) - r.append("Owners: " + items['Owners']) - r.append("New Branches: " + items['New Branches']) - r.append("InitialCC: " + items['InitialCC']) - r.append('') - return '\n'.join(r) - -def get_pkgdb_owners(pkgdb, pkg): - owners = {} - o = '' - for i in pkgdb.get_owners(pkg)['packageListings']: - branch = i['collection']['branchname'] - if branch not in branches: - continue - - owners[branch] = {} - owners[branch]['primary'] = i['owner'] - owners[branch]['comaint'] = [] - for j in i['people']: - #if 'commit' in j['aclOrder']: - if j['aclOrder']['commit'] != None and j['username'] != owners[branch]: - owners[branch]['comaint'].append(j['username']) - - for i in sorted(branches, reverse=True): - if i in owners: - o += "%s: %s" % (i, owners[i]['primary']) - if len(owners[i]['comaint']): - o += ' - %s' % ','.join(sorted(owners[i]['comaint'])) - o += '\n' - - return (owners, o) - -def process_no_request(bug, allcomments): - '''Deal with a ticket where no request was found.''' - while 1: - os.system('clear') - print "No CVS request found in bug %d\nhttp://bugzilla.redhat.com/%d." % (bug.bug_id, bug.bug_id) - ok = raw_input('\nWhat do? (n=Next, s=Show ticket, c=Comment, q=Quit):') - if ok == 'c': - bug_comment = edit_string('') - print bug_comment - ok = raw_input("\nPost this comment to the ticket (y/n)?") - if ok == 'y': - print "Updating bugzilla..." - bug.addcomment(bug_comment) - ok = raw_input("\nClear the fedora-cvs flag (y/n)?") - if ok == 'y': - print "Clearing the flag..." - bug.updateflags({'fedora-cvs':'X', 'nomail':1}) - break - elif ok == 'n': - return True - elif ok == 'q': - return False - elif ok == 's': - print - display_bug(bug, allcomments) - return True - -def process_new_request(bug, comment, allcomments, firstfound, pkgdb, branches): - '''Parse a new package request, try to repair line wrapping, and do some - basic validity checks.''' - warned = False - warnings = [] - items = parse_prefixed_lines(comment['text']) - request = clean_request(items) - - if not firstfound: - warnings.append("WARNING: CVS request was not the last comment.") - warned = True - if not 'Package Name' in items: - warnings.append("WARNING: No package name supplied.") - warned = True - if not 'Owners' in items: - warnings.append("WARNING: No owners provided.") - warned = True - if not 'Short Description' in items: - warnings.append("WARNING: No description provided.") - warned = True - for i in request['branches']: - if i not in branches: - warnings.append("WARNING: Invalid branch %s requested" % i) - warned = True - - short_desc = bug.short_desc - m=re.search('Review Request:\s([a-zA-Z0-9_+.-]+)\s+', short_desc, re.I) - if not m: - warnings.append("WARNING: Couldn't parse package name out of bug summary.") - warned = True - elif m.group(1) != items['Package Name']: - warnings.append("WARNING: Requested package name %s doesn't match bug summary %s" % (items['Package Name'], m.group(1))) - warned = True - - req_string = new_request_string(items, bug) - bug_comment = 'GIT done (by process-git-requests).\n' - - okprompt = 'Do it (yes=Yes, n=No, e=Edit request, s=Show ticket, c=Comment, q=Quit)?' - if warned: - prompt = 'Warnings present!\nDo it (a=Accept warnings, n=No, e=Edit request, s=Show ticket, c=Comment, q=Quit)?' - else: - prompt = okprompt - - # We have to loop until the user accepts the request - while 1: - # We have to loop until the user enters something that works - while 1: - os.system('clear') - if len(warnings): - print '\n'.join(warnings), "\n" - print "Currently assigned to: %s" % bug.assigned_to - print req_string - ok = raw_input(prompt) - if ok == 'a': - prompt = okprompt - warned = False - if ok == 'c': - bug_comment = edit_string('') - print bug_comment - ok = raw_input("\nPost this comment to the ticket (y/n)?") - if ok == 'y': - print "Updating bugzilla..." - bug.addcomment(bug_comment) - ok = raw_input("\nClear the fedora-cvs flag (y/n)?") - if ok == 'y': - print "Clearing the flag..." - bug.updateflags({'fedora-cvs':'X', 'nomail':1}) - return (False, True) - elif ok == 'e': - req_string = edit_string(req_string) - items=parse_prefixed_lines(req_string) - request = clean_request(items) - req_string = new_request_string(items, bug) - break - elif ok == 'n': - return (False, True) - elif ok == 'q': - return (False, False) - elif ok == 's': - print - display_bug(bug, allcomments) - elif ok == 'yes' and not warned: - bug_comment = edit_string(bug_comment) - print '\n', bug_comment - ok = raw_input('Go ahead (y/n)?') - if ok != 'y': - break - print 'Calling pkgdb...' - try: - add_package(pkgdb, request) - except Exception, e: - print "Pkgdb call failed:" - print e - raw_input('\nPress enter to continue to the next ticket.') - return (False, True) - - print 'Updating bugzilla...' - # XXX Need to handle errors here - bug.updateflags({'fedora-cvs':'+', 'nomail':1}) - bug.addcomment(bug_comment) - return (request['pkg'], True) - else: - pass - -def process_change_request(bug, comment, allcomments, firstfound, pkgdb, branches): - '''Parse a change request, try to repair line wrapping, and do some - basic validity checks.''' - owned = False - warned = False - warnings = [] - items = parse_prefixed_lines(comment['text']) - request = clean_request(items) - print "Looking up owners in pkgdb..." - (owners, owner_string) = get_pkgdb_owners(pkgdb, items['Package Name']) - - # Try to enforce EPEL branch rules - for i in owners.keys(): - if request['owner'] == owners[i]['primary'] or request['owner'] in owners[i]['comaint']: - owned = True - if not owned and items['New Branches'].find('EL') >= 0 and owners['devel']['primary'] in epel_ok: - warnings.append("NOTE: new branch owner not owner of other branches,\n but primary devel owner is OK with EPEL branches.") - elif not owned and items['New Branches'].find('EL') >= 0 and owners['devel']['primary'] in epel_ok_comaint: - warnings.append("NOTE: new branch owner not owner of other branches,\n but primary devel owner is OK with EPEL branches\n as long as they comaintain.") - elif not owned: - warnings.append("WARNING: new branch owner not owner of other branches.") - warned = True - - if not firstfound: - warnings.append("WARNING: GIT request was not the last comment.") - warned = True - if not 'Package Name' in items: - warnings.append("WARNING: No package name supplied.") - warned = True - if not 'Owners' in items: - warnings.append("WARNING: No owners provided.") - warned = True - if not 'New Branches' in items: - warnings.append("WARNING: No new branches requested.") - for i in request['branches']: - if i not in branches: - warnings.append("WARNING: Invalid branch %s requested" % i) - warned = True - - short_desc = bug.short_desc - req_string = change_request_string(items, bug) - bug_comment = 'GIT done (by process-git-requests).\n' - - okprompt = 'Do it (yes=Yes, n=No, e=Edit request, s=Show ticket, c=Comment, q=Quit)?' - if warned: - prompt = 'Warnings present!\nDo it (a=Accept warnings, n=No, e=Edit request, s=Show ticket, c=Comment, q=Quit)?' - else: - prompt = okprompt - - # We have to loop until the user accepts the request - while 1: - # We have to loop until the user enters something that works - while 1: - os.system('clear') - if len(warnings): - print '\n'.join(warnings), "\n" - print req_string + "\nCurrent branch owners - comaintainers:\n" + owner_string - ok = raw_input(prompt) - if ok == 'a': - prompt = okprompt - warned = False - if ok == 'c': - bug_comment = edit_string('') - print bug_comment - ok = raw_input("\nPost this comment to the ticket (y/n)?") - if ok == 'y': - print "Updating bugzilla..." - bug.addcomment(bug_comment) - ok = raw_input("\nClear the fedora-cvs flag (y/n)?") - if ok == 'y': - print "Clearing the flag..." - bug.updateflags({'fedora-cvs':'X', 'nomail':1}) - return (False, True) - elif ok == 'e': - req_string = edit_string(req_string) - items=parse_prefixed_lines(req_string) - request = clean_request(items) - req_string = change_request_string(items, bug) - break - elif ok == 'n': - return (False, True) - elif ok == 'q': - return (False, False) - elif ok == 's': - print - display_bug(bug, allcomments) - elif ok == 'yes' and not warned: - bug_comment = edit_string(bug_comment) - print '\n', bug_comment - ok = raw_input('Go ahead (y/n)?') - if ok != 'y': - break - print 'Calling pkgdb...' - try: - edit_package(pkgdb, request) - except Exception, e: - print "Pkgdb call failed:" - print e - raw_input('\nPress enter to continue to the next ticket.') - return (False, True) - - print 'Updating bugzilla...' - # XXX Need to handle errors here - bug.updateflags({'fedora-cvs':'+', 'nomail':1}) - bug.addcomment(bug_comment) - return (request['pkg'], True) - else: - pass - -if __name__ == '__main__': - branches = {} - processed = [] - options = parse_commandline() - print "Connecting to bugzilla..." - bz = bugzilla.Bugzilla(url=options.url) - print "Querying bugzilla..." - (bugs, comments) = run_query(bz) - print "Done; got %d." % len(bugs) - if not len(bugs): - print "No requests to process!" - exit(0) - - print "Connecting to pkgdb..." - config = parse_pkgdb_config() - pkgdb = PackageDB(config['pkgdb.url'], username=options.user, - debug=options.debug) - print "Getting valid branches...." - for i in pkgdb.get_collection_list(eol=False): - branches[i[0]['branchname']] = 1 - print "Done." - print - - # Iterate over bugs - for i in bugs: - firstfound = True - type = '' - print "Parsing bug %d - https://bugzilla.redhat.com/%d" % (i.bug_id, i.bug_id) - for j in reversed(comments['bugs'][str(i.bug_id)]['comments']): - if 'New Package CVS Request' in j['text']: - type = 'new' - break - if 'Package Change Request' in j['text']: - type = 'change' - break - firstfound = False - else: - if not process_no_request(i, comments['bugs'][str(i.bug_id)]['comments']): - break - - if type == 'new': - (package, more) = process_new_request(i, j, comments['bugs'][str(i.bug_id)]['comments'], firstfound, pkgdb, branches) - if package: - processed.append(package) - if not more: - break - elif type == 'change': - (package, more) = process_change_request(i, j, comments['bugs'][str(i.bug_id)]['comments'], firstfound, pkgdb, branches) - if package: - processed.append(package) - if not more: - break - - if len(processed): - print '\nYou must now run this on the git server\nto set up the git repository:' - print '/usr/local/bin/pkgdb2branch.py ' + ' '.join(processed) - - sys.exit(0) diff --git a/roles/distgit/tasks/main.yml b/roles/distgit/tasks/main.yml index 9c536b107d..bebb1a1177 100644 --- a/roles/distgit/tasks/main.yml +++ b/roles/distgit/tasks/main.yml @@ -45,7 +45,6 @@ - setup_git_package - mkbranch - pkgdb2-clone - - process-git-requests - name: install the Dist Git-related httpd config copy: src=git-smart-http.conf dest=/etc/httpd/conf.d/pkgs.fedoraproject.org/git-smart-http.conf