Infrastructure/fedora-infrastructure
3
0
Fork 0
Code Issues 70 Pull requests Projects Releases Packages Wiki Activity Actions
fedora-infrastructure/scripts/review-stats/review-stats.py
yceh 0a3276af7f List NEEDSPONSOR tickets with POST or ASSIGNED status 
FAS account name extracted from the first few comments of POST or ASSIGNED Review tickets blocked on NEEDSPONSOR are compared with the member list of FAS.
Notes:
1.adminfasusername and adminfaspassword fields needs to be added to review-stats.cfg for querying the member list.
2.Some new tickets are coloured green because they are assigned to user other than nobody@fedoraproject.org
Potential issues:
The script will stall rather than throwing exception if the network was disconnected while querying comments on a bug.(A bug of  python bugzilla modulus?)
2017-10-01 22:23:59 +08:00

743 lines
24 KiB
Python
Executable file
Raw Blame History

#!/usr/bin/python -t
import bugzilla
import datetime
import glob
import logging
import operator
import os
import string
import sys
import tempfile
import time
from configobj import ConfigObj, flatten_errors
from copy import deepcopy
from genshi.template import TemplateLoader
from optparse import OptionParser
from validate import Validator
from fedora.client.fas2 import AccountSystem
VERSION = "4.1"
# Red Hat's bugzilla
url = 'https://bugzilla.redhat.com/xmlrpc.cgi'
# Some magic bug numbers
ACCEPT = 163779
BUNDLED = 658489
FEATURE = 654686
GUIDELINES = 197974
LEGAL = 182235
NEEDSPONSOR = 177841
SCITECH = 505154
SECLAB = 563471
# These will show up in a query but aren't actual review tickets
trackers = set([ACCEPT, BUNDLED, FEATURE, NEEDSPONSOR, GUIDELINES, SCITECH, SECLAB])
# How many packages per submitter are allowed in the main queue
maxpackages = 5
# So the bugzilla module has some way to complain
logging.basicConfig()
fasusernamemap = {}
isnewpackager = {}
nonefasinfo = {}
approvedpackagers = {}
def parse_commandline():
usage = "usage: %prog [options] -c <bugzilla_config> -d <dest_dir> -t <template_dir>"
parser = OptionParser(usage)
parser.add_option("-c", "--config", dest="configfile",
help="configuration file name")
parser.add_option("-d", "--destination", dest="dirname",
help="destination directory")
parser.add_option("-f", "--frequency", dest="frequency",
help="update frequency", default="60")
parser.add_option("-t", "--templatedir", dest="templdir",
help="template directory")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
help="run verbosely")
(options, args) = parser.parse_args()
if str(options.dirname) == 'None':
parser.error("Please specify destination directory")
if not os.path.isdir(options.dirname):
parser.error("Please specify an existing destination directory")
if str(options.templdir) == 'None':
parser.error("Please specify templates directory")
if not os.path.isdir(options.templdir):
parser.error("Please specify an existing template directory")
return options
def parse_config(file):
v = Validator()
spec = '''
[global]
url = string(default='https://bugzilla.redhat.com/xmlrpc.cgi')
username = string()
password = string()
adminfasusername = string()
adminfaspassword = string()
'''.splitlines()
cfg = ConfigObj(file, configspec=spec)
res = cfg.validate(v, preserve_errors=True)
for entry in flatten_errors(cfg, res):
section_list, key, error = entry
section_list.append(key)
if not error:
error = 'Missing value or section.'
print(','.join(section_list), '=', error)
sys.exit(1)
return cfg['global']
def nobody(str):
'''Shorten the long "nobody's working on it" string.'''
if (str == "Nobody's working on this, feel free to take it"
or str == "nobody@fedoraproject.org"):
return "(Nobody)"
return str
def nosec(str):
'''Remove the seconds from an hh:mm:ss format string.'''
return str[0:str.rfind(':')]
def human_date(t):
'''Turn an ISO date into something more human-friendly.'''
t = str(t)
return t[0:4] + '-' + t[4:6] + '-' + t[6:8]
def human_time(t):
'''Turn an ISO date into something more human-friendly, with time.'''
t = str(t)
return t[0:4] + '-' + t[4:6] + '-' + t[6:8] + ' ' + t[9:]
def to_unicode(object, encoding='utf8', errors='replace'):
if isinstance(object, basestring):
if isinstance(object, str):
return unicode(object, encoding, errors)
else:
return object
return u''
def reporter(bug):
'''Extract the reporter from a bug, replacing an empty value with "(none)".
Yes, bugzilla will return a blank reporter for some reason.'''
if (bug.reporter) == '':
return "(none)"
return bug.reporter
def yrmonth(d):
'''Turn a bugzilla date into Month YYYY string.'''
m = ['January', 'February', 'March', 'April', 'May', 'June', 'July',
'August', 'September', 'October', 'November', 'December']
str = d.value
year = str[0:4]
month = int(str[4:6]) - 1
return m[month] + ' ' + year
def dbprint(str):
'''Print string if verbosity is turned on.'''
if verbose:
print(str)
def seq_max_split(seq, max_entries):
""" Given a seq, split into a list of lists of length max_entries each. """
ret = []
num = len(seq)
seq = list(seq) # Trying to use a set/etc. here is bad
beg = 0
while num > max_entries:
end = beg + max_entries
ret.append(seq[beg:end])
beg += max_entries
num -= max_entries
ret.append(seq[beg:])
return ret
def run_query(bz):
querydata = {}
bugdata = {}
alldeps = set([])
closeddeps = set([])
usermap = {}
querydata['include_fields'] = ['id', 'creation_time', 'last_change_time', 'bug_severity',
'alias', 'assigned_to', 'product', 'creator', 'creator_id', 'status', 'resolution',
'component', 'blocks', 'depends_on', 'summary',
'whiteboard', 'flags']
querydata['bug_status'] = ['NEW', 'ASSIGNED', 'MODIFIED','POST']
querydata['product'] = ['Fedora', 'Fedora EPEL']
querydata['component'] = ['Package Review']
querydata['query_format'] = 'advanced'
# Look up tickets with no fedora-review flag set
querydata['f1'] = 'flagtypes.name'
querydata['o1'] = 'notregexp'
querydata['v1'] = 'fedora-review[-+?]'
dbprint("Running main query.")
t = time.time()
bugs = filter(lambda b: b.id not in trackers, bz.query(querydata))
bugs = bugs[200:500]
dbprint("Done, took {0:.2f}.".format(time.time() - t))
for bug in bugs:
bugdata[bug.id] = {}
bugdata[bug.id]['hidden'] = []
bugdata[bug.id]['blocks'] = bug.blocks
bugdata[bug.id]['depends'] = bug.depends_on
bugdata[bug.id]['reviewflag'] = ' '
if bug.depends_on:
alldeps.update(bug.depends_on)
# Now have complete flag info; don't need to query it separately
for flag in bug.flags:
if (flag['name'] == 'needinfo'
and flag['status'] == '?'
and 'requestee' in flag
and flag['requestee'] == bug.creator):
bugdata[bug.id]['hidden'].append('needinfo')
# Find which of the dependencies are closed
dbprint("Looking up {0} bug deps.".format(len(alldeps)))
t = time.time()
for bug in filter(None, bz.query(bz.build_query(bug_id=list(alldeps), status=["CLOSED"]))):
closeddeps.add(bug.id)
dbprint("Done; took {0:.2f}.".format(time.time() - t))
# Hide tickets blocked by other bugs or those with various blockers and
# statuses.
def opendep(id):
return id not in closeddeps
for bug in bugs:
wb = string.lower(bug.whiteboard)
if bug.bug_status != 'CLOSED':
if wb.find('notready') >= 0:
bugdata[bug.id]['hidden'].append('notready')
if wb.find('buildfails') >= 0:
bugdata[bug.id]['hidden'].append('buildfails')
if wb.find('stalledsubmitter') >= 0:
bugdata[bug.id]['hidden'].append('stalled')
if wb.find('awaitingsubmitter') >= 0:
bugdata[bug.id]['hidden'].append('stalled')
if BUNDLED in bugdata[bug.id]['blocks']:
bugdata[bug.id]['hidden'].append('bundled')
if LEGAL in bugdata[bug.id]['blocks']:
bugdata[bug.id]['hidden'].append('legal')
if filter(opendep, bugdata[bug.id]['depends']):
bugdata[bug.id]['hidden'].append('blocked')
# Count up each submitter's tickets and hide excessive submissions. Want
# to make sure one submitter only has 'maxpackages' tickets in the NEW
# queue at one time. Already-hidden packages don't count.
submitters = {}
for i in bugs:
if i.reporter not in submitters:
submitters[i.reporter] = 0
# Don't count tickets which are already hidden
if len(bugdata[i.id]['hidden']):
continue
submitters[i.reporter] += 1
if (submitters[i.reporter] > maxpackages and
'nobody@fedoraproject.org' not in i.reporter):
bugdata[i.id]['hidden'].append('excessive')
# Now we need to look up the names of the users
for i in bugs:
if select_needsponsor(i, bugdata[i.id]):
usermap[i.reporter] = ''
dbprint("Looking up {0} user names.".format(len(usermap)))
t = time.time()
for i in bz._proxy.User.get({'names': usermap.keys()})['users']:
usermap[i['name']] = i['real_name']
dbprint("Done; took {0:.2f}.".format(time.time() - t))
# Now process the other three flags; not much special processing for them
querydata['o1'] = 'equals'
# for i in ['-', '+', '?']:
for i in ['-', '?']:
querydata['v1'] = 'fedora-review' + i
dbprint("Looking up tickets with flag {0}.".format(i))
t = time.time()
b1 = bz.query(querydata)
dbprint("Done; took {0:.2f}.".format(time.time() - t))
for bug in b1:
bugdata[bug.id] = {}
bugdata[bug.id]['hidden'] = []
bugdata[bug.id]['blocks'] = []
bugdata[bug.id]['depends'] = []
bugdata[bug.id]['reviewflag'] = i
bugs += b1
bugs.sort(key=operator.attrgetter('id'))
return [bugs, bugdata, usermap]
# Need to generate reports:
# "Accepted" and closed
# "Accepted" but still open
# "Accepted" means either fedora-review+ or blocking FE-ACCEPT
# fedora-review- and closed
# fedora-review- but still open
# fedora-review? and still optn
# fedora-review? but closed
# Tickets awaiting review but which were hidden for some reason
# That should be all tickets in the Package Review component
def write_html(loader, template, data, dir, fname):
'''Load and render the given template with the given data to the given
filename in the specified directory.'''
tmpl = loader.load(template)
output = tmpl.generate(**data)
path = os.path.join(dir, fname)
try:
f = open(path, "w")
except:
print("Error opening %s" % (path))
sys.exit(1)
for line in output.render().splitlines():
try:
f.write(line.encode('utf8'))
except UnicodeError as e:
print(e.encoding, e.reason, e.object)
f.close()
# Selection functions (should all be predicates)
def select_hidden(bug, bugd):
if len(bugd['hidden']) > 0:
return 1
return 0
def select_merge(bug, bugd):
if (bugd['reviewflag'] == ' '
and bug.bug_status != 'CLOSED'
and bug.short_desc.find('Merge Review') >= 0):
return 1
return 0
def get_fas_username(bug):
''' find the fas name of package submiter based on first few comments'''
comments = bug.getcomments()
# some review ticket have an empty first comment(ex.1282012)
for comment in comments:
username_idx = comment['text'].find('sername')
if username_idx != -1:
fas_username = comment['text'][username_idx:]
break
try:
fas_username = fas_username[fas_username.find(':')+1:]
if fas_username[0] == '\n':
fas_username = fas_username[1:][:fas_username.find('\n')]
def stripe_spaces(fas_username):
if fas_username[0] == ' ':
return fas_username[1:]
stripe_spaces(fas_username)
else:
return fas_username
except UnboundLocalError:
dbprint(comment)
return ''
#most have space before fas username, but some don't.(ex.879752)
def is_new_packager(bug):
#global variable used: isnewpackager,fasusernamemap,nonefasinfo
try:
return isnewpackager[bug.reporter]
except KeyError :
fasusername = get_fas_username(bug)
if fasusername == '' :
try:
nonefasinfo[bug.reporter].append(bug)
except KeyError :
nonefasinfo[bug.reporter] = [bug]
return False
else:
fasusernamemap[bug.reporter] = fasusername
isnewpackager[bug.reporter] = not (fasusername in approvedpackagers)
return not (fasusername in approvedpackagers)
def select_needsponsor(bug, bugd):
wb = string.lower(bug.whiteboard)
if (bugd['reviewflag'] == ' '
and 'needinfo' not in bugd['hidden']
and NEEDSPONSOR in bugd['blocks']
and LEGAL not in bugd['blocks']
and bug.bug_status != 'CLOSED'
and wb.find('buildfails') < 0
and wb.find('notready') < 0
and wb.find('stalledsubmitter') < 0
and wb.find('awaitingsubmitter') < 0
and (nobody(bug.assigned_to) == '(Nobody)'
or is_new_packager(bug))):
return 1
return 0
def select_review(bug, bugd):
if bugd['reviewflag'] == '?':
return 1
return 0
def select_trivial(bug, bugd):
if (bugd['reviewflag'] == ' '
and bug.bug_status != 'CLOSED'
and string.lower(bug.status_whiteboard).find('trivial') >= 0):
return 1
return 0
def select_epel(bug, bugd):
'''If someone assigns themself to a ticket, it's theirs regardless of
whether they set the flag properly or not.'''
if (bugd['reviewflag'] == ' '
and bug.product == 'Fedora EPEL'
and bug.bug_status != 'CLOSED'
and len(bugd['hidden']) == 0
and nobody(bug.assigned_to) == '(Nobody)'
and bug.short_desc.find('Merge Review') < 0):
return 1
return 0
def select_new(bug, bugd):
'''If someone assigns themself to a ticket, it's theirs regardless of
whether they set the flag properly or not.'''
if (bugd['reviewflag'] == ' '
and bug.product == 'Fedora'
and bug.bug_status != 'CLOSED'
and len(bugd['hidden']) == 0
and nobody(bug.assigned_to) == '(Nobody)'
and bug.short_desc.find('Merge Review') < 0):
return 1
return 0
def rowclass_plain(count):
if count % 2 == 1:
return 'bz_row_odd'
return 'bz_row_even'
# Yes, the even/odd classes look backwards, but it looks better this way
def rowclass_with_sponsor(bug, count):
rowclass = 'bz_row_odd'
if NEEDSPONSOR in bug['blocks']:
rowclass = 'bz_state_NEEDSPONSOR'
elif FEATURE in bug['blocks']:
rowclass = 'bz_state_FEATURE'
elif count % 2 == 1:
rowclass = 'bz_row_even'
return rowclass
def rowclass_accepted_packages(bug, count):
rowclass = 'bz_row_odd'
if (bug.bug_status in ['ASSIGNED','POST']
or nobody(bug.assigned_to) != '(Nobody)'):
rowclass = 'bz_state_NEEDSPONSOR'
elif count % 2 == 1:
rowclass = 'bz_row_even'
return rowclass
# The data from a standard row in a bug list
def std_row(bug, rowclass):
alias = ''
if bug.alias:
alias = to_unicode(bug.alias[0])
return {'id': bug.id,
'alias': alias,
'assignee': nobody(to_unicode(bug.assigned_to)),
'class': rowclass,
'lastchange': human_time(bug.last_change_time),
'status': bug.bug_status,
'summary': to_unicode(bug.short_desc),
}
def hidden_reason(reasons):
r = ''
if 'buildfails' in reasons:
r += 'B '
if 'blocked' in reasons:
r += 'D '
if 'excessive' in reasons:
r += 'E '
if 'legal' in reasons:
r += 'L '
if 'needinfo' in reasons:
r += 'Ni '
if 'notready' in reasons:
r += 'Nr '
if 'stalled' in reasons:
r += 'S '
return r
# Report generators
def report_hidden(bugs, bugdata, loader, tmpdir, subs):
data = deepcopy(subs)
data['description'] = 'This page lists all review tickets are hidden from the main review queues'
data['title'] = 'Hidden reviews'
for i in bugs:
if select_hidden(i, bugdata[i.id]):
rowclass = rowclass_with_sponsor(bugdata[i.id], data['count'])
data['bugs'].append(std_row(i, rowclass))
data['bugs'][-1]['reason'] = hidden_reason(bugdata[i.id]['hidden'])
data['count'] += 1
write_html(loader, 'hidden.html', data, tmpdir, 'HIDDEN.html')
return data['count']
def report_review(bugs, bugdata, loader, tmpdir, subs):
data = deepcopy(subs)
data['description'] = 'This page lists tickets currently under review'
data['title'] = 'Tickets under review'
for i in bugs:
if select_review(i, bugdata[i.id]):
rowclass = rowclass_plain(data['count'])
data['bugs'].append(std_row(i, rowclass))
data['count'] += 1
write_html(loader, 'plain.html', data, tmpdir, 'REVIEW.html')
return data['count']
def report_trivial(bugs, bugdata, loader, tmpdir, subs):
data = deepcopy(subs)
data['description'] = 'This page lists review tickets marked as trivial'
data['title'] = 'Trivial reviews'
for i in bugs:
if select_trivial(i, bugdata[i.id]):
rowclass = rowclass_plain(data['count'])
data['bugs'].append(std_row(i, rowclass))
data['count'] += 1
write_html(loader, 'plain.html', data, tmpdir, 'TRIVIAL.html')
return data['count']
def report_merge(bugs, bugdata, loader, tmpdir, subs):
data = deepcopy(subs)
data['description'] = 'This page lists all merge review tickets which need reviewers'
data['title'] = 'Merge reviews'
for i in bugs:
if select_merge(i, bugdata[i.id]):
rowclass = rowclass_plain(data['count'])
data['bugs'].append(std_row(i, rowclass))
data['count'] += 1
write_html(loader, 'plain.html', data, tmpdir, 'MERGE.html')
return data['count']
def report_needsponsor(bugs, bugdata, loader, usermap, tmpdir, subs):
data = deepcopy(subs)
data['description'] = ('This page lists new NEEDSPONSOR tickets submitted by new packagers who are not yet sponsored.'
'Tickets colored green are already being assigned for review.')
data['title'] = 'NEEDSPONSOR tickets'
curreporter = ''
curcount = 0
oldest = {}
selected = []
for i in bugs:
if select_needsponsor(i, bugdata[i.id]):
selected.append(i)
#cleanup bugs that have no fas username
for i in nonefasinfo:
try:
if isnewpackager[i]:
selected += nonefasinfo[i]
except KeyError :
pass
# Determine the oldest reported bug
for i in selected:
if i.reporter not in oldest:
oldest[i.reporter] = i.creation_time
elif i.creation_time < oldest[i.reporter]:
oldest[i.reporter] = i.creation_time
selected.sort(key=reporter)
selected.sort(key=lambda a: oldest[a.reporter])
for i in selected:
r = i.reporter
if curreporter != r:
if (r in usermap and len(usermap[r])):
name = usermap[r]
else:
name = r
data['packagers'].append({'email': r, 'name': name, 'oldest': human_date(oldest[r]), 'bugs': []})
curreporter = r
curcount = 0
rowclass = rowclass_accepted_packages(i, curcount)
data['packagers'][-1]['bugs'].append(std_row(i, rowclass))
data['count'] += 1
curcount += 1
write_html(loader, 'needsponsor.html', data, tmpdir, 'NEEDSPONSOR.html')
return data['count']
def report_epel(bugs, bugdata, loader, tmpdir, subs):
data = deepcopy(subs)
data['description'] = ('This page lists new, reviewable EPEL package review tickets.'
' Tickets colored green require a sponsor.')
data['title'] = 'New EPEL package review tickets'
curmonth = ''
curcount = 0
for i in bugs:
if select_epel(i, bugdata[i.id]):
if curmonth != yrmonth(i.creation_time):
if curcount > 0:
data['months'][-1]['month'] += (" (%d)" % curcount)
data['months'].append({'month': yrmonth(i.creation_time), 'bugs': []})
curmonth = yrmonth(i.creation_time)
curcount = 0
rowclass = rowclass_with_sponsor(bugdata[i.id], curcount)
data['months'][-1]['bugs'].append(std_row(i, rowclass))
data['count'] += 1
curcount += 1
if curcount > 0:
data['months'][-1]['month'] += (" (%d)" % curcount)
write_html(loader, 'bymonth.html', data, tmpdir, 'EPEL.html')
return data['count']
def report_new(bugs, bugdata, loader, tmpdir, subs):
data = deepcopy(subs)
data['description'] = ('This page lists new, reviewable Fedora package review tickets (excluding merge reviews).'
' Tickets colored green require a sponsor.')
data['title'] = 'New package review tickets'
curmonth = ''
curcount = 0
for i in bugs:
if select_new(i, bugdata[i.id]):
if curmonth != yrmonth(i.creation_time):
if curcount > 0:
data['months'][-1]['month'] += (" (%d)" % curcount)
data['months'].append({'month': yrmonth(i.creation_time), 'bugs': []})
curmonth = yrmonth(i.creation_time)
curcount = 0
rowclass = rowclass_with_sponsor(bugdata[i.id], curcount)
data['months'][-1]['bugs'].append(std_row(i, rowclass))
data['count'] += 1
curcount += 1
if curcount > 0:
data['months'][-1]['month'] += (" (%d)" % curcount)
write_html(loader, 'bymonth.html', data, tmpdir, 'NEW.html')
return data['count']
if __name__ == '__main__':
options = parse_commandline()
verbose = options.verbose
config = parse_config(options.configfile)
if config['maxpackages']:
maxpackages = int(config['maxpackages'])
dbprint("Limiting to {0} packages".format(maxpackages))
bz = bugzilla.RHBugzilla(url=config['url'], cookiefile=None, user=config['username'], password=config['password'])
t = time.time()
(bugs, bugdata, usermap) = run_query(bz)
approvedpackagers=[]
for indi in AccountSystem(username=config['adminfasusername'],password=config['adminfaspassword']).group_members('packager'):
approvedpackagers.append(indi.toDict()['username'].encode('utf-8'))
querytime = time.time() - t
# Don't bother running this stuff until the query completes, since it fails
# so often.
loader = TemplateLoader(options.templdir)
tmpdir = tempfile.mkdtemp(dir=options.dirname)
# The initial set of substitutions that's shared between the report functions
subs = {
'update': datetime.datetime.now().strftime('%Y-%m-%d %H:%M'),
'querytime': querytime,
'version': VERSION,
'count': 0,
'months': [],
'packagers': [],
'bugs': []}
args = {'bugs': bugs, 'bugdata': bugdata, 'loader': loader, 'tmpdir': tmpdir, 'subs': subs}
t = time.time()
subs['new'] = report_new(**args)
subs['epel'] = report_epel(**args)
subs['hidden'] = report_hidden(**args)
subs['needsponsor'] = report_needsponsor(usermap=usermap, **args)
subs['review'] = report_review(**args)
subs['trivial'] = report_trivial(**args)
# data['accepted_closed'] = report_accepted_closed(bugs, bugdata, loader, tmpdir)
# data['accepted_open'] = report_accepted_open(bugs, bugdata, loader, tmpdir)
# data['rejected_closed'] = report_rejected_closed(bugs, bugdata, loader, tmpdir)
# data['rejected_open'] = report_rejected_open(bugs, bugdata, loader, tmpdir)
# data['review_closed'] = report_review_closed(bugs, bugdata, loader, tmpdir)
# data['review_open'] = report_review_open(bugs, bugdata, loader, tmpdir)
subs['outputtime'] = time.time() - t
write_html(loader, 'index.html', subs, tmpdir, 'index.html')
for filename in glob.glob(os.path.join(tmpdir, '*')):
newFilename = os.path.basename(filename)
os.rename(filename, os.path.join(options.dirname, newFilename))
os.rmdir(tmpdir)
sys.exit(0)
Reference in a new issue View git blame Copy permalink