From b452a00bd311034dbbe92403901a2a1b87bafee5 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Thu, 24 Apr 2008 15:58:46 -0700 Subject: [PATCH] Rework this largely from what's in pkgdb-sync-bugzilla. --- scripts/bugzilla/bz-make-components.py | 379 +++++++++++++++++-------- 1 file changed, 263 insertions(+), 116 deletions(-) diff --git a/scripts/bugzilla/bz-make-components.py b/scripts/bugzilla/bz-make-components.py index f218e99..e2b4150 100755 --- a/scripts/bugzilla/bz-make-components.py +++ b/scripts/bugzilla/bz-make-components.py @@ -1,135 +1,282 @@ -#!/usr/bin/python +#!/usr/bin/python -tt +# -*- coding: utf-8 -*- +# +# Copyright © 2008 Red Hat, Inc. All rights reserved. +# +# This copyrighted material is made available to anyone wishing to use, modify, +# copy, or redistribute it subject to the terms and conditions of the GNU +# General Public License v.2. This program is distributed in the hope that it +# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the +# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. You should have +# received a copy of the GNU General Public License along with this program; +# if not, write to the Free Software Foundation, Inc., 51 Franklin Street, +# Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat trademarks that are +# incorporated in the source code or documentation are not subject to the GNU +# General Public License and may only be used or replicated with the express +# permission of Red Hat, Inc. +# +# Red Hat Author(s): Elliot Lee +# Toshio Kuratomi +# -#!/usr/bin/python2 +import sys +import os +import errno +import website +import crypt +import getopt +import xmlrpclib +from email.Message import Message +import smtplib -import sys, os, errno -import website, crypt -import getopt, re -GRANT_DIRECT = 0 -GRANT_DERIVED = 1 -GRANT_REGEXP = 2 +# Set this to the production bugzilla account when we're ready to go live +BZSERVER = 'https://bugdev.devel.redhat.com/bugzilla-cvs/xmlrpc.cgi' +#BZSERVER = 'https://bugzilla.redhat.com/xmlrpc.cgi' +#BZSERVER = 'https://bzprx.vip.phx.redhat.com/xmlrpc.cgi' +BZUSER='<%= bzAdminUser %>' +BZPASS='<%= bzAdminPassword %>' DRY_RUN = False -def get_bz_user_id(bzdbh, username): - bzdbc = bzdbh.cursor() - bzdbc.execute("SELECT userid FROM profiles WHERE login_name = %s", - (username.lower(),)) - if bzdbc.rowcount: - return bzdbc.fetchone()[0] +class Bugzilla(object): + def __init__(self, bzServer, username, password): + self.productCache = {} + self.bzXmlRpcServer = bzServer + self.username = username + self.password = password -opts, args = getopt.getopt(sys.argv[1:], '', ('usage', 'help')) -if len(args) < 1 or ('--usage','') in opts or ('--help','') in opts: - print """ -Usage: bz-make-components.py FILENAME... -""" - sys.exit(1) + self.server = xmlrpclib.Server(bzServer) -bzdbh = website.get_dbh('bugs', 'bugs') -bzdbc = bzdbh.cursor() + def add_edit_component(self, package, collection,owner, description, + qacontact=None, cclist=None): + '''Add or updatea component to have the values specified. + ''' + initialCCList = cclist or list() -bzdbh.commit() -need_emails = {} -for curfile in args: - if not os.path.exists(curfile): - continue - fh = open(curfile, 'r') - lnum = 0 - while 1: - aline = fh.readline() - lnum += 1 - if not aline: - break - aline = aline.strip() - if not aline or aline[0] == '#': - continue - - pieces = aline.split('|') + # Lookup product try: - product, component, description, owner, qa = pieces[:5] + product = self.productCache[collection] + except KeyError: + product = {} + try: + components = self.server.bugzilla.getProdCompDetails(collection, + self.username, self.password) + except xmlrpclib.Fault, e: + # Output something useful in args + e.args = (e.faultCode, e.faultString) + raise + except xmlrpclib.ProtocolError, e: + e.args = ('ProtocolError', e.errcode, e.errmsg) + raise + + # This changes from the form: + # {'component': 'PackageName', + # 'initialowner': 'OwnerEmail', + # 'initialqacontact': 'QAContactEmail', + # 'description': 'One sentence summary'} + # to: + # product['packagename'] = {'component': 'PackageName', + # 'initialowner': 'OwnerEmail', + # 'initialqacontact': 'QAContactEmail', + # 'description': 'One sentenct summary'} + # This allows us to check in a case insensitive manner for the + # package. + for record in components: + record['component'] = unicode(record['component'], 'utf-8') + try: + record['description'] = unicode(record['description'], 'utf-8') + except TypeError: + try: + record['description'] = unicode(record['description'].data, 'utf-8') + except: + record['description'] = None + product[record['component'].lower()] = record + + self.productCache[collection] = product + + pkgKey = package.lower() + if pkgKey in product: + # edit the package information + data = {} + + # Grab bugzilla email for things changable via xmlrpc + owner = owner.lower() + if qacontact: + qacontact = qacontact.lower() + else: + qacontact = 'extras-qa@fedoraproject.org' + + # Check for changes to the owner, qacontact, or description + if product[pkgKey]['initialowner'] != owner: + data['initialowner'] = owner + + if product[pkgKey]['description'] != description: + data['description'] = description + if product[pkgKey]['initialqacontact'] != qacontact and ( + qacontact or product[pkgKey]['initialqacontact']): + data['initialqacontact'] = qacontact + + if len(product[pkgKey]['initialcclist']) != len(initialCCList): + data['initialcclist'] = initialCCList + else: + for ccMember in product[pkgKey]['initialcclist']: + if ccMember not in initialCCList: + data['initialcclist'] = initialCCList + break + + if data: + ### FIXME: initialowner has been made mandatory for some + # reason. Asking dkl why. + data['initialowner'] = owner + + # Changes occurred. Submit a request to change via xmlrpc + data['product'] = collection + data['component'] = product[pkgKey]['component'] + if DRY_RUN: + print '[EDITCOMP] Changing via editComponent(%s, %s, "xxxxx")' % ( + data, self.username) + print '[EDITCOMP] Former values: %s|%s|%s' % ( + product[pkgKey]['initialowner'], + product[pkgKey]['description'], + product[pkgKey]['initialqacontact']) + else: + try: + self.server.bugzilla.editComponent(data, self.username, + self.password) + except xmlrpclib.Fault, e: + # Output something useful in args + e.args = (data, e.faultCode, e.faultString) + raise + except xmlrpclib.ProtocolError, e: + e.args = ('ProtocolError', e.errcode, e.errmsg) + raise + else: + # Add component + owner = owner.lower() + if qacontact: + qacontact = qacontact + else: + qacontact = 'extras-qa@fedoraproject.org' + + data = {'product': collection, + 'component': package, + 'description': description, + 'initialowner': owner, + 'initialqacontact': qacontact} + if initialCCList: + data['initialcclist'] = initialCCList + + if DRY_RUN: + print '[ADDCOMP] Adding new component AddComponent:(%s, %s, "xxxxx")' % ( + data, self.username) + else: + try: + self.server.bugzilla.addComponent(data, self.username, + self.password) + except xmlrpclib.Fault, e: + # Output something useful in args + e.args = (data, e.faultCode, e.faultString) + raise + +def parseOwnerFile(curfile, warnings): + pkgInfo = [] + ownerFile = file(curfile, 'r') + while line in ownerFile: + line = line.strip() + if not line or line[0] == '#': + continue + pieces = line.split('|') + try: + product, component, summary, owner, qa = pieces[:5] except: - print "Invalid line %s at %s:%s" % (aline, curfile, lnum) - cclist = [] + warnings.append('%s: Invalid line %s' % (curfile, line)) + owners = owner.split(',') owner = owners[0] - if len(owners) > 1: - for I in owners[1:]: - Inum = get_bz_user_id(bzdbh, I) - if Inum is None: - if not need_emails.has_key(I): - need_emails[I] = [] - need_emails[I].append((product, component, curfile, lnum)) - continue - cclist.append(Inum) - owner_num = get_bz_user_id(bzdbh, owner) - qa_num = get_bz_user_id(bzdbh, qa) - if owner_num is None: - if not need_emails.has_key(owner): - need_emails[owner] = [] - need_emails[owner].append((product, component, curfile, lnum)) -# print "Invalid owner %s at %s:%s" % (owner, curfile, lnum) - continue - if len(pieces) > 5 and pieces[5]: - for I in pieces[5].split(','): - Inum = get_bz_user_id(bzdbh, I) - if Inum is None: - if not need_emails.has_key(I): - need_emails[I] = [] - need_emails[I].append((product, component, curfile, lnum)) - continue - cclist.append(Inum) - - if product != "Fedora" and product[:len('Fedora ')] != 'Fedora ': - print "Invalid product %s at %s:%s" % (product, curfile, lnum) + cclist = owners[1:] or [] + if not owner: + warnings.append('%s: No owner in line %s' % (curfile, line)) continue - bzdbc.execute("SELECT id FROM products WHERE name = %s", (product,)) - if not bzdbc.rowcount: - if DRY_RUN: - print "Need to create product %s" %(product,) - sys.exit(0) - else: - bzdbc.execute("INSERT INTO products (name, description, disallownew, votesperuser, maxvotesperbug, votestoconfirm, defaultmilestone, depends) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)", (product, product, 0, 0, 10000, 0, '---', 0)) - bzdbc.execute("SELECT id FROM products WHERE name = %s", (product,)) - product_id = bzdbc.fetchone()[0] - bzdbc.execute("INSERT INTO versions (value, product_id) VALUES (%s, %s)", ("development", product_id)) - bzdbc.execute("INSERT INTO milestones (product_id, value, sortkey) VALUES (%s, '---', 0)", (product_id,)) - else: - product_id = bzdbc.fetchone()[0] + if len(pieces) > 5 and pieces[5].strip(): + for person in pieces[5].strip().split(','): + cclist.append(person.strip()) - bzdbc.execute("SELECT * FROM components WHERE product_id = %s AND name = %s", (product_id, component)) - if bzdbc.rowcount: - arow = bzdbc.fetchhash() - if DRY_RUN: - print("component update of %s: UPDATE components SET initialowner = %s, initialqacontact = %s, initialcclist = %s WHERE id = %s", - (component, owner_num, qa_num, ':'.join(map(str,cclist)), arow['id'])) - else: - bzdbc.execute("UPDATE components SET initialowner = %s, initialqacontact = %s, initialcclist = %s WHERE id = %s", - (owner_num, qa_num, ':'.join(map(str,cclist)), arow['id'])) - else: - if DRY_RUN: - print("create component: INSERT INTO components (name, product_id, description, initialowner, initialqacontact, initialcclist) VALUES (%s, %s, %s, %s, %s, %s)", - (component, product_id, description, owner_num, qa_num, ':'.join(map(str,cclist)))) - else: - bzdbc.execute("INSERT INTO components (name, product_id, description, initialowner, initialqacontact, initialcclist) VALUES (%s, %s, %s, %s, %s, %s)", - (component, product_id, description, owner_num, qa_num, ':'.join(map(str,cclist)))) -bzdbh.commit() + if product != 'Fedora' and not product.startswith('Fedora '): + warnings.append('%s: Invalid product %s in line %s' % + (curfile, product, line)) + continue + pkgInfo.append({'product': product, 'component': component, + 'owner': owner, 'summary': summary, 'qa': qa, 'cclist': cclist}) + + return pkgInfo + +def send_email(fromAddress, toAddress, subject, message): + '''Send an email if there's an error. + + This will be replaced by sending messages to a log later. + ''' + msg = Message() + msg.add_header('To', toAddress) + msg.add_header('From', fromAddress) + msg.add_header('Subject', subject) + msg.set_payload(message) + smtp = smtplib.SMTP('bastion') + smtp.sendmail(fromAddress, [toAddress], msg.as_string()) + smtp.quit() + +if __name__ == '__main__': + opts, args = getopt.getopt(sys.argv[1:], '', ('usage', 'help')) + if len(args) < 1 or ('--usage','') in opts or ('--help','') in opts: + print """Usage: bz-make-components.py FILENAME...""" + sys.exit(1) + + # Initialize connection to bugzilla + bugzilla = Bugzilla(BZSERVER, BZUSER, BZPASS) + + warnings = [] + # Iterate through the files in the argument list. Grab the owner + # information from each one and construct bugzilla information from it. + pkgData = [] + for curfile in args: + if not os.path.exists(curfile): + warnings.append('%s does not exist' % curfile) + continue + pkgData.extend(parseOwnerFile(curfile, warnings)) + + for pkgInfo in pkgData: + try: + bugzilla.add_edit_component(pkgInfo['product'], + pkgInfo['component'], pkgInfo['owner'], pkgInfo['summary'], + pkgInfo['qa'], pkgInfo['cclist']) + except ValueError, e: + # A username didn't have a bugzilla address + warnings.append(str(e.args)) + except DataChangedError, e: + # A Package or Collection was returned via xmlrpc but wasn't + # present when we tried to change it + warnings.append(str(e.args)) + except xmlrpclib.ProtocolError, e: + # Unrecoverable and likely means that nothing is going to + # succeed. + warnings.append(str(e.args)) + break + except xmlrpclib.Error, e: + # An error occurred in the xmlrpc call. Shouldn't happen but + # we better see what it is + warnings.append(str(e.args)) + + if warnings: + # print '[DEBUG]', '\n'.join(warnings) + send_email('accounts@fedoraproject.org', 'a.badger@gmail.com', + 'Errors while syncing bugzilla with owners.list', +''' +The following errors were encountered while updating bugzilla with information +from owners.list files. Please have the problem taken care of: -for I, J in need_emails.items(): - if not I.strip(): - print "Need an e-mail for", J - continue - print "Sending e-mail to", I - if DRY_RUN: - continue - website.send_email("accounts@fedora.redhat.com", I, "You need to create a bugzilla account for %s" % I, """ -In order to make bugzilla components for Fedora-related programs, we need to have an existing bugzilla account for -the listed owner. You (%s) do not have a bugzilla account, but are listed as the owner for the following components: %s +''' % ('\n\n'.join(warnings),)) -Please create a bugzilla account for %s immediately, because this amazingly stupid cron job will keep sending you an -e-mail every hour until you do :) - -- The management -""" % (I, '\n'.join(map(lambda x: "%s (%s)" % x[:2], J)), I)) + sys.exit(0)