diff --git a/roles/pkgdb2/files/pkgdb-sync-bugzilla b/roles/pkgdb2/files/pkgdb-sync-bugzilla new file mode 100755 index 0000000000..408019a7d5 --- /dev/null +++ b/roles/pkgdb2/files/pkgdb-sync-bugzilla @@ -0,0 +1,373 @@ +#!/usr/bin/python -tt +# -*- coding: utf-8 -*- +# +# Copyright © 2013-2014 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, modify, +# copy, or redistribute it subject to the terms and conditions of the GNU +# General Public License v.2, or (at your option) any later version. This +# program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY expressed or implied, including the implied warranties of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the GNU +# General Public License along with this program; if not, write to the Free +# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the source +# code or documentation are not subject to the GNU General Public License and +# may only be used or replicated with the express permission of Red Hat, Inc. +# +# Red Hat Author(s): Toshio Kuratomi +# Author(s): Mike Watters +# Author(s): Pierre-Yves Chibon +# +''' +sync information from the packagedb into bugzilla + +This short script takes information about package onwership and imports it +into bugzilla. +''' + +## These two lines are needed to run on EL6 +__requires__ = ['SQLAlchemy >= 0.7', 'jinja2 >= 2.4'] +import pkg_resources + +import argparse +import sys +import os +import itertools +import xmlrpclib +import codecs +import smtplib +import bugzilla +import requests +from email.Message import Message +from fedora.client.fas2 import AccountSystem + + +if 'PKGDB2_CONFIG' not in os.environ \ + and os.path.exists('/etc/pkgdb2/pkgdb2.cfg'): + print 'Using configuration file `/etc/pkgdb2/pkgdb2.cfg`' + os.environ['PKGDB2_CONFIG'] = '/etc/pkgdb2/pkgdb2.cfg' + + +try: + import pkgdb2 +except ImportError: + sys.path.insert( + 0, os.path.join(os.path.dirname(os.path.realpath(__file__)), '..')) + import pkgdb2 + + +BZSERVER = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_URL') +BZUSER = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_NOTIFY_USER') +BZPASS = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_NOTIFY_PASSWORD') +BZCOMPAPI = pkgdb2.APP.config.get('BUGZILLA_COMPONENT_API') +FASURL = pkgdb2.APP.config.get('PKGDB2_FAS_URL') +FASUSER = pkgdb2.APP.config.get('PKGDB2_FAS_USER') +FASPASS = pkgdb2.APP.config.get('PKGDB2_FAS_PASSWORD') +FASINSECURE = pkgdb2.APP.config.get('PKGDB2_FAS_INSECURE') +NOTIFYEMAIL = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_NOTIFY_EMAIL') +PKGDBSERVER = pkgdb2.APP.config.get('SITE_URL') +DRY_RUN = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_DRY_RUN', False) + +# When querying for current info, take segments of 1000 packages a time +BZ_PKG_SEGMENT = 1000 + +class DataChangedError(Exception): + '''Raised when data we are manipulating changes while we're modifying it.''' + pass + +def segment(iterable, chunk, fill=None): + '''Collect data into `chunk` sized block''' + args = [iter(iterable)] * chunk + return itertools.izip_longest(*args, fillvalue=fill) + +class ProductCache(dict): + def __init__(self, bz, acls): + self.bz = bz + self.acls = acls + + # Ask bugzilla for a section of the pkglist. + # Save the information from the section that we want. + def __getitem__(self, key): + try: + return super(ProductCache, self).__getitem__(key) + except KeyError: + # We can only cache products we have pkgdb information for + if key not in self.acls: + raise + + if BZCOMPAPI == 'getcomponentsdetails': + # Old API -- in python-bugzilla. But with current server, this + # gives ProxyError + products = self.server.getcomponentsdetails(key) + elif BZCOMPAPI == 'component.get': + # Way that's undocumented in the partner-bugzilla api but works + # currently + pkglist = acls[key].keys() + products = {} + for pkg_segment in segment(pkglist, BZ_PKG_SEGMENT): + # Format that bugzilla will understand. Strip None's that segment() pads + # out the final data segment() with + query = [dict(product=key, component=p) for p in pkg_segment if p is not None] + raw_data = self.bz._proxy.Component.get(dict(names=query)) + for package in raw_data['components']: + # Reformat data to be the same as what's returned from + # getcomponentsdetails + product = dict(initialowner=package['default_assignee'], + description=package['description'], + initialqacontact=package['default_qa_contact'], + initialcclist=package['default_cc']) + products[package['name'].lower()] = product + self[key] = products + + return super(ProductCache, self).__getitem__(key) + + +class Bugzilla(object): + + def __init__(self, bzServer, username, password, acls): + self.bzXmlRpcServer = bzServer + self.username = username + self.password = password + + self.server = bugzilla.Bugzilla( + url=self.bzXmlRpcServer, + user=self.username, + password=self.password) + self.productCache = ProductCache(self.server, acls) + + # Connect to the fedora account system + self.fas = AccountSystem( + base_url=FASURL, + username=FASUSER, + password=FASPASS) + self.userCache = self.fas.people_by_key( + key='username', + fields=['bugzilla_email']) + + def _get_bugzilla_email(self, username): + '''Return the bugzilla email address for a user. + + First looks in a cache for a username => bugzilla email. If not found, + reloads the cache from fas and tries again. + ''' + try: + return self.userCache[username]['bugzilla_email'].lower() + except KeyError: + if username.startswith('@'): + group = self.fas.group_by_name(username[1:]) + self.userCache[username] = { + 'bugzilla_email': group.mailing_list} + else: + person = self.fas.person_by_username(username) + bz_email = person.get('bugzilla_email', None) + if bz_email is None: + print '%s has no bugzilla email, valid account?' % username + else: + self.userCache[username] = {'bugzilla_email': bz_email} + return self.userCache[username]['bugzilla_email'].lower() + + def add_edit_component(self, package, collection, owner, description, + qacontact=None, cclist=None): + '''Add or update a component to have the values specified. + ''' + # Turn the cclist into something usable by bugzilla + if not cclist or 'people' not in cclist: + initialCCList = list() + else: + initialCCList = [ + self._get_bugzilla_email(cc) for cc in cclist['people']] + if 'groups' in cclist: + group_cc = [ + self._get_bugzilla_email(cc) for cc in cclist['groups']] + initialCCList.extend(group_cc) + + # Add owner to the cclist so comaintainers taking over a bug don't + # have to do this manually + owner = self._get_bugzilla_email(owner) + if owner not in initialCCList: + initialCCList.append(owner) + + # Lookup product + try: + product = self.productCache[collection] + except xmlrpclib.Fault as e: + # Output something useful in args + e.args = (e.faultCode, e.faultString) + raise + except xmlrpclib.ProtocolError as e: + e.args = ('ProtocolError', e.errcode, e.errmsg) + raise + + pkgKey = package.lower() + if pkgKey in product: + # edit the package information + data = {} + + # Grab bugzilla email for things changable via xmlrpc + if qacontact: + qacontact = self._get_bugzilla_email(qacontact) + else: + qacontact = 'extras-qa@fedoraproject.org' + + # Check for changes to the owner, qacontact, or description + if product[pkgKey]['initialowner'] != owner: + data['initialowner'] = owner + + if product[pkgKey]['description'] != description: + data['description'] = description + if product[pkgKey]['initialqacontact'] != qacontact and ( + qacontact or product[pkgKey]['initialqacontact']): + data['initialqacontact'] = qacontact + + if len(product[pkgKey]['initialcclist']) != len(initialCCList): + data['initialcclist'] = initialCCList + else: + for ccMember in product[pkgKey]['initialcclist']: + if ccMember not in initialCCList: + data['initialcclist'] = initialCCList + break + + if data: + ### FIXME: initialowner has been made mandatory for some + # reason. Asking dkl why. + data['initialowner'] = owner + + # Changes occurred. Submit a request to change via xmlrpc + data['product'] = collection + data['component'] = package + if DRY_RUN: + print '[EDITCOMP] Changing via editComponent(' \ + '%s, %s, "xxxxx")' % (data, self.username) + print '[EDITCOMP] Former values: %s|%s|%s|%s' % ( + product[pkgKey]['initialowner'], + product[pkgKey]['description'], + product[pkgKey]['initialqacontact'], + product[pkgKey]['initialcclist']) + else: + try: + self.server.editcomponent(data) + except xmlrpclib.Fault, e: + # Output something useful in args + e.args = (data, e.faultCode, e.faultString) + raise + except xmlrpclib.ProtocolError, e: + e.args = ('ProtocolError', e.errcode, e.errmsg) + raise + else: + # Add component + if qacontact: + qacontact = self._get_bugzilla_email(qacontact) + else: + qacontact = 'extras-qa@fedoraproject.org' + + data = { + 'product': collection, + 'component': package, + 'description': description, + 'initialowner': owner, + 'initialqacontact': qacontact + } + if initialCCList: + data['initialcclist'] = initialCCList + + if DRY_RUN: + print '[ADDCOMP] Adding new component AddComponent:(' \ + '%s, %s, "xxxxx")' % (data, self.username) + else: + try: + self.server.addcomponent(data) + except xmlrpclib.Fault, e: + # Output something useful in args + e.args = (data, e.faultCode, e.faultString) + raise + + +def send_email(fromAddress, toAddress, subject, message): + '''Send an email if there's an error. + + This will be replaced by sending messages to a log later. + ''' + msg = Message() + msg.add_header('To', ','.join(toAddress)) + msg.add_header('From', fromAddress) + msg.add_header('Subject', subject) + msg.set_payload(message) + smtp = smtplib.SMTP('bastion') + smtp.sendmail(fromAddress, toAddress, msg.as_string()) + smtp.quit() + + +if __name__ == '__main__': + sys.stdout = codecs.getwriter('utf-8')(sys.stdout) + + + parser = argparse.ArgumentParser( + description='Script syncing information between pkgdb and bugzilla' + ) + parser.add_argument( + '--debug', dest='debug', action='store_true', default=False, + help='Print the changes instead of making them in bugzilla') + + args = parser.parse_args() + + if args.debug: + DRY_RUN = True + + # Non-fatal errors to alert people about + errors = [] + + # Get bugzilla information from the package database + req = requests.get('%s/api/bugzilla/?format=json' % PKGDBSERVER) + acls = req.json()['bugzillaAcls'] + + # Initialize the connection to bugzilla + bugzilla = Bugzilla(BZSERVER, BZUSER, BZPASS, acls) + + for product in acls.keys(): + if product not in ('Fedora', 'Fedora EPEL'): + continue + for pkg in acls[product]: + if DRY_RUN: + print pkg + pkgInfo = acls[product][pkg] + try: + bugzilla.add_edit_component( + pkg, + product, + pkgInfo['owner'], + pkgInfo['summary'], + pkgInfo['qacontact'], + pkgInfo['cclist']) + except ValueError, e: + # A username didn't have a bugzilla address + errors.append(str(e.args)) + except DataChangedError, e: + # A Package or Collection was returned via xmlrpc but wasn't + # present when we tried to change it + errors.append(str(e.args)) + except xmlrpclib.ProtocolError, e: + # Unrecoverable and likely means that nothing is going to + # succeed. + errors.append(str(e.args)) + break + except xmlrpclib.Error, e: + # An error occurred in the xmlrpc call. Shouldn't happen but + # we better see what it is + errors.append(str(e.args)) + + # Send notification of errors + if errors: + #print '[DEBUG]', '\n'.join(errors) + send_email('accounts@fedoraproject.org', + NOTIFYEMAIL, + 'Errors while syncing bugzilla with the PackageDB', +''' +The following errors were encountered while updating bugzilla with information +from the Package Database. Please have the problems taken care of: + +%s +''' % ('\n'.join(errors),)) + + sys.exit(0)