Add current version of the pkgdb-sync-bugzilla script
This commit is contained in:
Pierre-Yves Chibon
parent
db74cc47c6
commit
400b94bd5d
1 changed files with 373 additions and 0 deletions
373
roles/pkgdb2/files/pkgdb-sync-bugzilla
Executable file
373
roles/pkgdb2/files/pkgdb-sync-bugzilla
Executable file
|
|
@ -0,0 +1,373 @@
|
||||||
|
#!/usr/bin/python -tt
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
#
|
||||||
|
# Copyright © 2013-2014 Red Hat, Inc.
|
||||||
|
#
|
||||||
|
# This copyrighted material is made available to anyone wishing to use, modify,
|
||||||
|
# copy, or redistribute it subject to the terms and conditions of the GNU
|
||||||
|
# General Public License v.2, or (at your option) any later version. This
|
||||||
|
# program is distributed in the hope that it will be useful, but WITHOUT ANY
|
||||||
|
# WARRANTY expressed or implied, including the implied warranties of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
|
||||||
|
# Public License for more details. You should have received a copy of the GNU
|
||||||
|
# General Public License along with this program; if not, write to the Free
|
||||||
|
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||||
|
# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the source
|
||||||
|
# code or documentation are not subject to the GNU General Public License and
|
||||||
|
# may only be used or replicated with the express permission of Red Hat, Inc.
|
||||||
|
#
|
||||||
|
# Red Hat Author(s): Toshio Kuratomi <tkuratom@redhat.com>
|
||||||
|
# Author(s): Mike Watters <valholla75@fedoraproject.org>
|
||||||
|
# Author(s): Pierre-Yves Chibon <pingou@pingoured.fr>
|
||||||
|
#
|
||||||
|
'''
|
||||||
|
sync information from the packagedb into bugzilla
|
||||||
|
|
||||||
|
This short script takes information about package onwership and imports it
|
||||||
|
into bugzilla.
|
||||||
|
'''
|
||||||
|
|
||||||
|
## These two lines are needed to run on EL6
|
||||||
|
__requires__ = ['SQLAlchemy >= 0.7', 'jinja2 >= 2.4']
|
||||||
|
import pkg_resources
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import itertools
|
||||||
|
import xmlrpclib
|
||||||
|
import codecs
|
||||||
|
import smtplib
|
||||||
|
import bugzilla
|
||||||
|
import requests
|
||||||
|
from email.Message import Message
|
||||||
|
from fedora.client.fas2 import AccountSystem
|
||||||
|
|
||||||
|
|
||||||
|
if 'PKGDB2_CONFIG' not in os.environ \
|
||||||
|
and os.path.exists('/etc/pkgdb2/pkgdb2.cfg'):
|
||||||
|
print 'Using configuration file `/etc/pkgdb2/pkgdb2.cfg`'
|
||||||
|
os.environ['PKGDB2_CONFIG'] = '/etc/pkgdb2/pkgdb2.cfg'
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
import pkgdb2
|
||||||
|
except ImportError:
|
||||||
|
sys.path.insert(
|
||||||
|
0, os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
|
||||||
|
import pkgdb2
|
||||||
|
|
||||||
|
|
||||||
|
BZSERVER = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_URL')
|
||||||
|
BZUSER = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_NOTIFY_USER')
|
||||||
|
BZPASS = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_NOTIFY_PASSWORD')
|
||||||
|
BZCOMPAPI = pkgdb2.APP.config.get('BUGZILLA_COMPONENT_API')
|
||||||
|
FASURL = pkgdb2.APP.config.get('PKGDB2_FAS_URL')
|
||||||
|
FASUSER = pkgdb2.APP.config.get('PKGDB2_FAS_USER')
|
||||||
|
FASPASS = pkgdb2.APP.config.get('PKGDB2_FAS_PASSWORD')
|
||||||
|
FASINSECURE = pkgdb2.APP.config.get('PKGDB2_FAS_INSECURE')
|
||||||
|
NOTIFYEMAIL = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_NOTIFY_EMAIL')
|
||||||
|
PKGDBSERVER = pkgdb2.APP.config.get('SITE_URL')
|
||||||
|
DRY_RUN = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_DRY_RUN', False)
|
||||||
|
|
||||||
|
# When querying for current info, take segments of 1000 packages a time
|
||||||
|
BZ_PKG_SEGMENT = 1000
|
||||||
|
|
||||||
|
class DataChangedError(Exception):
|
||||||
|
'''Raised when data we are manipulating changes while we're modifying it.'''
|
||||||
|
pass
|
||||||
|
|
||||||
|
def segment(iterable, chunk, fill=None):
|
||||||
|
'''Collect data into `chunk` sized block'''
|
||||||
|
args = [iter(iterable)] * chunk
|
||||||
|
return itertools.izip_longest(*args, fillvalue=fill)
|
||||||
|
|
||||||
|
class ProductCache(dict):
|
||||||
|
def __init__(self, bz, acls):
|
||||||
|
self.bz = bz
|
||||||
|
self.acls = acls
|
||||||
|
|
||||||
|
# Ask bugzilla for a section of the pkglist.
|
||||||
|
# Save the information from the section that we want.
|
||||||
|
def __getitem__(self, key):
|
||||||
|
try:
|
||||||
|
return super(ProductCache, self).__getitem__(key)
|
||||||
|
except KeyError:
|
||||||
|
# We can only cache products we have pkgdb information for
|
||||||
|
if key not in self.acls:
|
||||||
|
raise
|
||||||
|
|
||||||
|
if BZCOMPAPI == 'getcomponentsdetails':
|
||||||
|
# Old API -- in python-bugzilla. But with current server, this
|
||||||
|
# gives ProxyError
|
||||||
|
products = self.server.getcomponentsdetails(key)
|
||||||
|
elif BZCOMPAPI == 'component.get':
|
||||||
|
# Way that's undocumented in the partner-bugzilla api but works
|
||||||
|
# currently
|
||||||
|
pkglist = acls[key].keys()
|
||||||
|
products = {}
|
||||||
|
for pkg_segment in segment(pkglist, BZ_PKG_SEGMENT):
|
||||||
|
# Format that bugzilla will understand. Strip None's that segment() pads
|
||||||
|
# out the final data segment() with
|
||||||
|
query = [dict(product=key, component=p) for p in pkg_segment if p is not None]
|
||||||
|
raw_data = self.bz._proxy.Component.get(dict(names=query))
|
||||||
|
for package in raw_data['components']:
|
||||||
|
# Reformat data to be the same as what's returned from
|
||||||
|
# getcomponentsdetails
|
||||||
|
product = dict(initialowner=package['default_assignee'],
|
||||||
|
description=package['description'],
|
||||||
|
initialqacontact=package['default_qa_contact'],
|
||||||
|
initialcclist=package['default_cc'])
|
||||||
|
products[package['name'].lower()] = product
|
||||||
|
self[key] = products
|
||||||
|
|
||||||
|
return super(ProductCache, self).__getitem__(key)
|
||||||
|
|
||||||
|
|
||||||
|
class Bugzilla(object):
|
||||||
|
|
||||||
|
def __init__(self, bzServer, username, password, acls):
|
||||||
|
self.bzXmlRpcServer = bzServer
|
||||||
|
self.username = username
|
||||||
|
self.password = password
|
||||||
|
|
||||||
|
self.server = bugzilla.Bugzilla(
|
||||||
|
url=self.bzXmlRpcServer,
|
||||||
|
user=self.username,
|
||||||
|
password=self.password)
|
||||||
|
self.productCache = ProductCache(self.server, acls)
|
||||||
|
|
||||||
|
# Connect to the fedora account system
|
||||||
|
self.fas = AccountSystem(
|
||||||
|
base_url=FASURL,
|
||||||
|
username=FASUSER,
|
||||||
|
password=FASPASS)
|
||||||
|
self.userCache = self.fas.people_by_key(
|
||||||
|
key='username',
|
||||||
|
fields=['bugzilla_email'])
|
||||||
|
|
||||||
|
def _get_bugzilla_email(self, username):
|
||||||
|
'''Return the bugzilla email address for a user.
|
||||||
|
|
||||||
|
First looks in a cache for a username => bugzilla email. If not found,
|
||||||
|
reloads the cache from fas and tries again.
|
||||||
|
'''
|
||||||
|
try:
|
||||||
|
return self.userCache[username]['bugzilla_email'].lower()
|
||||||
|
except KeyError:
|
||||||
|
if username.startswith('@'):
|
||||||
|
group = self.fas.group_by_name(username[1:])
|
||||||
|
self.userCache[username] = {
|
||||||
|
'bugzilla_email': group.mailing_list}
|
||||||
|
else:
|
||||||
|
person = self.fas.person_by_username(username)
|
||||||
|
bz_email = person.get('bugzilla_email', None)
|
||||||
|
if bz_email is None:
|
||||||
|
print '%s has no bugzilla email, valid account?' % username
|
||||||
|
else:
|
||||||
|
self.userCache[username] = {'bugzilla_email': bz_email}
|
||||||
|
return self.userCache[username]['bugzilla_email'].lower()
|
||||||
|
|
||||||
|
def add_edit_component(self, package, collection, owner, description,
|
||||||
|
qacontact=None, cclist=None):
|
||||||
|
'''Add or update a component to have the values specified.
|
||||||
|
'''
|
||||||
|
# Turn the cclist into something usable by bugzilla
|
||||||
|
if not cclist or 'people' not in cclist:
|
||||||
|
initialCCList = list()
|
||||||
|
else:
|
||||||
|
initialCCList = [
|
||||||
|
self._get_bugzilla_email(cc) for cc in cclist['people']]
|
||||||
|
if 'groups' in cclist:
|
||||||
|
group_cc = [
|
||||||
|
self._get_bugzilla_email(cc) for cc in cclist['groups']]
|
||||||
|
initialCCList.extend(group_cc)
|
||||||
|
|
||||||
|
# Add owner to the cclist so comaintainers taking over a bug don't
|
||||||
|
# have to do this manually
|
||||||
|
owner = self._get_bugzilla_email(owner)
|
||||||
|
if owner not in initialCCList:
|
||||||
|
initialCCList.append(owner)
|
||||||
|
|
||||||
|
# Lookup product
|
||||||
|
try:
|
||||||
|
product = self.productCache[collection]
|
||||||
|
except xmlrpclib.Fault as e:
|
||||||
|
# Output something useful in args
|
||||||
|
e.args = (e.faultCode, e.faultString)
|
||||||
|
raise
|
||||||
|
except xmlrpclib.ProtocolError as e:
|
||||||
|
e.args = ('ProtocolError', e.errcode, e.errmsg)
|
||||||
|
raise
|
||||||
|
|
||||||
|
pkgKey = package.lower()
|
||||||
|
if pkgKey in product:
|
||||||
|
# edit the package information
|
||||||
|
data = {}
|
||||||
|
|
||||||
|
# Grab bugzilla email for things changable via xmlrpc
|
||||||
|
if qacontact:
|
||||||
|
qacontact = self._get_bugzilla_email(qacontact)
|
||||||
|
else:
|
||||||
|
qacontact = 'extras-qa@fedoraproject.org'
|
||||||
|
|
||||||
|
# Check for changes to the owner, qacontact, or description
|
||||||
|
if product[pkgKey]['initialowner'] != owner:
|
||||||
|
data['initialowner'] = owner
|
||||||
|
|
||||||
|
if product[pkgKey]['description'] != description:
|
||||||
|
data['description'] = description
|
||||||
|
if product[pkgKey]['initialqacontact'] != qacontact and (
|
||||||
|
qacontact or product[pkgKey]['initialqacontact']):
|
||||||
|
data['initialqacontact'] = qacontact
|
||||||
|
|
||||||
|
if len(product[pkgKey]['initialcclist']) != len(initialCCList):
|
||||||
|
data['initialcclist'] = initialCCList
|
||||||
|
else:
|
||||||
|
for ccMember in product[pkgKey]['initialcclist']:
|
||||||
|
if ccMember not in initialCCList:
|
||||||
|
data['initialcclist'] = initialCCList
|
||||||
|
break
|
||||||
|
|
||||||
|
if data:
|
||||||
|
### FIXME: initialowner has been made mandatory for some
|
||||||
|
# reason. Asking dkl why.
|
||||||
|
data['initialowner'] = owner
|
||||||
|
|
||||||
|
# Changes occurred. Submit a request to change via xmlrpc
|
||||||
|
data['product'] = collection
|
||||||
|
data['component'] = package
|
||||||
|
if DRY_RUN:
|
||||||
|
print '[EDITCOMP] Changing via editComponent(' \
|
||||||
|
'%s, %s, "xxxxx")' % (data, self.username)
|
||||||
|
print '[EDITCOMP] Former values: %s|%s|%s|%s' % (
|
||||||
|
product[pkgKey]['initialowner'],
|
||||||
|
product[pkgKey]['description'],
|
||||||
|
product[pkgKey]['initialqacontact'],
|
||||||
|
product[pkgKey]['initialcclist'])
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
self.server.editcomponent(data)
|
||||||
|
except xmlrpclib.Fault, e:
|
||||||
|
# Output something useful in args
|
||||||
|
e.args = (data, e.faultCode, e.faultString)
|
||||||
|
raise
|
||||||
|
except xmlrpclib.ProtocolError, e:
|
||||||
|
e.args = ('ProtocolError', e.errcode, e.errmsg)
|
||||||
|
raise
|
||||||
|
else:
|
||||||
|
# Add component
|
||||||
|
if qacontact:
|
||||||
|
qacontact = self._get_bugzilla_email(qacontact)
|
||||||
|
else:
|
||||||
|
qacontact = 'extras-qa@fedoraproject.org'
|
||||||
|
|
||||||
|
data = {
|
||||||
|
'product': collection,
|
||||||
|
'component': package,
|
||||||
|
'description': description,
|
||||||
|
'initialowner': owner,
|
||||||
|
'initialqacontact': qacontact
|
||||||
|
}
|
||||||
|
if initialCCList:
|
||||||
|
data['initialcclist'] = initialCCList
|
||||||
|
|
||||||
|
if DRY_RUN:
|
||||||
|
print '[ADDCOMP] Adding new component AddComponent:(' \
|
||||||
|
'%s, %s, "xxxxx")' % (data, self.username)
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
self.server.addcomponent(data)
|
||||||
|
except xmlrpclib.Fault, e:
|
||||||
|
# Output something useful in args
|
||||||
|
e.args = (data, e.faultCode, e.faultString)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def send_email(fromAddress, toAddress, subject, message):
|
||||||
|
'''Send an email if there's an error.
|
||||||
|
|
||||||
|
This will be replaced by sending messages to a log later.
|
||||||
|
'''
|
||||||
|
msg = Message()
|
||||||
|
msg.add_header('To', ','.join(toAddress))
|
||||||
|
msg.add_header('From', fromAddress)
|
||||||
|
msg.add_header('Subject', subject)
|
||||||
|
msg.set_payload(message)
|
||||||
|
smtp = smtplib.SMTP('bastion')
|
||||||
|
smtp.sendmail(fromAddress, toAddress, msg.as_string())
|
||||||
|
smtp.quit()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.stdout = codecs.getwriter('utf-8')(sys.stdout)
|
||||||
|
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description='Script syncing information between pkgdb and bugzilla'
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--debug', dest='debug', action='store_true', default=False,
|
||||||
|
help='Print the changes instead of making them in bugzilla')
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.debug:
|
||||||
|
DRY_RUN = True
|
||||||
|
|
||||||
|
# Non-fatal errors to alert people about
|
||||||
|
errors = []
|
||||||
|
|
||||||
|
# Get bugzilla information from the package database
|
||||||
|
req = requests.get('%s/api/bugzilla/?format=json' % PKGDBSERVER)
|
||||||
|
acls = req.json()['bugzillaAcls']
|
||||||
|
|
||||||
|
# Initialize the connection to bugzilla
|
||||||
|
bugzilla = Bugzilla(BZSERVER, BZUSER, BZPASS, acls)
|
||||||
|
|
||||||
|
for product in acls.keys():
|
||||||
|
if product not in ('Fedora', 'Fedora EPEL'):
|
||||||
|
continue
|
||||||
|
for pkg in acls[product]:
|
||||||
|
if DRY_RUN:
|
||||||
|
print pkg
|
||||||
|
pkgInfo = acls[product][pkg]
|
||||||
|
try:
|
||||||
|
bugzilla.add_edit_component(
|
||||||
|
pkg,
|
||||||
|
product,
|
||||||
|
pkgInfo['owner'],
|
||||||
|
pkgInfo['summary'],
|
||||||
|
pkgInfo['qacontact'],
|
||||||
|
pkgInfo['cclist'])
|
||||||
|
except ValueError, e:
|
||||||
|
# A username didn't have a bugzilla address
|
||||||
|
errors.append(str(e.args))
|
||||||
|
except DataChangedError, e:
|
||||||
|
# A Package or Collection was returned via xmlrpc but wasn't
|
||||||
|
# present when we tried to change it
|
||||||
|
errors.append(str(e.args))
|
||||||
|
except xmlrpclib.ProtocolError, e:
|
||||||
|
# Unrecoverable and likely means that nothing is going to
|
||||||
|
# succeed.
|
||||||
|
errors.append(str(e.args))
|
||||||
|
break
|
||||||
|
except xmlrpclib.Error, e:
|
||||||
|
# An error occurred in the xmlrpc call. Shouldn't happen but
|
||||||
|
# we better see what it is
|
||||||
|
errors.append(str(e.args))
|
||||||
|
|
||||||
|
# Send notification of errors
|
||||||
|
if errors:
|
||||||
|
#print '[DEBUG]', '\n'.join(errors)
|
||||||
|
send_email('accounts@fedoraproject.org',
|
||||||
|
NOTIFYEMAIL,
|
||||||
|
'Errors while syncing bugzilla with the PackageDB',
|
||||||
|
'''
|
||||||
|
The following errors were encountered while updating bugzilla with information
|
||||||
|
from the Package Database. Please have the problems taken care of:
|
||||||
|
|
||||||
|
%s
|
||||||
|
''' % ('\n'.join(errors),))
|
||||||
|
|
||||||
|
sys.exit(0)
|
||||||
Loading…
Add table
Add a link
Reference in a new issue