857 lines
34 KiB
Python
857 lines
34 KiB
Python
# -*- coding: utf-8 -*-
|
|
#
|
|
# Copyright © 2013-2019 Red Hat, Inc.
|
|
#
|
|
# This copyrighted material is made available to anyone wishing to use, modify,
|
|
# copy, or redistribute it subject to the terms and conditions of the GNU
|
|
# General Public License v.2, or (at your option) any later version. This
|
|
# program is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
# WARRANTY expressed or implied, including the implied warranties of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
|
|
# Public License for more details. You should have received a copy of the GNU
|
|
# General Public License along with this program; if not, write to the Free
|
|
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the source
|
|
# code or documentation are not subject to the GNU General Public License and
|
|
# may only be used or replicated with the express permission of Red Hat, Inc.
|
|
#
|
|
# Red Hat Author(s): Toshio Kuratomi <tkuratom@redhat.com>
|
|
# Author(s): Mike Watters <valholla75@fedoraproject.org>
|
|
# Author(s): Pierre-Yves Chibon <pingou@pingoured.fr>
|
|
# Author(s): Matt Prahl <mprahl@redhat.com>
|
|
# Author(s): Ralph Bean <rbean@redhat.com
|
|
# Author(s): Patrick Uiterwijk <puiterwijk@redhat.com>
|
|
#
|
|
'''
|
|
sync information from the Pagure into bugzilla
|
|
|
|
This ... script takes information about package onwership and imports it
|
|
into bugzilla.
|
|
'''
|
|
|
|
import argparse
|
|
import datetime
|
|
from email.message import EmailMessage
|
|
import itertools
|
|
import json
|
|
from operator import itemgetter
|
|
import os
|
|
import re
|
|
import smtplib
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import xmlrpc.client
|
|
|
|
from bugzilla import Bugzilla
|
|
import dogpile.cache
|
|
import fedora.client
|
|
from fedora.client.fas2 import AccountSystem
|
|
import requests
|
|
from requests.adapters import HTTPAdapter
|
|
from urllib3.util import Retry
|
|
import yaml
|
|
|
|
from . import package_summaries
|
|
from .config import config, email_overrides, load_configuration
|
|
|
|
|
|
cache = dogpile.cache.make_region().configure(
|
|
'dogpile.cache.memory',
|
|
expiration_time=3600,
|
|
)
|
|
|
|
|
|
def retry_session():
|
|
session = requests.Session()
|
|
retry = Retry(
|
|
total=5,
|
|
read=5,
|
|
connect=5,
|
|
backoff_factor=0.3,
|
|
status_forcelist=(500, 502, 504),
|
|
)
|
|
adapter = HTTPAdapter(max_retries=retry)
|
|
session.mount('http://', adapter)
|
|
session.mount('https://', adapter)
|
|
return session
|
|
|
|
|
|
def resilient_partial(fn, *initial, **kwargs):
|
|
""" A decorator that partially applies arguments.
|
|
|
|
It additionally catches all raised exceptions, prints them, but then returns
|
|
None instead of propagating the failures.
|
|
|
|
This is used to protect functions used in a threadpool. If one fails, we
|
|
want to know about it, but we don't want it to kill the whole program. So
|
|
catch its error, log it, but proceed.
|
|
"""
|
|
def wrapper(*additional):
|
|
try:
|
|
full = initial + additional
|
|
return fn(*full, **kwargs)
|
|
except Exception:
|
|
traceback.print_exc()
|
|
return None
|
|
wrapper.__name__ = fn.__name__
|
|
wrapper.__doc__ = fn.__doc__
|
|
return wrapper
|
|
|
|
|
|
class DataChangedError(Exception):
|
|
'''Raised when data we are manipulating changes while we're modifying it.'''
|
|
pass
|
|
|
|
|
|
def segment(iterable, chunk, fill=None):
|
|
'''Collect data into `chunk` sized block'''
|
|
args = [iter(iterable)] * chunk
|
|
return itertools.zip_longest(*args, fillvalue=fill)
|
|
|
|
|
|
class BugzillaProxy:
|
|
|
|
def __init__(self, bz_server, username, password, config,
|
|
pre_cache_users=True):
|
|
self.bz_xmlrpc_server = bz_server
|
|
self.username = username
|
|
self.password = password
|
|
|
|
self.server = Bugzilla(
|
|
url=self.bz_xmlrpc_server,
|
|
user=self.username,
|
|
password=self.password)
|
|
|
|
self.product_cache = {}
|
|
self.user_cache = {}
|
|
self.inverted_user_cache = {}
|
|
|
|
# Connect to the fedora account system
|
|
self.fas = AccountSystem(
|
|
base_url=config['fas']['url'],
|
|
username=config['fas']['username'],
|
|
password=config['fas']['password'])
|
|
|
|
self.config = config
|
|
|
|
if not pre_cache_users:
|
|
return
|
|
|
|
if config['verbose']:
|
|
print("Pre-caching FAS users and their Bugzilla email addresses.")
|
|
|
|
try:
|
|
self.user_cache = self.fas.people_by_key(
|
|
key='username',
|
|
fields=['bugzilla_email'])
|
|
except fedora.client.ServerError:
|
|
# Sometimes, building the user cache up front fails with a timeout.
|
|
# It's ok, we build the cache as-needed later in the script.
|
|
pass
|
|
else:
|
|
self.invert_user_cache()
|
|
|
|
def build_product_cache(self, pagure_projects):
|
|
""" Cache the bugzilla info about each package in each product.
|
|
"""
|
|
|
|
products = {}
|
|
if self.config['bugzilla']['compat_api'] == 'getcomponentsdetails':
|
|
# Old API -- in python-bugzilla. But with current server, this
|
|
# gives ProxyError
|
|
for collection, product in self.config["products"].items():
|
|
self.product_cache[collection] = self.server.getcomponentsdetails(product)
|
|
elif self.config['bugzilla']['compat_api'] == 'component.get':
|
|
# Way that's undocumented in the partner-bugzilla api but works
|
|
# currently
|
|
products = {}
|
|
for collection, product in self.config["products"].items():
|
|
|
|
# restrict the list of info returned to only the packages of
|
|
# interest
|
|
pkglist = [
|
|
project["name"]
|
|
for project in pagure_projects
|
|
if product in project["products"]
|
|
]
|
|
for pkg_segment in segment(pkglist, self.config['bugzilla']['req_segment']):
|
|
# Format that bugzilla will understand. Strip None's that
|
|
# segment() pads out the final data segment() with
|
|
query = [
|
|
dict(
|
|
product=self.config['products'][collection],
|
|
component=p
|
|
)
|
|
for p in pkg_segment
|
|
if p is not None
|
|
]
|
|
raw_data = self.server._proxy.Component.get(dict(names=query))
|
|
for package in raw_data['components']:
|
|
# Reformat data to be the same as what's returned from
|
|
# getcomponentsdetails
|
|
product = dict(
|
|
initialowner=package['default_assignee'],
|
|
description=package['description'],
|
|
initialqacontact=package['default_qa_contact'],
|
|
initialcclist=package['default_cc']
|
|
)
|
|
products[package['name'].lower()] = product
|
|
self.product_cache[collection] = products
|
|
|
|
def invert_user_cache(self):
|
|
""" Takes the user_cache built when querying FAS and invert it so
|
|
that the bugzilla_email is the key and the username the value.
|
|
"""
|
|
for username in self.user_cache:
|
|
bz_email = self.user_cache[username]['bugzilla_email'].lower()
|
|
self.inverted_user_cache[bz_email] = username
|
|
|
|
def _get_bugzilla_email(self, username):
|
|
'''Return the bugzilla email address for a user.
|
|
|
|
First looks in a cache for a username => bugzilla email. If not found,
|
|
reloads the cache from fas and tries again.
|
|
'''
|
|
try:
|
|
bz_email = self.user_cache[username]['bugzilla_email'].lower()
|
|
except KeyError:
|
|
if username.startswith('@'):
|
|
group = self.fas.group_by_name(username[1:])
|
|
bz_email = group.mailing_list
|
|
if bz_email is None:
|
|
return
|
|
self.user_cache[username] = {
|
|
'bugzilla_email': bz_email}
|
|
else:
|
|
person = self.fas.person_by_username(username)
|
|
bz_email = person.get('bugzilla_email', None)
|
|
if bz_email is None:
|
|
return
|
|
self.user_cache[username] = {'bugzilla_email': bz_email}
|
|
bz_email = bz_email.lower()
|
|
self.inverted_user_cache[bz_email] = username
|
|
bz_email = email_overrides.get(bz_email, bz_email)
|
|
return bz_email
|
|
|
|
def add_edit_component(self, package, collection, owner, description=None,
|
|
qacontact=None, cclist=None, print_fas_names=False):
|
|
'''Add or update a component to have the values specified.
|
|
'''
|
|
# Turn the cclist into something usable by bugzilla
|
|
initial_cc_emails = []
|
|
initial_cc_fasnames = []
|
|
for watcher in cclist:
|
|
bz_email = self._get_bugzilla_email(watcher)
|
|
if bz_email:
|
|
initial_cc_emails.append(bz_email)
|
|
initial_cc_fasnames.append(watcher)
|
|
else:
|
|
print(f"** {watcher} has no bugzilla_email or mailing_list set "
|
|
f"({collection}/{package}) **")
|
|
|
|
# Add owner to the cclist so comaintainers taking over a bug don't
|
|
# have to do this manually
|
|
owner_email = self._get_bugzilla_email(owner)
|
|
if owner_email not in initial_cc_emails:
|
|
initial_cc_emails.append(owner_email)
|
|
initial_cc_fasnames.append(owner)
|
|
|
|
# Lookup product
|
|
try:
|
|
product = self.product_cache[collection]
|
|
except xmlrpc.client.Fault as e:
|
|
# Output something useful in args
|
|
e.args = (e.faultCode, e.faultString)
|
|
raise
|
|
except xmlrpc.client.ProtocolError as e:
|
|
e.args = ('ProtocolError', e.errcode, e.errmsg)
|
|
raise
|
|
|
|
# Set the qacontact_email and name
|
|
if qacontact:
|
|
qacontact_email = self._get_bugzilla_email(qacontact)
|
|
else:
|
|
qacontact_email = 'extras-qa@fedoraproject.org'
|
|
qacontact = '<default: extras-qa@...>'
|
|
|
|
pkg_key = package.lower()
|
|
if pkg_key in product:
|
|
# edit the package information
|
|
data = {}
|
|
|
|
# Check for changes to the owner, qacontact, or description
|
|
if product[pkg_key]['initialowner'] != owner_email:
|
|
data['initialowner'] = owner_email
|
|
|
|
if description and product[pkg_key]['description'] != description:
|
|
data['description'] = description
|
|
|
|
if qacontact and product[pkg_key]['initialqacontact'] != qacontact_email:
|
|
data['initialqacontact'] = qacontact_email
|
|
|
|
if len(product[pkg_key]['initialcclist']) != len(initial_cc_emails):
|
|
data['initialcclist'] = initial_cc_emails
|
|
else:
|
|
for cc_member in product[pkg_key]['initialcclist']:
|
|
if cc_member not in initial_cc_emails:
|
|
data['initialcclist'] = initial_cc_emails
|
|
break
|
|
|
|
if data:
|
|
# Changes occurred. Submit a request to change via xmlrpc
|
|
data['product'] = self.config['products'][collection]
|
|
data['component'] = package
|
|
|
|
if self.config["verbose"]:
|
|
print(f'[EDITCOMP] {data["product"]}/{data["component"]}')
|
|
for key in ["initialowner", "description", "initialqacontact", "initialcclist"]:
|
|
if data.get(key):
|
|
|
|
old_value = product[pkg_key][key]
|
|
if not isinstance(old_value, str):
|
|
old_value = sorted(old_value)
|
|
new_value = data.get(key)
|
|
if not isinstance(new_value, str):
|
|
new_value = sorted(new_value)
|
|
|
|
if print_fas_names and key in ('initialowner',
|
|
'initialqacontact',
|
|
'initialcclist'):
|
|
if key == 'initialowner':
|
|
new_value = owner
|
|
elif key == 'initialqacontact':
|
|
new_value = qacontact
|
|
else:
|
|
new_value = sorted(initial_cc_fasnames)
|
|
|
|
from_fas_names = []
|
|
for email in product[pkg_key][key]:
|
|
if email in self.inverted_user_cache:
|
|
from_fas_names.append(self.inverted_user_cache[email])
|
|
elif email == 'extras-qa@fedoraproject.org':
|
|
from_fas_names.append("<default: extras-qa@...>")
|
|
if from_fas_names:
|
|
if len(from_fas_names) < len(product[pkg_key][key]):
|
|
x = len(product[pkg_key][key]) - len(from_fas_names)
|
|
from_fas_names.append(f"And {x} more")
|
|
old_value = f"from `{from_fas_names}`"
|
|
else:
|
|
old_value = ""
|
|
print(
|
|
f" {key} changed {old_value}"
|
|
f" to FAS name(s) `{new_value}`")
|
|
else:
|
|
print(f" {key} changed from `{old_value}` to `{new_value}`")
|
|
|
|
# FIXME: initialowner has been made mandatory for some
|
|
# reason. Asking dkl why.
|
|
data['initialowner'] = owner_email
|
|
if not self.config["dryrun"]:
|
|
try:
|
|
self.server.editcomponent(data)
|
|
except xmlrpc.client.Fault as e:
|
|
# Output something useful in args
|
|
e.args = (data, e.faultCode, e.faultString)
|
|
raise
|
|
except xmlrpc.client.ProtocolError as e:
|
|
e.args = ('ProtocolError', e.errcode, e.errmsg)
|
|
raise
|
|
else:
|
|
# Add component
|
|
data = {
|
|
'product': self.config['products'][collection],
|
|
'component': package,
|
|
'description': description or 'NA',
|
|
'initialowner': owner_email,
|
|
'initialqacontact': qacontact_email
|
|
}
|
|
if initial_cc_emails:
|
|
data['initialcclist'] = initial_cc_emails
|
|
|
|
if self.config["verbose"]:
|
|
print('[ADDCOMP] %s/%s' % (data["product"], data["component"]))
|
|
for key in ["initialowner", "description", "initialqacontact", "initialcclist"]:
|
|
if print_fas_names and key in ('initialowner',
|
|
'initialqacontact',
|
|
'initialcclist'):
|
|
if key == 'initialowner':
|
|
value = owner
|
|
elif key == 'initialqacontact':
|
|
value = qacontact
|
|
else:
|
|
value = initial_cc_fasnames
|
|
print(f" {key} set to FAS name(s) `{value}`")
|
|
else:
|
|
print(f" {key} set to {data.get(key)}")
|
|
if not self.config["dryrun"]:
|
|
try:
|
|
self.server.addcomponent(data)
|
|
except xmlrpc.client.Fault as e:
|
|
# Output something useful in args
|
|
e.args = (data, e.faultCode, e.faultString)
|
|
raise
|
|
|
|
|
|
def send_email(from_address, to_address, subject, message, cc_address=None):
|
|
'''Send an email if there's an error.
|
|
|
|
This will be replaced by sending messages to a log later.
|
|
'''
|
|
if env == 'staging':
|
|
# Send no email in staging...
|
|
pass
|
|
else:
|
|
msg = EmailMessage()
|
|
msg.add_header('To', ','.join(to_address))
|
|
msg.add_header('From', from_address)
|
|
msg.add_header('Subject', subject)
|
|
if cc_address is not None:
|
|
msg.add_header('Cc', ','.join(cc_address))
|
|
to_address += cc_address
|
|
msg.set_payload(message)
|
|
smtp = smtplib.SMTP('bastion')
|
|
smtp.sendmail(from_address, to_address, msg.as_string())
|
|
smtp.quit()
|
|
|
|
|
|
def _get_pdc_branches(session, repo):
|
|
"""
|
|
Gets the branches on a project. This function is used for mapping.
|
|
:param repo: the project dict
|
|
:return: a list of the repo's branches
|
|
"""
|
|
branches_url = '{0}component-branches/'.format(env['pdc_url'])
|
|
params = dict(
|
|
global_component=repo['name'],
|
|
type=env['pdc_types'][repo['namespace']]
|
|
)
|
|
if config["verbose"]:
|
|
print('Querying {0} {1}'.format(branches_url, params))
|
|
rv = session.get(branches_url, params=params, timeout=60)
|
|
|
|
# If the project's branches can't be reported, just return no branches and
|
|
# it will be skipped later on
|
|
if not rv.ok:
|
|
print(('The connection to "{0}" failed with the status code {1} and '
|
|
'output "{2}"'.format(branches_url, rv.status_code, rv.text)),
|
|
file=sys.stderr)
|
|
return []
|
|
|
|
data = rv.json()
|
|
return [branch['name'] for branch in data['results']]
|
|
|
|
|
|
def _is_retired(product, project):
|
|
branches = project['branches']
|
|
if product == 'Fedora EPEL':
|
|
for branch, active in branches:
|
|
if re.match(r'^epel\d+$', branch):
|
|
if active:
|
|
return False
|
|
# No active branches means it is retired.
|
|
return True
|
|
else:
|
|
for branch, active in branches:
|
|
if active:
|
|
return False
|
|
return True
|
|
|
|
|
|
class ScriptExecError(RuntimeError):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
self.errorcode = kwargs.pop('errorcode', 1)
|
|
super().__init__(*args, **kwargs)
|
|
|
|
|
|
class DistgitBugzillaSync:
|
|
|
|
def notify_users(self, errors):
|
|
''' Browse the list of errors and when we can retrieve the email
|
|
address, use it to notify the user about the issue.
|
|
'''
|
|
data = {}
|
|
if os.path.exists(self.env['data_cache']):
|
|
try:
|
|
with open(self.env['data_cache']) as stream:
|
|
data = json.load(stream)
|
|
except Exception as err:
|
|
print('Could not read the json file at %s: \nError: %s' % (
|
|
env['data_cache'], err))
|
|
|
|
new_data = {}
|
|
seen = []
|
|
for error in errors:
|
|
notify_user = False
|
|
if 'The name ' in error and ' is not a valid username' in error:
|
|
user_email = error.split(' is not a valid username')[0].split(
|
|
'The name ')[1].strip()
|
|
now = datetime.datetime.utcnow()
|
|
|
|
# See if we already know about this user
|
|
if user_email in data and data[user_email]['last_update']:
|
|
last_update = datetime.datetime.fromtimestamp(
|
|
int(data[user_email]['last_update']))
|
|
# Only notify users once per hour
|
|
if (now - last_update).seconds >= 3600:
|
|
notify_user = True
|
|
else:
|
|
new_data[user_email] = data[user_email]
|
|
elif not data or user_email not in data:
|
|
notify_user = True
|
|
|
|
# Ensure we notify the user only once, no matter how many errors we
|
|
# got concerning them.
|
|
if user_email not in seen:
|
|
seen.append(user_email)
|
|
else:
|
|
notify_user = False
|
|
|
|
if notify_user:
|
|
send_email(
|
|
self.env['email_from'],
|
|
[user_email],
|
|
subject='Please fix your bugzilla.redhat.com account',
|
|
message=self.env['tmpl_user_email'],
|
|
cc_address=self.env['notify_emails'],
|
|
)
|
|
|
|
new_data[user_email] = {
|
|
'last_update': time.mktime(now.timetuple())
|
|
}
|
|
|
|
with open(env['data_cache'], 'w') as stream:
|
|
json.dump(new_data, stream)
|
|
|
|
def get_cli_arguments(self):
|
|
""" Set the CLI argument parser and return the argument parsed.
|
|
"""
|
|
parser = argparse.ArgumentParser(
|
|
description='Script syncing information between Pagure and bugzilla'
|
|
)
|
|
parser.add_argument(
|
|
'--dry-run', dest='dryrun', action='store_true', default=False,
|
|
help='Do not actually make any changes - Overrides the configuration')
|
|
parser.add_argument(
|
|
'--verbose', dest='verbose', action='store_true', default=False,
|
|
help='Print actions verbosely - Overrides the configuration')
|
|
parser.add_argument(
|
|
'--debug', dest='debug', action='store_true', default=False,
|
|
help='Combination of --verbose and --dry-run')
|
|
parser.add_argument(
|
|
'--env', dest='env',
|
|
help='Run the script for a specific environment, overrides the one '
|
|
'set in the configuration file')
|
|
parser.add_argument(
|
|
'--add-config-file', metavar='CONFIG_FILE',
|
|
dest='addl_config_files', action='append',
|
|
help="File(s) from which to read overriding configuration")
|
|
parser.add_argument(
|
|
'--add-email-overrides-file', metavar='EMAIL_OVERRIDES_FILE',
|
|
dest='addl_email_overrides_files', action='append',
|
|
help="File(s) from which to read additional email overrides")
|
|
parser.add_argument(
|
|
'-p', '--project', dest='projects', nargs='+',
|
|
help='Update one or more projects (provided as namespace/name), '
|
|
'in all of its products')
|
|
parser.add_argument(
|
|
'--print-fas-names', action='store_true', default=False,
|
|
help="Print FAS names rather than email addresses in output, useful when pasting into "
|
|
"public fora")
|
|
|
|
self.args = parser.parse_args()
|
|
|
|
def get_pagure_projects(self, project_list=None):
|
|
""" Builds a large list of all the projects on pagure.
|
|
Each item in that list is a dict containing:
|
|
- the namespace of the project
|
|
- the name of the project
|
|
- the point of contact of this project (ie: the default assignee
|
|
in bugzilla)
|
|
- the watchers of this project (ie: the initial CC list in bugzilla)
|
|
"""
|
|
|
|
# Get the initial ownership and CC data from pagure
|
|
# This part is easy.
|
|
poc_url = self.env['distgit_url'] + '/extras/pagure_poc.json'
|
|
if self.env["verbose"]:
|
|
print("Querying %r for points of contact." % poc_url)
|
|
pagure_namespace_to_poc = self.session.get(poc_url, timeout=120).json()
|
|
|
|
cc_url = self.env['distgit_url'] + '/extras/pagure_bz.json'
|
|
if self.env["verbose"]:
|
|
print("Querying %r for initial cc list." % cc_url)
|
|
pagure_namespace_to_cc = self.session.get(cc_url, timeout=120).json()
|
|
|
|
# Combine and collapse those two into a single list:
|
|
self.pagure_projects = []
|
|
if project_list:
|
|
project_list = set(tuple(p.split("/", 1)) for p in project_list)
|
|
for namespace, entries in pagure_namespace_to_poc.items():
|
|
for name, poc in entries.items():
|
|
if not project_list or (namespace, name) in project_list:
|
|
self.pagure_projects.append(dict(
|
|
namespace=namespace,
|
|
name=name,
|
|
poc=poc,
|
|
watchers=pagure_namespace_to_cc[namespace][name],
|
|
))
|
|
|
|
def add_branches_products_and_summaries(self):
|
|
""" For each project retrieved, this method adds branches, products
|
|
and summary information.
|
|
|
|
The branches are retrieved from PDC.
|
|
|
|
The products are determined based on the branches.
|
|
|
|
The summaries are coming from the primary.xml file of the Rawhide repodata
|
|
in Koji.
|
|
"""
|
|
branches_url = "/".join([
|
|
self.env['pdc_url'].split('rest_api')[0].rstrip("/"),
|
|
'extras/active_branches.json',
|
|
])
|
|
if self.env["verbose"]:
|
|
print("Querying %r for EOL information." % branches_url)
|
|
|
|
pdc_branches = self.session.get(branches_url, timeout=120).json()
|
|
for idx, project in enumerate(self.pagure_projects):
|
|
# Summary
|
|
summary = None
|
|
if project["namespace"] == "rpms":
|
|
summary = self.rpm_summaries.get(project["name"])
|
|
project["summary"] = summary
|
|
|
|
# Branches
|
|
if project['namespace'] not in self.env['pdc_types']:
|
|
project['branches'] = []
|
|
project['products'] = []
|
|
if self.env["verbose"]:
|
|
print(
|
|
f'! Namespace {project["namespace"]} not found in the pdc_type '
|
|
f'configuration key, project {project["namespace"]}/{project["name"]} '
|
|
'ignored'
|
|
)
|
|
continue
|
|
|
|
pdc_type = self.env['pdc_types'][project['namespace']]
|
|
project['branches'] = pdc_branches.get(pdc_type, {}).get(project['name'], [])
|
|
if not project['branches']:
|
|
if self.env["verbose"]:
|
|
print(f"! No PDC branch found for {project['namespace']}/{project['name']}")
|
|
|
|
# Products
|
|
products = set()
|
|
for branch, active in project.get('branches'):
|
|
if re.match(r'^epel\d+$', branch):
|
|
products.add('Fedora EPEL')
|
|
else:
|
|
products.add(self.env['namespace_to_product'][project['namespace']])
|
|
project['products'] = list(products)
|
|
|
|
products_poc = {}
|
|
for product in products:
|
|
owner = project["poc"]
|
|
# Check if the project is retired in PDC, and if so set assignee to orphan.
|
|
if _is_retired(product, project):
|
|
owner = 'orphan'
|
|
|
|
# Check if the Bugzilla ticket assignee has been manually overridden
|
|
override_yaml = self._get_override_yaml(project, self.session)
|
|
if override_yaml.get(product) \
|
|
and isinstance(override_yaml[product], str):
|
|
owner = override_yaml[product]
|
|
products_poc[product] = owner
|
|
project['products_poc'] = products_poc
|
|
|
|
self.pagure_projects[idx] = project
|
|
|
|
@cache.cache_on_arguments()
|
|
def _get_override_yaml(self, project, session):
|
|
pagure_override_url = '{0}/{1}/raw/master/f/{2}/{3}'.format(
|
|
self.env['pagure_url'].rstrip('/'),
|
|
self.env['bugzilla']['override_repo'],
|
|
project['namespace'],
|
|
project['name'],
|
|
)
|
|
|
|
if self.env["verbose"]:
|
|
print('Querying {0}'.format(pagure_override_url))
|
|
override_rv = session.get(pagure_override_url, timeout=30)
|
|
if override_rv.status_code == 200:
|
|
override_yaml = yaml.safe_load(override_rv.text)
|
|
return override_yaml.get('bugzilla_contact', {})
|
|
return {}
|
|
|
|
@classmethod
|
|
def main(cls):
|
|
"""The entrypoint for running the script."""
|
|
dbs = cls()
|
|
try:
|
|
dbs.run()
|
|
except ScriptExecError as e:
|
|
print(str(e), file=sys.stderr)
|
|
sys.exit(e.errorcode)
|
|
else:
|
|
sys.exit(0)
|
|
|
|
def run(self):
|
|
"""Run the script."""
|
|
global envname, env, projects_dict
|
|
times = {
|
|
"start": time.time(),
|
|
}
|
|
|
|
self.get_cli_arguments()
|
|
|
|
load_configuration(addl_config_files=self.args.addl_config_files,
|
|
addl_email_overrides_files=self.args.addl_email_overrides_files)
|
|
self.config = config
|
|
|
|
envname = self.config['environment']
|
|
if self.args.env:
|
|
if self.args.env in self.config['environments']:
|
|
envname = self.args.env
|
|
else:
|
|
raise ScriptExecError(f"Invalid environment specified: {self.args.env}")
|
|
|
|
self.env = self.config['environments'][envname]
|
|
|
|
if self.args.debug:
|
|
self.env["verbose"] = True
|
|
self.env["dryrun"] = True
|
|
|
|
if self.args.verbose:
|
|
self.env["verbose"] = True
|
|
|
|
if self.args.dryrun:
|
|
self.env["dryrun"] = True
|
|
|
|
# Non-fatal errors to alert people about
|
|
errors = []
|
|
|
|
self.session = retry_session()
|
|
|
|
if self.env["verbose"]:
|
|
print("Building a cache of the rpm package summaries")
|
|
self.rpm_summaries = package_summaries.get_package_summaries()
|
|
|
|
self.get_pagure_projects(self.args.projects)
|
|
self.add_branches_products_and_summaries()
|
|
|
|
if self.env["verbose"]:
|
|
print(f"{len(self.pagure_projects)} projects to consider")
|
|
|
|
if not self.pagure_projects:
|
|
return
|
|
|
|
if self.env["verbose"]:
|
|
times["data structure end"] = time.time()
|
|
delta = times["data structure end"] - times["start"]
|
|
print("Ran for %s seconds -- ie: %.2f minutes" % (delta, delta/60.0))
|
|
|
|
# Initialize the connection to bugzilla
|
|
bugzilla = BugzillaProxy(
|
|
bz_server=self.env['bugzilla']['url'],
|
|
username=self.env['bugzilla']['user'],
|
|
password=self.env['bugzilla']['password'],
|
|
config=self.env,
|
|
pre_cache_users=not self.args.projects or self.args.print_fas_names,
|
|
)
|
|
|
|
if self.env["verbose"]:
|
|
times["FAS cache building end"] = time.time()
|
|
delta = times["FAS cache building end"] - times["data structure end"]
|
|
print(f"Ran for {delta} seconds -- ie: {delta/60} minutes")
|
|
print("Building bugzilla's products in-memory cache")
|
|
|
|
bugzilla.build_product_cache(self.pagure_projects)
|
|
|
|
if self.env["verbose"]:
|
|
times["BZ cache building end"] = time.time()
|
|
delta = times["BZ cache building end"] - times["FAS cache building end"]
|
|
print(f"Ran for {delta} seconds -- ie: {delta/60} minutes")
|
|
if self.env["dryrun"]:
|
|
print("Querying bugzilla but not doing anything")
|
|
else:
|
|
print("Updating bugzilla")
|
|
|
|
for project in sorted(self.pagure_projects, key=itemgetter('name')):
|
|
for product in project["products"]:
|
|
if product not in self.env['products']:
|
|
if self.env["verbose"]:
|
|
print(f"Ignoring: {product}/{project['name']}")
|
|
continue
|
|
|
|
try:
|
|
bugzilla.add_edit_component(
|
|
package=project["name"],
|
|
collection=product,
|
|
owner=project["products_poc"][product],
|
|
description=project['summary'],
|
|
qacontact=None,
|
|
cclist=project['watchers'],
|
|
print_fas_names=self.args.print_fas_names,
|
|
)
|
|
except ValueError as e:
|
|
# A username didn't have a bugzilla address
|
|
errors.append(str(e.args))
|
|
except DataChangedError as e:
|
|
# A Package or Collection was returned via xmlrpc but wasn't
|
|
# present when we tried to change it
|
|
errors.append(str(e.args))
|
|
except xmlrpc.client.ProtocolError as e:
|
|
# Unrecoverable and likely means that nothing is going to
|
|
# succeed.
|
|
errors.append(str(e.args))
|
|
break
|
|
except xmlrpc.client.Error as e:
|
|
# An error occurred in the xmlrpc call. Shouldn't happen but
|
|
# we better see what it is
|
|
errors.append('%s -- %s' % (project["name"], e.args[-1]))
|
|
|
|
# Send notification of errors
|
|
if errors:
|
|
if self.env["verbose"] or self.env["dryrun"]:
|
|
print('[DEBUG]', '\n'.join(errors))
|
|
else:
|
|
self.notify_users(errors)
|
|
send_email(
|
|
self.env['email_from'],
|
|
self.env['notify_emails'],
|
|
'Errors while syncing bugzilla with the PackageDB',
|
|
self.env['tmpl_admin_email'].format(errors='\n'.join(errors))
|
|
)
|
|
else:
|
|
with open(self.env['data_cache'], 'w') as stream:
|
|
json.dump({}, stream)
|
|
|
|
if self.env["verbose"]:
|
|
times["end"] = time.time()
|
|
|
|
print(" ----------")
|
|
print("Building the data structure")
|
|
delta = times["data structure end"] - times["start"]
|
|
print(f" Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")
|
|
|
|
print("Building the FAS cache")
|
|
delta = times["FAS cache building end"] - times["data structure end"]
|
|
print(f" Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")
|
|
|
|
print("Building the bugzilla cache")
|
|
delta = times["BZ cache building end"] - times["FAS cache building end"]
|
|
print(f" Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")
|
|
|
|
print("Interacting with bugzilla")
|
|
delta = times["end"] - times["BZ cache building end"]
|
|
print(f" Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")
|
|
|
|
print("Total")
|
|
delta = times["end"] - times["start"]
|
|
print(f" Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
DistgitBugzillaSync.main()
|