Infrastructure/distgit-bugzilla-sync
3
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions
distgit-bugzilla-sync/distgit_bugzilla_sync/script.py
Pierre-Yves Chibon 9283999992 Keep the errors as we are seeing them and print or send a report at the end 
While the script runs, we're keeping in memory a list of all the errors
we have encountered and at the end of the run we send to the admins a
report with all of them, categorized in a way that will hopefully make
it easier to fix for them.

We're also adding an option to preserve the function to send the report
to the admins but disable sending the notifications to the
users/packagers.
This would be useful to start get an idea of the number of people who
would have issues with the script.

Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>
2019-11-25 15:27:45 +01:00

1008 lines
41 KiB
Python
Raw Blame History

# -*- coding: utf-8 -*-
#
# Copyright © 2013-2019 Red Hat, Inc.
#
# This copyrighted material is made available to anyone wishing to use, modify,
# copy, or redistribute it subject to the terms and conditions of the GNU
# General Public License v.2, or (at your option) any later version. This
# program is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY expressed or implied, including the implied warranties of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details. You should have received a copy of the GNU
# General Public License along with this program; if not, write to the Free
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the source
# code or documentation are not subject to the GNU General Public License and
# may only be used or replicated with the express permission of Red Hat, Inc.
#
# Red Hat Author(s): Toshio Kuratomi <tkuratom@redhat.com>
# Author(s): Mike Watters <valholla75@fedoraproject.org>
# Author(s): Pierre-Yves Chibon <pingou@pingoured.fr>
# Author(s): Matt Prahl <mprahl@redhat.com>
# Author(s): Ralph Bean <rbean@redhat.com
# Author(s): Patrick Uiterwijk <puiterwijk@redhat.com>
#
'''
sync information from the Pagure into bugzilla
This ... script takes information about package onwership and imports it
into bugzilla.
'''
import argparse
import collections
import datetime
from email.message import EmailMessage
import itertools
import json
from operator import itemgetter
import os
import re
import smtplib
import sys
import time
import traceback
import xmlrpc.client
from bugzilla import Bugzilla
import dogpile.cache
import fedora.client
from fedora.client.fas2 import AccountSystem
import requests
from requests.adapters import HTTPAdapter
from urllib3.util import Retry
import yaml
from . import package_summaries
from .config import config, email_overrides, load_configuration
cache = dogpile.cache.make_region().configure(
'dogpile.cache.memory',
expiration_time=3600,
)
def retry_session():
session = requests.Session()
retry = Retry(
total=5,
read=5,
connect=5,
backoff_factor=0.3,
status_forcelist=(500, 502, 504),
)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
return session
def resilient_partial(fn, *initial, **kwargs):
""" A decorator that partially applies arguments.
It additionally catches all raised exceptions, prints them, but then returns
None instead of propagating the failures.
This is used to protect functions used in a threadpool. If one fails, we
want to know about it, but we don't want it to kill the whole program. So
catch its error, log it, but proceed.
"""
def wrapper(*additional):
try:
full = initial + additional
return fn(*full, **kwargs)
except Exception:
traceback.print_exc()
return None
wrapper.__name__ = fn.__name__
wrapper.__doc__ = fn.__doc__
return wrapper
class DataChangedError(Exception):
'''Raised when data we are manipulating changes while we're modifying it.'''
pass
def segment(iterable, chunk, fill=None):
'''Collect data into `chunk` sized block'''
args = [iter(iterable)] * chunk
return itertools.zip_longest(*args, fillvalue=fill)
class BugzillaProxy:
def __init__(self, bz_server, username, password, config,
pre_cache_users=True):
self.bz_xmlrpc_server = bz_server
self.username = username
self.password = password
self.server = Bugzilla(
url=self.bz_xmlrpc_server,
user=self.username,
password=self.password)
self.product_cache = {}
self.user_cache = {}
self.inverted_user_cache = {}
self.errors = []
# Connect to the fedora account system
self.fas = AccountSystem(
base_url=config['fas']['url'],
username=config['fas']['username'],
password=config['fas']['password'])
self.config = config
if not pre_cache_users:
return
if config['verbose']:
print("Pre-caching FAS users and their Bugzilla email addresses.")
try:
self.user_cache = self.fas.people_by_key(
key='username',
fields=['bugzilla_email'])
except fedora.client.ServerError:
# Sometimes, building the user cache up front fails with a timeout.
# It's ok, we build the cache as-needed later in the script.
pass
else:
self.invert_user_cache()
def build_product_cache(self, pagure_projects):
""" Cache the bugzilla info about each package in each product.
"""
if self.config['bugzilla']['compat_api'] == 'getcomponentsdetails':
# Old API -- in python-bugzilla. But with current server, this
# gives ProxyError
for collection, product in self.config["products"].items():
self.product_cache[collection] = self.server.getcomponentsdetails(product)
elif self.config['bugzilla']['compat_api'] == 'component.get':
# Way that's undocumented in the partner-bugzilla api but works
# currently
for collection, product in self.config["products"].items():
# restrict the list of info returned to only the packages of
# interest
pkglist = [
project["name"]
for project in pagure_projects
if product in project["products"]
]
products = {}
for pkg_segment in segment(pkglist, self.config['bugzilla']['req_segment']):
# Format that bugzilla will understand. Strip None's that
# segment() pads out the final data segment() with
query = [
dict(
product=self.config['products'][collection],
component=p
)
for p in pkg_segment
if p is not None
]
raw_data = self.server._proxy.Component.get(dict(names=query))
for package in raw_data['components']:
# Reformat data to be the same as what's returned from
# getcomponentsdetails
product = dict(
initialowner=package['default_assignee'],
description=package['description'],
initialqacontact=package['default_qa_contact'],
initialcclist=package['default_cc']
)
products[package['name'].lower()] = product
self.product_cache[collection] = products
def invert_user_cache(self):
""" Takes the user_cache built when querying FAS and invert it so
that the bugzilla_email is the key and the username the value.
"""
for username in self.user_cache:
bz_email = self.user_cache[username]['bugzilla_email'].lower()
self.inverted_user_cache[bz_email] = username
def _get_bugzilla_email(self, username):
'''Return the bugzilla email address for a user.
First looks in a cache for a username => bugzilla email. If not found,
reloads the cache from fas and tries again.
'''
try:
bz_email = self.user_cache[username]['bugzilla_email'].lower()
except KeyError:
if username.startswith('@'):
group = self.fas.group_by_name(username[1:])
bz_email = group.mailing_list
if bz_email is None:
return
self.user_cache[username] = {
'bugzilla_email': bz_email}
else:
person = self.fas.person_by_username(username)
bz_email = person.get('bugzilla_email', None)
if bz_email is None:
return
self.user_cache[username] = {'bugzilla_email': bz_email}
bz_email = bz_email.lower()
self.inverted_user_cache[bz_email] = username
bz_email = email_overrides.get(bz_email, bz_email)
return bz_email
def update_open_bugs(self, new_poc, prev_poc, product, name):
'''Change the package owner
:arg new_poc: email of the new point of contact.
:arg prev_poc: Username of the previous point of contact
:arg product: The product of the package to change in bugzilla
:arg name: Name of the package to change the owner.
'''
bz_query = {}
bz_query['product'] = product
bz_query['component'] = name
bz_query['bug_status'] = [
'NEW', 'ASSIGNED', 'ON_DEV', 'ON_QA', 'MODIFIED', 'POST',
'FAILS_QA', 'PASSES_QA', 'RELEASE_PENDING']
# Update only maintained releases
bz_query['version'] = self.config["products"][product]["versions"]
query_results = self.server.query(bz_query)
for bug in query_results:
if bug.assigned_to == prev_poc and bug.assigned_to != new_poc:
if self.config["verbose"]:
print(
f' - reassigning bug #{bug.bug_id} '
f'from {bug.assigned_to} to {new_poc}'
)
if not self.config["dryrun"]:
try:
bug.setassignee(
assigned_to=new_poc,
comment=self.config['bz_maintainer_change_comment'],
)
except xmlrpc.client.Fault as e:
# Output something useful in args
e.args = (new_poc, e.faultCode, e.faultString)
raise
except xmlrpc.client.ProtocolError as e:
e.args = ('ProtocolError', e.errcode, e.errmsg)
raise
def add_edit_component(self, package, collection, owner, description=None,
qacontact=None, cclist=None, print_fas_names=False,
retired=False):
'''Add or update a component to have the values specified.
'''
# Turn the cclist into something usable by bugzilla
initial_cc_emails = []
initial_cc_fasnames = []
for watcher in cclist:
bz_email = self._get_bugzilla_email(watcher)
if bz_email:
initial_cc_emails.append(bz_email)
initial_cc_fasnames.append(watcher)
else:
self.errors.append(
f"{watcher} has no bugzilla_email or mailing_list set on "
f"({collection}/{package})"
)
if self.config["verbose"]:
print(f"** {watcher} has no bugzilla_email or mailing_list set "
f"({collection}/{package}) **")
# Add owner to the cclist so comaintainers taking over a bug don't
# have to do this manually
owner_email = self._get_bugzilla_email(owner)
if owner_email not in initial_cc_emails:
initial_cc_emails.append(owner_email)
initial_cc_fasnames.append(owner)
# Lookup product
try:
product = self.product_cache[collection]
except xmlrpc.client.Fault as e:
# Output something useful in args
e.args = (e.faultCode, e.faultString)
raise
except xmlrpc.client.ProtocolError as e:
e.args = ('ProtocolError', e.errcode, e.errmsg)
raise
# Set the qacontact_email and name
default_qa_contact_email = self.config['default_qa_contact_email']
default_qa_contact = f"<default: {default_qa_contact_email.split('@', 1)[0]}@...>"
if qacontact:
qacontact_email = self._get_bugzilla_email(qacontact)
else:
qacontact = default_qa_contact
qacontact_email = default_qa_contact_email
pkg_key = package.lower()
if pkg_key in product:
# edit the package information
data = {}
# Check for changes to the owner, qacontact, or description
if product[pkg_key]['initialowner'] != owner_email:
data['initialowner'] = owner_email
if description and product[pkg_key]['description'] != description:
data['description'] = description
if qacontact and product[pkg_key]['initialqacontact'] != qacontact_email:
data['initialqacontact'] = qacontact_email
if len(product[pkg_key]['initialcclist']) != len(initial_cc_emails):
data['initialcclist'] = initial_cc_emails
else:
for cc_member in product[pkg_key]['initialcclist']:
if cc_member not in initial_cc_emails:
data['initialcclist'] = initial_cc_emails
break
data["is_active"] = not retired
if data:
# Changes occurred. Submit a request to change via xmlrpc
data['product'] = self.config['products'][collection]
data['component'] = package
if self.config["verbose"]:
print(f'[EDITCOMP] {data["product"]}/{data["component"]}')
for key in ["initialowner", "description", "initialqacontact", "initialcclist"]:
if data.get(key):
old_value = product[pkg_key][key]
if not isinstance(old_value, str):
old_value = sorted(old_value)
new_value = data.get(key)
if not isinstance(new_value, str):
new_value = sorted(new_value)
if print_fas_names and key in ('initialowner',
'initialqacontact',
'initialcclist'):
if key == 'initialowner':
new_value = owner
elif key == 'initialqacontact':
new_value = qacontact
else:
new_value = sorted(initial_cc_fasnames)
from_fas_names = []
for email in product[pkg_key][key]:
if email in self.inverted_user_cache:
from_fas_names.append(self.inverted_user_cache[email])
elif email == default_qa_contact_email:
from_fas_names.append(default_qa_contact)
if from_fas_names:
if len(from_fas_names) < len(product[pkg_key][key]):
x = len(product[pkg_key][key]) - len(from_fas_names)
from_fas_names.append(f"And {x} more")
old_value = f"from `{from_fas_names}`"
else:
old_value = ""
print(
f" {key} changed {old_value}"
f" to FAS name(s) `{new_value}`")
else:
print(f" {key} changed from `{old_value}` to `{new_value}`")
owner_changed = "initialowner" in data
# FIXME: initialowner has been made mandatory for some
# reason. Asking dkl why.
data['initialowner'] = owner_email
if not self.config["dryrun"]:
try:
self.server.editcomponent(data)
except xmlrpc.client.Fault as e:
# Output something useful in args
e.args = (data, e.faultCode, e.faultString)
raise
except xmlrpc.client.ProtocolError as e:
e.args = ('ProtocolError', e.errcode, e.errmsg)
raise
if owner_changed:
self.update_open_bugs(
new_poc=owner_email,
prev_poc=product[pkg_key]['initialowner'],
name=package,
product=self.config['products'][collection],
)
else:
if self.config.get("print-no-change"):
bz_product_name = self.config['products'][collection].get(
'bz_product_name', collection
)
print(f"[NOCHANGE] {package}/{bz_product_name}")
else:
bz_product_name = self.config['products'][collection].get('bz_product_name', collection)
if retired:
if self.config['verbose']:
print(f"[NOADD] {bz_product_name}/{package} (is retired)")
return
# Add component
data = {
'product': bz_product_name,
'component': package,
'description': description or 'NA',
'initialowner': owner_email,
'initialqacontact': qacontact_email,
'is_active': not retired,
}
if initial_cc_emails:
data['initialcclist'] = initial_cc_emails
if self.config["verbose"]:
print(f"[ADDCOMP] {bz_product_name}/{package}")
for key in ["initialowner", "description", "initialqacontact",
"initialcclist", "is_active"]:
if print_fas_names and key in ('initialowner',
'initialqacontact',
'initialcclist'):
if key == 'initialowner':
value = owner
elif key == 'initialqacontact':
value = qacontact
else:
value = initial_cc_fasnames
print(f" {key} set to FAS name(s) `{value}`")
else:
print(f" {key} set to {data.get(key)}")
if not self.config["dryrun"]:
try:
self.server.addcomponent(data)
except xmlrpc.client.Fault as e:
# Output something useful in args
e.args = (data, e.faultCode, e.faultString)
raise
def _get_pdc_branches(session, repo):
"""
Gets the branches on a project. This function is used for mapping.
:param repo: the project dict
:return: a list of the repo's branches
"""
branches_url = '{0}component-branches/'.format(env['pdc_url'])
params = dict(
global_component=repo['name'],
type=env['pdc_types'][repo['namespace']]
)
if config["verbose"]:
print('Querying {0} {1}'.format(branches_url, params))
rv = session.get(branches_url, params=params, timeout=60)
# If the project's branches can't be reported, just return no branches and
# it will be skipped later on
if not rv.ok:
print(('The connection to "{0}" failed with the status code {1} and '
'output "{2}"'.format(branches_url, rv.status_code, rv.text)),
file=sys.stderr)
return []
data = rv.json()
return [branch['name'] for branch in data['results']]
class ScriptExecError(RuntimeError):
def __init__(self, *args, **kwargs):
self.errorcode = kwargs.pop('errorcode', 1)
super().__init__(*args, **kwargs)
class DistgitBugzillaSync:
# cache placeholders for properties which are computed once
_namespace_to_product = None
_product_to_branch_regex = None
_branch_regex_to_product = None
errors = collections.defaultdict(list)
def send_email(self, from_address, to_address, subject, message, cc_address=None):
'''Send an email if there's an error.
This will be replaced by sending messages to a log later.
'''
if not self.env['email']['send_mails']:
return
msg = EmailMessage()
msg.add_header('To', ','.join(to_address))
msg.add_header('From', from_address)
msg.add_header('Subject', subject)
if cc_address is not None:
msg.add_header('Cc', ','.join(cc_address))
to_address += cc_address
msg.set_payload(message)
smtp = smtplib.SMTP(self.env['email']['smtp_host'])
smtp.sendmail(from_address, to_address, msg.as_string())
smtp.quit()
def notify_users(self, errors):
''' Browse the list of errors and when we can retrieve the email
address, use it to notify the user about the issue.
'''
data = {}
if os.path.exists(self.env['data_cache']):
try:
with open(self.env['data_cache']) as stream:
data = json.load(stream)
except Exception as err:
print('Could not read the json file at %s: \nError: %s' % (
env['data_cache'], err))
new_data = {}
seen = []
for error in errors:
notify_user = False
if 'The name ' in error and ' is not a valid username' in error:
user_email = error.split(' is not a valid username')[0].split(
'The name ')[1].strip()
now = datetime.datetime.utcnow()
# See if we already know about this user
if user_email in data and data[user_email]['last_update']:
last_update = datetime.datetime.fromtimestamp(
int(data[user_email]['last_update']))
# Only notify users once per hour
if (now - last_update).seconds >= 3600:
notify_user = True
else:
new_data[user_email] = data[user_email]
elif not data or user_email not in data:
notify_user = True
# Ensure we notify the user only once, no matter how many errors we
# got concerning them.
if user_email not in seen:
seen.append(user_email)
else:
notify_user = False
if notify_user:
self.send_email(
self.env['email']['from'],
[user_email],
subject='Please fix your bugzilla.redhat.com account',
message=self.env['email']['templates']['user_notification'],
cc_address=self.env['email']['notify_admins'],
)
new_data[user_email] = {
'last_update': time.mktime(now.timetuple())
}
with open(env['data_cache'], 'w') as stream:
json.dump(new_data, stream)
def get_cli_arguments(self):
""" Set the CLI argument parser and return the argument parsed.
"""
parser = argparse.ArgumentParser(
description='Script syncing information between Pagure and bugzilla'
)
parser.add_argument(
'--dry-run', dest='dryrun', action='store_true', default=False,
help='Do not actually make any changes - Overrides the configuration')
parser.add_argument(
'--verbose', dest='verbose', action='store_true', default=False,
help='Print actions verbosely - Overrides the configuration')
parser.add_argument(
'--debug', dest='debug', action='store_true', default=False,
help='Combination of --verbose and --dry-run')
parser.add_argument(
'--env', dest='env',
help='Run the script for a specific environment, overrides the one '
'set in the configuration file')
parser.add_argument(
'--add-config-file', metavar='CONFIG_FILE',
dest='addl_config_files', action='append',
help="File(s) from which to read overriding configuration")
parser.add_argument(
'--add-email-overrides-file', metavar='EMAIL_OVERRIDES_FILE',
dest='addl_email_overrides_files', action='append',
help="File(s) from which to read additional email overrides")
parser.add_argument(
'-p', '--project', dest='projects', nargs='+',
help='Update one or more projects (provided as namespace/name), '
'in all of its products')
parser.add_argument(
'--print-fas-names', action='store_true', default=False,
help="Print FAS names rather than email addresses in output, useful when pasting into "
"public fora")
parser.add_argument(
'--print-no-change', action='store_true', default=False,
help="Print elements that are not being changed as they are checked")
parser.add_argument(
'--no-user-notifications', dest="user_notifications", action='store_false',
default=True,
help="Do not notify every packager whose account is wrongly set-up, but do send the "
"overall report to the admins")
self.args = parser.parse_args()
def get_pagure_projects(self, project_list=None):
""" Builds a large list of all the projects on pagure.
Each item in that list is a dict containing:
- the namespace of the project
- the name of the project
- the point of contact of this project (ie: the default assignee
in bugzilla)
- the watchers of this project (ie: the initial CC list in bugzilla)
"""
# Get the initial ownership and CC data from pagure
# This part is easy.
poc_url = self.env['distgit_url'] + '/extras/pagure_poc.json'
if self.env["verbose"]:
print("Querying %r for points of contact." % poc_url)
pagure_namespace_to_poc = self.session.get(poc_url, timeout=120).json()
cc_url = self.env['distgit_url'] + '/extras/pagure_bz.json'
if self.env["verbose"]:
print("Querying %r for initial cc list." % cc_url)
pagure_namespace_to_cc = self.session.get(cc_url, timeout=120).json()
# Combine and collapse those two into a single list:
self.pagure_projects = []
if project_list:
project_list = set(tuple(p.split("/", 1)) for p in project_list)
for namespace, entries in pagure_namespace_to_poc.items():
for name, poc in entries.items():
if not project_list or (namespace, name) in project_list:
self.pagure_projects.append(dict(
namespace=namespace,
name=name,
poc=poc,
watchers=pagure_namespace_to_cc[namespace][name],
))
@property
def namespace_to_product(self):
if self._namespace_to_product is None:
self._namespace_to_product = {
p['namespace']: n
for n, p in self.env['products'].items() if 'namespace' in p
}
return self._namespace_to_product
@property
def product_to_branch_regex(self):
if self._product_to_branch_regex is None:
self._product_to_branch_regex = {
n: re.compile(p['branch_regex'])
for n, p in self.env['products'].items() if 'branch_regex' in p
}
return self._product_to_branch_regex
@property
def branch_regex_to_product(self):
if self._branch_regex_to_product is None:
self._branch_regex_to_product = {n: r for r, n in self.product_to_branch_regex.items()}
return self._branch_regex_to_product
def _is_retired(self, product, project):
branches = project['branches']
branch_regex = self.product_to_branch_regex.get(product)
if branch_regex:
for branch, active in branches:
if branch_regex.match(branch) and active:
return False
# No active branches means it is retired.
return True
else:
for branch, active in branches:
if active:
return False
return True
def add_branches_products_and_summaries(self):
""" For each project retrieved, this method adds branches, products
and summary information.
The branches are retrieved from PDC.
The products are determined based on the branches.
The summaries are coming from the primary.xml file of the Rawhide repodata
in Koji.
"""
branches_url = "/".join([
self.env['pdc_url'].split('rest_api')[0].rstrip("/"),
'extras/active_branches.json',
])
if self.env["verbose"]:
print("Querying %r for EOL information." % branches_url)
pdc_branches = self.session.get(branches_url, timeout=120).json()
for idx, project in enumerate(self.pagure_projects):
# Summary
summary = None
if project["namespace"] == "rpms":
summary = self.rpm_summaries.get(project["name"])
project["summary"] = summary
# Branches
if project['namespace'] not in self.env['pdc_types']:
project['branches'] = []
project['products'] = []
self.errors["configuration"].append(
f'Namespace `{project["namespace"]}` not found in the pdc_type '
f'configuration key, project {project["namespace"]}/{project["name"]} '
'ignored'
)
continue
pdc_type = self.env['pdc_types'][project['namespace']]
project['branches'] = pdc_branches.get(pdc_type, {}).get(project['name'], [])
if not project['branches']:
self.errors["PDC"].append(
f"No PDC branch found for {project['namespace']}/{project['name']}")
# Products
products = set()
for branch, active in project.get('branches'):
for regex, product in self.branch_regex_to_product.items():
if regex.match(branch):
products.add(product)
break
else:
products.add(self.namespace_to_product[project['namespace']])
project['products'] = list(products)
products_poc = {}
products_retired = {}
for product in products:
owner = project["poc"]
# Check if the project is retired in PDC, and if so set assignee to orphan.
if self._is_retired(product, project):
owner = 'orphan'
products_retired[product] = True
else:
products_retired[product] = False
# Check if the Bugzilla ticket assignee has been manually overridden
override_yaml = self._get_override_yaml(project, self.session)
if override_yaml.get(product) \
and isinstance(override_yaml[product], str):
owner = override_yaml[product]
products_poc[product] = owner
project['products_poc'] = products_poc
project["products_retired"] = products_retired
self.pagure_projects[idx] = project
@cache.cache_on_arguments()
def _get_override_yaml(self, project, session):
pagure_override_url = '{0}/{1}/raw/master/f/{2}/{3}'.format(
self.env['pagure_url'].rstrip('/'),
self.env['bugzilla']['override_repo'],
project['namespace'],
project['name'],
)
if self.env["verbose"]:
print('Querying {0}'.format(pagure_override_url))
override_rv = session.get(pagure_override_url, timeout=30)
output = {}
if override_rv.status_code == 200:
try:
override_yaml = yaml.safe_load(override_rv.text)
output = override_yaml.get('bugzilla_contact', {})
except yaml.YAMLError:
self.errors["SCM overrides"].append(
f"Failed to load yaml file at: {pagure_override_url}")
except AttributeError:
self.errors["SCM overrides"].append(
f"Invalid yaml file at: {pagure_override_url}")
return output
@classmethod
def main(cls):
"""The entrypoint for running the script."""
dbs = cls()
try:
dbs.run()
except ScriptExecError as e:
print(str(e), file=sys.stderr)
sys.exit(e.errorcode)
else:
sys.exit(0)
def run(self):
"""Run the script."""
global envname, env, projects_dict
times = {
"start": time.time(),
}
self.get_cli_arguments()
load_configuration(addl_config_files=self.args.addl_config_files,
addl_email_overrides_files=self.args.addl_email_overrides_files)
self.config = config
envname = self.config['environment']
if self.args.env:
if self.args.env in self.config['environments']:
envname = self.args.env
else:
raise ScriptExecError(f"Invalid environment specified: {self.args.env}")
self.env = self.config['environments'][envname]
if self.args.debug:
self.env["verbose"] = True
self.env["dryrun"] = True
if self.args.verbose:
self.env["verbose"] = True
if self.args.dryrun:
self.env["dryrun"] = True
self.env["print-no-change"] = self.args.print_no_change
# Non-fatal errors to alert people about
errors = []
self.session = retry_session()
if self.env["verbose"]:
print("Building a cache of the rpm package summaries")
self.rpm_summaries = package_summaries.get_package_summaries()
self.get_pagure_projects(self.args.projects)
self.add_branches_products_and_summaries()
if self.env["verbose"]:
print(f"{len(self.pagure_projects)} projects to consider")
if not self.pagure_projects:
return
times["data structure end"] = time.time()
# Initialize the connection to bugzilla
bugzilla = BugzillaProxy(
bz_server=self.env['bugzilla']['url'],
username=self.env['bugzilla']['user'],
password=self.env['bugzilla']['password'],
config=self.env,
pre_cache_users=not self.args.projects or self.args.print_fas_names,
)
times["FAS cache building end"] = time.time()
if self.env["verbose"]:
print("Building bugzilla's products in-memory cache")
bugzilla.build_product_cache(self.pagure_projects)
times["BZ cache building end"] = time.time()
if self.env["verbose"]:
if self.env["dryrun"]:
print("Querying bugzilla but not doing anything")
else:
print("Updating bugzilla")
for project in sorted(self.pagure_projects, key=itemgetter('name')):
for product in project["products"]:
if product not in self.env['products']:
if self.env["verbose"]:
print(f"Ignoring: {product}/{project['name']}")
continue
try:
bugzilla.add_edit_component(
package=project["name"],
collection=product,
owner=project["products_poc"][product],
description=project['summary'],
qacontact=None,
cclist=project['watchers'],
print_fas_names=self.args.print_fas_names,
retired=project["products_retired"][product],
)
except ValueError as e:
# A username didn't have a bugzilla address
self.errors["bugzilla_raw"].append(str(e.args))
self.errors["bugzilla"].append(
f"Failed to update: `{product}/{project['name']}`:"
f"\n {e}"
f"\n {e.args}"
)
except DataChangedError as e:
# A Package or Collection was returned via xmlrpc but wasn't
# present when we tried to change it
self.errors["bugzilla_raw"].append(str(e.args))
self.errors["bugzilla"].append(
f"Failed to update: `{product}/{project['name']}`: "
f"\n {e}"
f"\n {e.args}"
)
except xmlrpc.client.ProtocolError as e:
# Unrecoverable and likely means that nothing is going to
# succeed.
self.errors["bugzilla_raw"].append(str(e.args))
self.errors["bugzilla"].append(
f"Failed to update: `{product}/{project['name']}`: "
f"\n {e}"
f"\n {e.args}"
)
break
except xmlrpc.client.Error as e:
# An error occurred in the xmlrpc call. Shouldn't happen but
# we better see what it is
self.errors["bugzilla_raw"].append('%s -- %s' % (project["name"], e.args[-1]))
self.errors["bugzilla"].append(
f"Failed to update: `{product}/{project['name']}`: "
f"\n {e}"
f"\n {e.args}"
)
# Send notification of errors
if self.errors:
if not self.env["dryrun"] and self.args.user_notifications:
self.notify_users(self.errors["bugzilla"])
# Build the report for the admins
report = ["ERROR REPORT"]
for key in ["configuration", "PDC", "SCM overrides", "bugzilla"]:
if self.errors[key]:
report.append(key)
report.append(' - {}'.format("\n - ".join(self.errors[key])))
report.append('')
if self.env["verbose"] or self.env["dryrun"]:
print("*" * 80)
print('\n'.join(report))
else:
self.send_email(
self.env['email']['from'],
self.env['email']['notify_admins'],
'Errors while syncing bugzilla with the PackageDB',
self.env['email']['templates']['admin_notification'].format(
errors='\n'.join(report)
)
)
else:
with open(self.env['data_cache'], 'w') as stream:
json.dump({}, stream)
if self.env["verbose"]:
times["end"] = time.time()
print(" ----------")
print("Building the data structure")
delta = times["data structure end"] - times["start"]
print(f" Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")
print("Building the FAS cache")
delta = times["FAS cache building end"] - times["data structure end"]
print(f" Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")
print("Building the bugzilla cache")
delta = times["BZ cache building end"] - times["FAS cache building end"]
print(f" Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")
print("Interacting with bugzilla")
delta = times["end"] - times["BZ cache building end"]
print(f" Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")
print("Total")
delta = times["end"] - times["start"]
print(f" Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")
if __name__ == '__main__':
DistgitBugzillaSync.main()
Reference in a new issue View git blame Copy permalink