From 928399999281a765fe5c2e0ca0da00ef04e49520 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mon, 25 Nov 2019 13:00:48 +0100 Subject: [PATCH] Keep the errors as we are seeing them and print or send a report at the end While the script runs, we're keeping in memory a list of all the errors we have encountered and at the end of the run we send to the admins a report with all of them, categorized in a way that will hopefully make it easier to fix for them. We're also adding an option to preserve the function to send the report to the admins but disable sending the notifications to the users/packagers. This would be useful to start get an idea of the number of people who would have issues with the script. Signed-off-by: Pierre-Yves Chibon --- distgit_bugzilla_sync/script.py | 93 +++++++++++++++++++++++++-------- 1 file changed, 72 insertions(+), 21 deletions(-) diff --git a/distgit_bugzilla_sync/script.py b/distgit_bugzilla_sync/script.py index cb3d66a..7789392 100644 --- a/distgit_bugzilla_sync/script.py +++ b/distgit_bugzilla_sync/script.py @@ -30,6 +30,7 @@ into bugzilla. ''' import argparse +import collections import datetime from email.message import EmailMessage import itertools @@ -126,6 +127,7 @@ class BugzillaProxy: self.product_cache = {} self.user_cache = {} self.inverted_user_cache = {} + self.errors = [] # Connect to the fedora account system self.fas = AccountSystem( @@ -286,8 +288,13 @@ class BugzillaProxy: initial_cc_emails.append(bz_email) initial_cc_fasnames.append(watcher) else: - print(f"** {watcher} has no bugzilla_email or mailing_list set " - f"({collection}/{package}) **") + self.errors.append( + f"{watcher} has no bugzilla_email or mailing_list set on " + f"({collection}/{package})" + ) + if self.config["verbose"]: + print(f"** {watcher} has no bugzilla_email or mailing_list set " + f"({collection}/{package}) **") # Add owner to the cclist so comaintainers taking over a bug don't # have to do this manually @@ -499,6 +506,7 @@ class DistgitBugzillaSync: _namespace_to_product = None _product_to_branch_regex = None _branch_regex_to_product = None + errors = collections.defaultdict(list) def send_email(self, from_address, to_address, subject, message, cc_address=None): '''Send an email if there's an error. @@ -615,6 +623,11 @@ class DistgitBugzillaSync: parser.add_argument( '--print-no-change', action='store_true', default=False, help="Print elements that are not being changed as they are checked") + parser.add_argument( + '--no-user-notifications', dest="user_notifications", action='store_false', + default=True, + help="Do not notify every packager whose account is wrongly set-up, but do send the " + "overall report to the admins") self.args = parser.parse_args() @@ -723,19 +736,18 @@ class DistgitBugzillaSync: if project['namespace'] not in self.env['pdc_types']: project['branches'] = [] project['products'] = [] - if self.env["verbose"]: - print( - f'! Namespace {project["namespace"]} not found in the pdc_type ' - f'configuration key, project {project["namespace"]}/{project["name"]} ' - 'ignored' - ) + self.errors["configuration"].append( + f'Namespace `{project["namespace"]}` not found in the pdc_type ' + f'configuration key, project {project["namespace"]}/{project["name"]} ' + 'ignored' + ) continue pdc_type = self.env['pdc_types'][project['namespace']] project['branches'] = pdc_branches.get(pdc_type, {}).get(project['name'], []) if not project['branches']: - if self.env["verbose"]: - print(f"! No PDC branch found for {project['namespace']}/{project['name']}") + self.errors["PDC"].append( + f"No PDC branch found for {project['namespace']}/{project['name']}") # Products products = set() @@ -782,10 +794,18 @@ class DistgitBugzillaSync: if self.env["verbose"]: print('Querying {0}'.format(pagure_override_url)) override_rv = session.get(pagure_override_url, timeout=30) + output = {} if override_rv.status_code == 200: - override_yaml = yaml.safe_load(override_rv.text) - return override_yaml.get('bugzilla_contact', {}) - return {} + try: + override_yaml = yaml.safe_load(override_rv.text) + output = override_yaml.get('bugzilla_contact', {}) + except yaml.YAMLError: + self.errors["SCM overrides"].append( + f"Failed to load yaml file at: {pagure_override_url}") + except AttributeError: + self.errors["SCM overrides"].append( + f"Invalid yaml file at: {pagure_override_url}") + return output @classmethod def main(cls): @@ -895,33 +915,64 @@ class DistgitBugzillaSync: ) except ValueError as e: # A username didn't have a bugzilla address - errors.append(str(e.args)) + self.errors["bugzilla_raw"].append(str(e.args)) + self.errors["bugzilla"].append( + f"Failed to update: `{product}/{project['name']}`:" + f"\n {e}" + f"\n {e.args}" + ) except DataChangedError as e: # A Package or Collection was returned via xmlrpc but wasn't # present when we tried to change it - errors.append(str(e.args)) + self.errors["bugzilla_raw"].append(str(e.args)) + self.errors["bugzilla"].append( + f"Failed to update: `{product}/{project['name']}`: " + f"\n {e}" + f"\n {e.args}" + ) except xmlrpc.client.ProtocolError as e: # Unrecoverable and likely means that nothing is going to # succeed. - errors.append(str(e.args)) + self.errors["bugzilla_raw"].append(str(e.args)) + self.errors["bugzilla"].append( + f"Failed to update: `{product}/{project['name']}`: " + f"\n {e}" + f"\n {e.args}" + ) break except xmlrpc.client.Error as e: # An error occurred in the xmlrpc call. Shouldn't happen but # we better see what it is - errors.append('%s -- %s' % (project["name"], e.args[-1])) + self.errors["bugzilla_raw"].append('%s -- %s' % (project["name"], e.args[-1])) + self.errors["bugzilla"].append( + f"Failed to update: `{product}/{project['name']}`: " + f"\n {e}" + f"\n {e.args}" + ) # Send notification of errors - if errors: + if self.errors: + if not self.env["dryrun"] and self.args.user_notifications: + self.notify_users(self.errors["bugzilla"]) + + # Build the report for the admins + report = ["ERROR REPORT"] + for key in ["configuration", "PDC", "SCM overrides", "bugzilla"]: + if self.errors[key]: + report.append(key) + report.append(' - {}'.format("\n - ".join(self.errors[key]))) + report.append('') + if self.env["verbose"] or self.env["dryrun"]: - print('[DEBUG]', '\n'.join(errors)) + print("*" * 80) + print('\n'.join(report)) else: - self.notify_users(errors) self.send_email( self.env['email']['from'], self.env['email']['notify_admins'], 'Errors while syncing bugzilla with the PackageDB', self.env['email']['templates']['admin_notification'].format( - errors='\n'.join(errors) + errors='\n'.join(report) ) ) else: