diff --git a/distgit_bugzilla_sync/script.py b/distgit_bugzilla_sync/script.py index dcd56ba..2b61021 100644 --- a/distgit_bugzilla_sync/script.py +++ b/distgit_bugzilla_sync/script.py @@ -53,10 +53,9 @@ from urllib3.util import Retry import yaml from . import package_summary +from .config import config, email_overrides, load_configuration -env = 'staging' - cache = dogpile.cache.make_region().configure( 'dogpile.cache.memory', expiration_time=3600, @@ -78,123 +77,9 @@ def retry_session(): return session -if env == 'staging': - BZSERVER = 'https://partner-bugzilla.redhat.com' -else: - BZSERVER = 'https://bugzilla.redhat.com' - -BZUSER = '{{ bugzilla_user }}' -BZPASS = '{{ bugzilla_password }}' -BZCOMPAPI = 'component.get' -FASUSER = '{{ fedorathirdpartyUser }}' -FASPASS = '{{ fedorathirdpartyPassword }}' -BUGZILLA_OVERRIDE_REPO = 'releng/fedora-scm-requests' -NOTIFYEMAIL = [ - 'kevin@fedoraproject.org', - 'pingou@fedoraproject.org', -] VERBOSE = False DRYRUN = False -if env == 'staging': - FASURL = 'https://admin.stg.fedoraproject.org/accounts' - FASINSECURE = True - PAGUREURL = 'https://stg.pagure.io' - PAGURE_DIST_GIT_URL = 'https://src.stg.fedoraproject.org' - PDCURL = 'https://pdc.stg.fedoraproject.org/rest_api/v1/' - MDAPIURL = 'https://apps.stg.fedoraproject.org/mdapi/' -else: - FASURL = 'https://admin.fedoraproject.org/accounts' - FASINSECURE = False - PAGUREURL = 'https://pagure.io' - PAGURE_DIST_GIT_URL = 'https://src.fedoraproject.org' - PDCURL = 'https://pdc.fedoraproject.org/rest_api/v1/' - MDAPIURL = 'https://apps.fedoraproject.org/mdapi/' - - -EMAIL_FROM = 'accounts@fedoraproject.org' -DATA_CACHE = '/var/tmp/pagure_sync_bz.json' - -PRODUCTS = { - 'Fedora': 'Fedora', - 'Fedora Container': 'Fedora Container Images', - 'Fedora Modules': 'Fedora Modules', - 'Fedora EPEL': 'Fedora EPEL', -} - -NAMESPACE_TO_PRODUCT = { - 'rpms': 'Fedora', # except EPEL... - 'container': 'Fedora Container', - 'modules': 'Fedora Modules', -} - -# This maps bugzilla products to "lead" branches in PDC. If the lead branch is -# retired, then we in turn set the default assignee to "orphan" for all new bugs -# in the given product. -PRODUCTS_TO_LEAD_BRANCH = { - # If rawhide is retired, then all new bugs go to orphan for Fedora. - 'Fedora': 'master', - # Same for containers. - 'Fedora Container': 'master', - # Same for modules. - 'Fedora Modules': 'master', - # If epel7 is retired, then all new epel bugs go to orphan. - 'Fedora EPEL': 'epel7', -} -PDC_TYPES = { - 'rpms': 'rpm', - 'modules': 'module', - 'container': 'container', -} -INVERSE_PDC_TYPES = {v: k for k, v in PDC_TYPES.items()} - - -# When querying for current info, take segments of 1000 packages a time -BZ_PKG_SEGMENT = 1000 - - -TMPL_EMAIL_ADMIN = ''' -The following errors were encountered while updating bugzilla with information -from the Package Database. Please have the problems taken care of: - -%s -''' - -# PkgDB sync bugzilla email -PKGDB_SYNC_BUGZILLA_EMAIL = """Greetings. - -You are receiving this email because there's a problem with your -bugzilla.redhat.com account. - -If you recently changed the email address associated with your -Fedora account in the Fedora Account System, it is now out of sync -with your bugzilla.redhat.com account. This leads to problems -with Fedora packages you own or are CC'ed on bug reports for. - -Please take one of the following actions: - -a) login to your old bugzilla.redhat.com account and change the email -address to match your current email in the Fedora account system. -https://bugzilla.redhat.com login, click preferences, account -information and enter new email address. - -b) Create a new account in bugzilla.redhat.com to match your -email listed in your Fedora account system account. -https://bugzilla.redhat.com/ click 'new account' and enter email -address. - -c) Change your Fedora Account System email to match your existing -bugzilla.redhat.com account. -https://admin.fedoraproject.org/accounts login, click on 'my account', -then 'edit' and change your email address. - -If you have questions or concerns, please let us know. - -Your prompt attention in this matter is appreciated. - -The Fedora admins. -""" - def resilient_partial(fn, *initial, **kwargs): """ A decorator that partially applies arguments. @@ -244,20 +129,20 @@ class ProductCache(dict): if key not in self.acls: raise - if BZCOMPAPI == 'getcomponentsdetails': + if env['bugzilla']['compat_api'] == 'getcomponentsdetails': # Old API -- in python-bugzilla. But with current server, this # gives ProxyError products = self.bz.getcomponentsdetails(key) - elif BZCOMPAPI == 'component.get': + elif env['bugzilla']['compat_api'] == 'component.get': # Way that's undocumented in the partner-bugzilla api but works # currently pkglist = list(projects_dict[key]) products = {} - for pkg_segment in segment(pkglist, BZ_PKG_SEGMENT): + for pkg_segment in segment(pkglist, env['bugzilla']['req_segment']): # Format that bugzilla will understand. Strip None's that # segment() pads out the final data segment() with query = [ - dict(product=PRODUCTS[key], component=p) + dict(product=env['products'][key], component=p) for p in pkg_segment if p is not None ] raw_data = self.bz._proxy.Component.get(dict(names=query)) @@ -290,9 +175,9 @@ class BugzillaProxy: # Connect to the fedora account system self.fas = AccountSystem( - base_url=FASURL, - username=FASUSER, - password=FASPASS) + base_url=env['fas']['url'], + username=env['fas']['username'], + password=env['fas']['password']) try: self.userCache = self.fas.people_by_key( @@ -393,7 +278,7 @@ class BugzillaProxy: data['initialowner'] = owner # Changes occurred. Submit a request to change via xmlrpc - data['product'] = PRODUCTS[collection] + data['product'] = env['products'][collection] data['component'] = package if VERBOSE: print('[EDITCOMP] Changing via editComponent(' @@ -421,7 +306,7 @@ class BugzillaProxy: qacontact = 'extras-qa@fedoraproject.org' data = { - 'product': PRODUCTS[collection], + 'product': env['products'][collection], 'component': package, 'description': description or 'NA', 'initialowner': owner, @@ -469,13 +354,13 @@ def notify_users(errors): address, use it to notify the user about the issue. ''' data = {} - if os.path.exists(DATA_CACHE): + if os.path.exists(env['data_cache']): try: - with open(DATA_CACHE) as stream: + with open(env['data_cache']) as stream: data = json.load(stream) except Exception as err: print('Could not read the json file at %s: \nError: %s' % ( - DATA_CACHE, err)) + env['data_cache'], err)) new_data = {} seen = [] @@ -507,25 +392,25 @@ def notify_users(errors): if notify_user: send_email( - EMAIL_FROM, + env['email_from'], [user_email], subject='Please fix your bugzilla.redhat.com account', - message=PKGDB_SYNC_BUGZILLA_EMAIL, - ccAddress=NOTIFYEMAIL, + message=env['tmpl_user_email'], + ccAddress=env['notify_emails'], ) new_data[user_email] = { 'last_update': time.mktime(now.timetuple()) } - with open(DATA_CACHE, 'w') as stream: + with open(env['data_cache'], 'w') as stream: json.dump(new_data, stream) @cache.cache_on_arguments() def _get_override_yaml(project, session): pagure_override_url = '{0}/{1}/raw/master/f/{2}/{3}'.format( - PAGUREURL.rstrip('/'), BUGZILLA_OVERRIDE_REPO, project['namespace'], + env['pagure_url'].rstrip('/'), env['bugzilla']['override_repo'], project['namespace'], project['name']) if VERBOSE: @@ -543,10 +428,10 @@ def _get_pdc_branches(session, repo): :param repo: the project dict :return: a list of the repo's branches """ - branches_url = '{0}component-branches/'.format(PDCURL) + branches_url = '{0}component-branches/'.format(env['pdc_url']) params = dict( global_component=repo['name'], - type=PDC_TYPES[repo['namespace']] + type=env['pdc_types'][repo['namespace']] ) if VERBOSE: print('Querying {0} {1}'.format(branches_url, params)) @@ -634,7 +519,7 @@ def _to_legacy_schema(product_and_project_and_summary, session=None): def main(): """The entrypoint to the script.""" - global VERBOSE, DRYRUN, projects_dict + global envname, env, VERBOSE, DRYRUN, projects_dict start = time.time() parser = argparse.ArgumentParser( @@ -652,6 +537,11 @@ def main(): args = parser.parse_args() + load_configuration() + + envname = config['environment'] + env = config['environments'][envname] + if args.debug: VERBOSE = True DRYRUN = True @@ -677,11 +567,11 @@ def main(): # Get the initial ownership and CC data from pagure # This part is easy. - poc_url = PAGURE_DIST_GIT_URL + '/extras/pagure_poc.json' + poc_url = env['distgit_url'] + '/extras/pagure_poc.json' if VERBOSE: print("Querying %r for points of contact." % poc_url) pagure_namespace_to_poc = session.get(poc_url, timeout=120).json() - cc_url = PAGURE_DIST_GIT_URL + '/extras/pagure_bz.json' + cc_url = env['distgit_url'] + '/extras/pagure_bz.json' if VERBOSE: print("Querying %r for initial cc list." % cc_url) pagure_namespace_to_cc = session.get(cc_url, timeout=120).json() @@ -707,18 +597,18 @@ def main(): p for p in pagure_projects if p['namespace'] != 'modules' ] - branches_url = PDCURL.split('rest_api')[0] + 'extras/active_branches.json' + branches_url = env['pdc_url'].split('rest_api')[0] + 'extras/active_branches.json' if VERBOSE: print("Querying %r for EOL information." % branches_url) pdc_branches = session.get(branches_url, timeout=120).json() for proj in pagure_projects: - if proj['namespace'] not in PDC_TYPES: + if proj['namespace'] not in env['pdc_types']: proj['branches'] = [] if VERBOSE: print('! Namespace {namespace} unknown to PDC, project ' '{namespace}/{name} ignored'.format(**proj)) continue - pdc_type = PDC_TYPES[proj['namespace']] + pdc_type = env['pdc_types'][proj['namespace']] proj['branches'] = pdc_branches.get(pdc_type, {}).get(proj['name'], []) if not proj['branches'] and VERBOSE: print("! No PDC branch found for {namespace}/{name}".format(**proj)) @@ -733,7 +623,7 @@ def main(): if re.match(r'^epel\d+$', branch): products.add('Fedora EPEL') else: - products.add(NAMESPACE_TO_PRODUCT[project['namespace']]) + products.add(env['namespace_to_product'][project['namespace']]) project['products'] = list(products) # Now, we must transform the data we collected into something that PkgDB @@ -751,10 +641,13 @@ def main(): projects_dict[response['product']][response['project']] = response # Initialize the connection to bugzilla - bugzilla = BugzillaProxy(BZSERVER, BZUSER, BZPASS, projects_dict) + bugzilla = BugzillaProxy(env['bugzilla']['url'], + env['bugzilla']['user'], + env['bugzilla']['password'], + projects_dict) for product, pkgs in projects_dict.items(): - if product not in PRODUCTS: + if product not in env['products']: continue for pkgname, pkginfo in sorted(projects_dict[product].items(), key=lambda x: x[0]): @@ -793,12 +686,13 @@ def main(): else: notify_users(errors) send_email( - EMAIL_FROM, - NOTIFYEMAIL, + env['email_from'], + env['notify_emails'], 'Errors while syncing bugzilla with the PackageDB', - TMPL_EMAIL_ADMIN % ('\n'.join(errors),)) + env['tmpl_admin_email'] % ('\n'.join(errors),)) + ) else: - with open(DATA_CACHE, 'w') as stream: + with open(env['data_cache'], 'w') as stream: json.dump({}, stream) if VERBOSE: