use configuration from files

Signed-off-by: Nils Philippsen <nils@redhat.com>

distgit_bugzilla_sync/script.py

@@ -53,10 +53,9 @@ from urllib3.util import Retry
 import yaml
 
 from . import package_summary
+from .config import config, email_overrides, load_configuration
 
 
-env = 'staging'
-
 cache = dogpile.cache.make_region().configure(
     'dogpile.cache.memory',
     expiration_time=3600,
@@ -78,123 +77,9 @@ def retry_session():
     return session
 
 
-if env == 'staging':
-    BZSERVER = 'https://partner-bugzilla.redhat.com'
-else:
-    BZSERVER = 'https://bugzilla.redhat.com'
-
-BZUSER = '{{ bugzilla_user }}'
-BZPASS = '{{ bugzilla_password }}'
-BZCOMPAPI = 'component.get'
-FASUSER = '{{ fedorathirdpartyUser }}'
-FASPASS = '{{ fedorathirdpartyPassword }}'
-BUGZILLA_OVERRIDE_REPO = 'releng/fedora-scm-requests'
-NOTIFYEMAIL = [
-    'kevin@fedoraproject.org',
-    'pingou@fedoraproject.org',
-]
 VERBOSE = False
 DRYRUN = False
 
-if env == 'staging':
-    FASURL = 'https://admin.stg.fedoraproject.org/accounts'
-    FASINSECURE = True
-    PAGUREURL = 'https://stg.pagure.io'
-    PAGURE_DIST_GIT_URL = 'https://src.stg.fedoraproject.org'
-    PDCURL = 'https://pdc.stg.fedoraproject.org/rest_api/v1/'
-    MDAPIURL = 'https://apps.stg.fedoraproject.org/mdapi/'
-else:
-    FASURL = 'https://admin.fedoraproject.org/accounts'
-    FASINSECURE = False
-    PAGUREURL = 'https://pagure.io'
-    PAGURE_DIST_GIT_URL = 'https://src.fedoraproject.org'
-    PDCURL = 'https://pdc.fedoraproject.org/rest_api/v1/'
-    MDAPIURL = 'https://apps.fedoraproject.org/mdapi/'
-
-
-EMAIL_FROM = 'accounts@fedoraproject.org'
-DATA_CACHE = '/var/tmp/pagure_sync_bz.json'
-
-PRODUCTS = {
-    'Fedora': 'Fedora',
-    'Fedora Container': 'Fedora Container Images',
-    'Fedora Modules': 'Fedora Modules',
-    'Fedora EPEL': 'Fedora EPEL',
-}
-
-NAMESPACE_TO_PRODUCT = {
-    'rpms': 'Fedora',  # except EPEL...
-    'container': 'Fedora Container',
-    'modules': 'Fedora Modules',
-}
-
-# This maps bugzilla products to "lead" branches in PDC.  If the lead branch is
-# retired, then we in turn set the default assignee to "orphan" for all new bugs
-# in the given product.
-PRODUCTS_TO_LEAD_BRANCH = {
-    # If rawhide is retired, then all new bugs go to orphan for Fedora.
-    'Fedora': 'master',
-    # Same for containers.
-    'Fedora Container': 'master',
-    # Same for modules.
-    'Fedora Modules': 'master',
-    # If epel7 is retired, then all new epel bugs go to orphan.
-    'Fedora EPEL': 'epel7',
-}
-PDC_TYPES = {
-    'rpms': 'rpm',
-    'modules': 'module',
-    'container': 'container',
-}
-INVERSE_PDC_TYPES = {v: k for k, v in PDC_TYPES.items()}
-
-
-# When querying for current info, take segments of 1000 packages a time
-BZ_PKG_SEGMENT = 1000
-
-
-TMPL_EMAIL_ADMIN = '''
-The following errors were encountered while updating bugzilla with information
-from the Package Database.  Please have the problems taken care of:
-
-%s
-'''
-
-# PkgDB sync bugzilla email
-PKGDB_SYNC_BUGZILLA_EMAIL = """Greetings.
-
-You are receiving this email because there's a problem with your
-bugzilla.redhat.com account.
-
-If you recently changed the email address associated with your
-Fedora account in the Fedora Account System, it is now out of sync
-with your bugzilla.redhat.com account. This leads to problems
-with Fedora packages you own or are CC'ed on bug reports for.
-
-Please take one of the following actions:
-
-a) login to your old bugzilla.redhat.com account and change the email
-address to match your current email in the Fedora account system.
-https://bugzilla.redhat.com login, click preferences, account
-information and enter new email address.
-
-b) Create a new account in bugzilla.redhat.com to match your
-email listed in your Fedora account system account.
-https://bugzilla.redhat.com/ click 'new account' and enter email
-address.
-
-c) Change your Fedora Account System email to match your existing
-bugzilla.redhat.com account.
-https://admin.fedoraproject.org/accounts login, click on 'my account',
-then 'edit' and change your email address.
-
-If you have questions or concerns, please let us know.
-
-Your prompt attention in this matter is appreciated.
-
-The Fedora admins.
-"""
-
 
 def resilient_partial(fn, *initial, **kwargs):
     """ A decorator that partially applies arguments.
@@ -244,20 +129,20 @@ class ProductCache(dict):
             if key not in self.acls:
                 raise
 
-        if BZCOMPAPI == 'getcomponentsdetails':
+        if env['bugzilla']['compat_api'] == 'getcomponentsdetails':
             # Old API -- in python-bugzilla.  But with current server, this
             # gives ProxyError
             products = self.bz.getcomponentsdetails(key)
-        elif BZCOMPAPI == 'component.get':
+        elif env['bugzilla']['compat_api'] == 'component.get':
             # Way that's undocumented in the partner-bugzilla api but works
             # currently
             pkglist = list(projects_dict[key])
             products = {}
-            for pkg_segment in segment(pkglist, BZ_PKG_SEGMENT):
+            for pkg_segment in segment(pkglist, env['bugzilla']['req_segment']):
                 # Format that bugzilla will understand.  Strip None's that
                 # segment() pads out the final data segment() with
                 query = [
-                    dict(product=PRODUCTS[key], component=p)
+                    dict(product=env['products'][key], component=p)
                     for p in pkg_segment if p is not None
                 ]
                 raw_data = self.bz._proxy.Component.get(dict(names=query))
@@ -290,9 +175,9 @@ class BugzillaProxy:
 
         # Connect to the fedora account system
         self.fas = AccountSystem(
-            base_url=FASURL,
+            base_url=env['fas']['url'],
-            username=FASUSER,
+            username=env['fas']['username'],
-            password=FASPASS)
+            password=env['fas']['password'])
 
         try:
             self.userCache = self.fas.people_by_key(
@@ -393,7 +278,7 @@ class BugzillaProxy:
                 data['initialowner'] = owner
 
                 # Changes occurred.  Submit a request to change via xmlrpc
-                data['product'] = PRODUCTS[collection]
+                data['product'] = env['products'][collection]
                 data['component'] = package
                 if VERBOSE:
                     print('[EDITCOMP] Changing via editComponent('
@@ -421,7 +306,7 @@ class BugzillaProxy:
                 qacontact = 'extras-qa@fedoraproject.org'
 
             data = {
-                'product': PRODUCTS[collection],
+                'product': env['products'][collection],
                 'component': package,
                 'description': description or 'NA',
                 'initialowner': owner,
@@ -469,13 +354,13 @@ def notify_users(errors):
     address, use it to notify the user about the issue.
     '''
     data = {}
-    if os.path.exists(DATA_CACHE):
+    if os.path.exists(env['data_cache']):
         try:
-            with open(DATA_CACHE) as stream:
+            with open(env['data_cache']) as stream:
                 data = json.load(stream)
         except Exception as err:
             print('Could not read the json file at %s: \nError:  %s' % (
-                  DATA_CACHE, err))
+                  env['data_cache'], err))
 
     new_data = {}
     seen = []
@@ -507,25 +392,25 @@ def notify_users(errors):
 
             if notify_user:
                 send_email(
-                    EMAIL_FROM,
+                    env['email_from'],
                     [user_email],
                     subject='Please fix your bugzilla.redhat.com account',
-                    message=PKGDB_SYNC_BUGZILLA_EMAIL,
+                    message=env['tmpl_user_email'],
-                    ccAddress=NOTIFYEMAIL,
+                    ccAddress=env['notify_emails'],
                 )
 
                 new_data[user_email] = {
                     'last_update': time.mktime(now.timetuple())
                 }
 
-    with open(DATA_CACHE, 'w') as stream:
+    with open(env['data_cache'], 'w') as stream:
         json.dump(new_data, stream)
 
 
 @cache.cache_on_arguments()
 def _get_override_yaml(project, session):
     pagure_override_url = '{0}/{1}/raw/master/f/{2}/{3}'.format(
-        PAGUREURL.rstrip('/'), BUGZILLA_OVERRIDE_REPO, project['namespace'],
+        env['pagure_url'].rstrip('/'), env['bugzilla']['override_repo'], project['namespace'],
         project['name'])
 
     if VERBOSE:
@@ -543,10 +428,10 @@ def _get_pdc_branches(session, repo):
     :param repo: the project dict
     :return: a list of the repo's branches
     """
-    branches_url = '{0}component-branches/'.format(PDCURL)
+    branches_url = '{0}component-branches/'.format(env['pdc_url'])
     params = dict(
         global_component=repo['name'],
-        type=PDC_TYPES[repo['namespace']]
+        type=env['pdc_types'][repo['namespace']]
     )
     if VERBOSE:
         print('Querying {0} {1}'.format(branches_url, params))
@@ -634,7 +519,7 @@ def _to_legacy_schema(product_and_project_and_summary, session=None):
 
 def main():
     """The entrypoint to the script."""
-    global VERBOSE, DRYRUN, projects_dict
+    global envname, env, VERBOSE, DRYRUN, projects_dict
     start = time.time()
 
     parser = argparse.ArgumentParser(
@@ -652,6 +537,11 @@ def main():
 
     args = parser.parse_args()
 
+    load_configuration()
+
+    envname = config['environment']
+    env = config['environments'][envname]
+
     if args.debug:
         VERBOSE = True
         DRYRUN = True
@@ -677,11 +567,11 @@ def main():
 
     # Get the initial ownership and CC data from pagure
     # This part is easy.
-    poc_url = PAGURE_DIST_GIT_URL + '/extras/pagure_poc.json'
+    poc_url = env['distgit_url'] + '/extras/pagure_poc.json'
     if VERBOSE:
         print("Querying %r for points of contact." % poc_url)
     pagure_namespace_to_poc = session.get(poc_url, timeout=120).json()
-    cc_url = PAGURE_DIST_GIT_URL + '/extras/pagure_bz.json'
+    cc_url = env['distgit_url'] + '/extras/pagure_bz.json'
     if VERBOSE:
         print("Querying %r for initial cc list." % cc_url)
     pagure_namespace_to_cc = session.get(cc_url, timeout=120).json()
@@ -707,18 +597,18 @@ def main():
             p for p in pagure_projects if p['namespace'] != 'modules'
         ]
 
-    branches_url = PDCURL.split('rest_api')[0] + 'extras/active_branches.json'
+    branches_url = env['pdc_url'].split('rest_api')[0] + 'extras/active_branches.json'
     if VERBOSE:
         print("Querying %r for EOL information." % branches_url)
     pdc_branches = session.get(branches_url, timeout=120).json()
     for proj in pagure_projects:
-        if proj['namespace'] not in PDC_TYPES:
+        if proj['namespace'] not in env['pdc_types']:
             proj['branches'] = []
             if VERBOSE:
                 print('! Namespace {namespace} unknown to PDC, project '
                       '{namespace}/{name} ignored'.format(**proj))
             continue
-        pdc_type = PDC_TYPES[proj['namespace']]
+        pdc_type = env['pdc_types'][proj['namespace']]
         proj['branches'] = pdc_branches.get(pdc_type, {}).get(proj['name'], [])
         if not proj['branches'] and VERBOSE:
             print("! No PDC branch found for {namespace}/{name}".format(**proj))
@@ -733,7 +623,7 @@ def main():
             if re.match(r'^epel\d+$', branch):
                 products.add('Fedora EPEL')
             else:
-                products.add(NAMESPACE_TO_PRODUCT[project['namespace']])
+                products.add(env['namespace_to_product'][project['namespace']])
         project['products'] = list(products)
 
     # Now, we must transform the data we collected into something that PkgDB
@@ -751,10 +641,13 @@ def main():
         projects_dict[response['product']][response['project']] = response
 
     # Initialize the connection to bugzilla
-    bugzilla = BugzillaProxy(BZSERVER, BZUSER, BZPASS, projects_dict)
+    bugzilla = BugzillaProxy(env['bugzilla']['url'],
+                             env['bugzilla']['user'],
+                             env['bugzilla']['password'],
+                             projects_dict)
 
     for product, pkgs in projects_dict.items():
-        if product not in PRODUCTS:
+        if product not in env['products']:
             continue
         for pkgname, pkginfo in sorted(projects_dict[product].items(),
                                        key=lambda x: x[0]):
@@ -793,12 +686,13 @@ def main():
         else:
             notify_users(errors)
             send_email(
-                EMAIL_FROM,
+                env['email_from'],
-                NOTIFYEMAIL,
+                env['notify_emails'],
                 'Errors while syncing bugzilla with the PackageDB',
-                TMPL_EMAIL_ADMIN % ('\n'.join(errors),))
+                env['tmpl_admin_email'] % ('\n'.join(errors),))
+            )
     else:
-        with open(DATA_CACHE, 'w') as stream:
+        with open(env['data_cache'], 'w') as stream:
             json.dump({}, stream)
 
     if VERBOSE: