use configuration from files
Signed-off-by: Nils Philippsen <nils@redhat.com>
This commit is contained in:
Nils Philippsen
parent
344dc75864
commit
cdb0b03ea4
1 changed files with 42 additions and 148 deletions
|
|
@ -53,10 +53,9 @@ from urllib3.util import Retry
|
|||
import yaml
|
||||
|
||||
from . import package_summary
|
||||
from .config import config, email_overrides, load_configuration
|
||||
|
||||
|
||||
env = 'staging'
|
||||
|
||||
cache = dogpile.cache.make_region().configure(
|
||||
'dogpile.cache.memory',
|
||||
expiration_time=3600,
|
||||
|
|
@ -78,123 +77,9 @@ def retry_session():
|
|||
return session
|
||||
|
||||
|
||||
if env == 'staging':
|
||||
BZSERVER = 'https://partner-bugzilla.redhat.com'
|
||||
else:
|
||||
BZSERVER = 'https://bugzilla.redhat.com'
|
||||
|
||||
BZUSER = '{{ bugzilla_user }}'
|
||||
BZPASS = '{{ bugzilla_password }}'
|
||||
BZCOMPAPI = 'component.get'
|
||||
FASUSER = '{{ fedorathirdpartyUser }}'
|
||||
FASPASS = '{{ fedorathirdpartyPassword }}'
|
||||
BUGZILLA_OVERRIDE_REPO = 'releng/fedora-scm-requests'
|
||||
NOTIFYEMAIL = [
|
||||
'kevin@fedoraproject.org',
|
||||
'pingou@fedoraproject.org',
|
||||
]
|
||||
VERBOSE = False
|
||||
DRYRUN = False
|
||||
|
||||
if env == 'staging':
|
||||
FASURL = 'https://admin.stg.fedoraproject.org/accounts'
|
||||
FASINSECURE = True
|
||||
PAGUREURL = 'https://stg.pagure.io'
|
||||
PAGURE_DIST_GIT_URL = 'https://src.stg.fedoraproject.org'
|
||||
PDCURL = 'https://pdc.stg.fedoraproject.org/rest_api/v1/'
|
||||
MDAPIURL = 'https://apps.stg.fedoraproject.org/mdapi/'
|
||||
else:
|
||||
FASURL = 'https://admin.fedoraproject.org/accounts'
|
||||
FASINSECURE = False
|
||||
PAGUREURL = 'https://pagure.io'
|
||||
PAGURE_DIST_GIT_URL = 'https://src.fedoraproject.org'
|
||||
PDCURL = 'https://pdc.fedoraproject.org/rest_api/v1/'
|
||||
MDAPIURL = 'https://apps.fedoraproject.org/mdapi/'
|
||||
|
||||
|
||||
EMAIL_FROM = 'accounts@fedoraproject.org'
|
||||
DATA_CACHE = '/var/tmp/pagure_sync_bz.json'
|
||||
|
||||
PRODUCTS = {
|
||||
'Fedora': 'Fedora',
|
||||
'Fedora Container': 'Fedora Container Images',
|
||||
'Fedora Modules': 'Fedora Modules',
|
||||
'Fedora EPEL': 'Fedora EPEL',
|
||||
}
|
||||
|
||||
NAMESPACE_TO_PRODUCT = {
|
||||
'rpms': 'Fedora', # except EPEL...
|
||||
'container': 'Fedora Container',
|
||||
'modules': 'Fedora Modules',
|
||||
}
|
||||
|
||||
# This maps bugzilla products to "lead" branches in PDC. If the lead branch is
|
||||
# retired, then we in turn set the default assignee to "orphan" for all new bugs
|
||||
# in the given product.
|
||||
PRODUCTS_TO_LEAD_BRANCH = {
|
||||
# If rawhide is retired, then all new bugs go to orphan for Fedora.
|
||||
'Fedora': 'master',
|
||||
# Same for containers.
|
||||
'Fedora Container': 'master',
|
||||
# Same for modules.
|
||||
'Fedora Modules': 'master',
|
||||
# If epel7 is retired, then all new epel bugs go to orphan.
|
||||
'Fedora EPEL': 'epel7',
|
||||
}
|
||||
PDC_TYPES = {
|
||||
'rpms': 'rpm',
|
||||
'modules': 'module',
|
||||
'container': 'container',
|
||||
}
|
||||
INVERSE_PDC_TYPES = {v: k for k, v in PDC_TYPES.items()}
|
||||
|
||||
|
||||
# When querying for current info, take segments of 1000 packages a time
|
||||
BZ_PKG_SEGMENT = 1000
|
||||
|
||||
|
||||
TMPL_EMAIL_ADMIN = '''
|
||||
The following errors were encountered while updating bugzilla with information
|
||||
from the Package Database. Please have the problems taken care of:
|
||||
|
||||
%s
|
||||
'''
|
||||
|
||||
# PkgDB sync bugzilla email
|
||||
PKGDB_SYNC_BUGZILLA_EMAIL = """Greetings.
|
||||
|
||||
You are receiving this email because there's a problem with your
|
||||
bugzilla.redhat.com account.
|
||||
|
||||
If you recently changed the email address associated with your
|
||||
Fedora account in the Fedora Account System, it is now out of sync
|
||||
with your bugzilla.redhat.com account. This leads to problems
|
||||
with Fedora packages you own or are CC'ed on bug reports for.
|
||||
|
||||
Please take one of the following actions:
|
||||
|
||||
a) login to your old bugzilla.redhat.com account and change the email
|
||||
address to match your current email in the Fedora account system.
|
||||
https://bugzilla.redhat.com login, click preferences, account
|
||||
information and enter new email address.
|
||||
|
||||
b) Create a new account in bugzilla.redhat.com to match your
|
||||
email listed in your Fedora account system account.
|
||||
https://bugzilla.redhat.com/ click 'new account' and enter email
|
||||
address.
|
||||
|
||||
c) Change your Fedora Account System email to match your existing
|
||||
bugzilla.redhat.com account.
|
||||
https://admin.fedoraproject.org/accounts login, click on 'my account',
|
||||
then 'edit' and change your email address.
|
||||
|
||||
If you have questions or concerns, please let us know.
|
||||
|
||||
Your prompt attention in this matter is appreciated.
|
||||
|
||||
The Fedora admins.
|
||||
"""
|
||||
|
||||
|
||||
def resilient_partial(fn, *initial, **kwargs):
|
||||
""" A decorator that partially applies arguments.
|
||||
|
|
@ -244,20 +129,20 @@ class ProductCache(dict):
|
|||
if key not in self.acls:
|
||||
raise
|
||||
|
||||
if BZCOMPAPI == 'getcomponentsdetails':
|
||||
if env['bugzilla']['compat_api'] == 'getcomponentsdetails':
|
||||
# Old API -- in python-bugzilla. But with current server, this
|
||||
# gives ProxyError
|
||||
products = self.bz.getcomponentsdetails(key)
|
||||
elif BZCOMPAPI == 'component.get':
|
||||
elif env['bugzilla']['compat_api'] == 'component.get':
|
||||
# Way that's undocumented in the partner-bugzilla api but works
|
||||
# currently
|
||||
pkglist = list(projects_dict[key])
|
||||
products = {}
|
||||
for pkg_segment in segment(pkglist, BZ_PKG_SEGMENT):
|
||||
for pkg_segment in segment(pkglist, env['bugzilla']['req_segment']):
|
||||
# Format that bugzilla will understand. Strip None's that
|
||||
# segment() pads out the final data segment() with
|
||||
query = [
|
||||
dict(product=PRODUCTS[key], component=p)
|
||||
dict(product=env['products'][key], component=p)
|
||||
for p in pkg_segment if p is not None
|
||||
]
|
||||
raw_data = self.bz._proxy.Component.get(dict(names=query))
|
||||
|
|
@ -290,9 +175,9 @@ class BugzillaProxy:
|
|||
|
||||
# Connect to the fedora account system
|
||||
self.fas = AccountSystem(
|
||||
base_url=FASURL,
|
||||
username=FASUSER,
|
||||
password=FASPASS)
|
||||
base_url=env['fas']['url'],
|
||||
username=env['fas']['username'],
|
||||
password=env['fas']['password'])
|
||||
|
||||
try:
|
||||
self.userCache = self.fas.people_by_key(
|
||||
|
|
@ -393,7 +278,7 @@ class BugzillaProxy:
|
|||
data['initialowner'] = owner
|
||||
|
||||
# Changes occurred. Submit a request to change via xmlrpc
|
||||
data['product'] = PRODUCTS[collection]
|
||||
data['product'] = env['products'][collection]
|
||||
data['component'] = package
|
||||
if VERBOSE:
|
||||
print('[EDITCOMP] Changing via editComponent('
|
||||
|
|
@ -421,7 +306,7 @@ class BugzillaProxy:
|
|||
qacontact = 'extras-qa@fedoraproject.org'
|
||||
|
||||
data = {
|
||||
'product': PRODUCTS[collection],
|
||||
'product': env['products'][collection],
|
||||
'component': package,
|
||||
'description': description or 'NA',
|
||||
'initialowner': owner,
|
||||
|
|
@ -469,13 +354,13 @@ def notify_users(errors):
|
|||
address, use it to notify the user about the issue.
|
||||
'''
|
||||
data = {}
|
||||
if os.path.exists(DATA_CACHE):
|
||||
if os.path.exists(env['data_cache']):
|
||||
try:
|
||||
with open(DATA_CACHE) as stream:
|
||||
with open(env['data_cache']) as stream:
|
||||
data = json.load(stream)
|
||||
except Exception as err:
|
||||
print('Could not read the json file at %s: \nError: %s' % (
|
||||
DATA_CACHE, err))
|
||||
env['data_cache'], err))
|
||||
|
||||
new_data = {}
|
||||
seen = []
|
||||
|
|
@ -507,25 +392,25 @@ def notify_users(errors):
|
|||
|
||||
if notify_user:
|
||||
send_email(
|
||||
EMAIL_FROM,
|
||||
env['email_from'],
|
||||
[user_email],
|
||||
subject='Please fix your bugzilla.redhat.com account',
|
||||
message=PKGDB_SYNC_BUGZILLA_EMAIL,
|
||||
ccAddress=NOTIFYEMAIL,
|
||||
message=env['tmpl_user_email'],
|
||||
ccAddress=env['notify_emails'],
|
||||
)
|
||||
|
||||
new_data[user_email] = {
|
||||
'last_update': time.mktime(now.timetuple())
|
||||
}
|
||||
|
||||
with open(DATA_CACHE, 'w') as stream:
|
||||
with open(env['data_cache'], 'w') as stream:
|
||||
json.dump(new_data, stream)
|
||||
|
||||
|
||||
@cache.cache_on_arguments()
|
||||
def _get_override_yaml(project, session):
|
||||
pagure_override_url = '{0}/{1}/raw/master/f/{2}/{3}'.format(
|
||||
PAGUREURL.rstrip('/'), BUGZILLA_OVERRIDE_REPO, project['namespace'],
|
||||
env['pagure_url'].rstrip('/'), env['bugzilla']['override_repo'], project['namespace'],
|
||||
project['name'])
|
||||
|
||||
if VERBOSE:
|
||||
|
|
@ -543,10 +428,10 @@ def _get_pdc_branches(session, repo):
|
|||
:param repo: the project dict
|
||||
:return: a list of the repo's branches
|
||||
"""
|
||||
branches_url = '{0}component-branches/'.format(PDCURL)
|
||||
branches_url = '{0}component-branches/'.format(env['pdc_url'])
|
||||
params = dict(
|
||||
global_component=repo['name'],
|
||||
type=PDC_TYPES[repo['namespace']]
|
||||
type=env['pdc_types'][repo['namespace']]
|
||||
)
|
||||
if VERBOSE:
|
||||
print('Querying {0} {1}'.format(branches_url, params))
|
||||
|
|
@ -634,7 +519,7 @@ def _to_legacy_schema(product_and_project_and_summary, session=None):
|
|||
|
||||
def main():
|
||||
"""The entrypoint to the script."""
|
||||
global VERBOSE, DRYRUN, projects_dict
|
||||
global envname, env, VERBOSE, DRYRUN, projects_dict
|
||||
start = time.time()
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
|
|
@ -652,6 +537,11 @@ def main():
|
|||
|
||||
args = parser.parse_args()
|
||||
|
||||
load_configuration()
|
||||
|
||||
envname = config['environment']
|
||||
env = config['environments'][envname]
|
||||
|
||||
if args.debug:
|
||||
VERBOSE = True
|
||||
DRYRUN = True
|
||||
|
|
@ -677,11 +567,11 @@ def main():
|
|||
|
||||
# Get the initial ownership and CC data from pagure
|
||||
# This part is easy.
|
||||
poc_url = PAGURE_DIST_GIT_URL + '/extras/pagure_poc.json'
|
||||
poc_url = env['distgit_url'] + '/extras/pagure_poc.json'
|
||||
if VERBOSE:
|
||||
print("Querying %r for points of contact." % poc_url)
|
||||
pagure_namespace_to_poc = session.get(poc_url, timeout=120).json()
|
||||
cc_url = PAGURE_DIST_GIT_URL + '/extras/pagure_bz.json'
|
||||
cc_url = env['distgit_url'] + '/extras/pagure_bz.json'
|
||||
if VERBOSE:
|
||||
print("Querying %r for initial cc list." % cc_url)
|
||||
pagure_namespace_to_cc = session.get(cc_url, timeout=120).json()
|
||||
|
|
@ -707,18 +597,18 @@ def main():
|
|||
p for p in pagure_projects if p['namespace'] != 'modules'
|
||||
]
|
||||
|
||||
branches_url = PDCURL.split('rest_api')[0] + 'extras/active_branches.json'
|
||||
branches_url = env['pdc_url'].split('rest_api')[0] + 'extras/active_branches.json'
|
||||
if VERBOSE:
|
||||
print("Querying %r for EOL information." % branches_url)
|
||||
pdc_branches = session.get(branches_url, timeout=120).json()
|
||||
for proj in pagure_projects:
|
||||
if proj['namespace'] not in PDC_TYPES:
|
||||
if proj['namespace'] not in env['pdc_types']:
|
||||
proj['branches'] = []
|
||||
if VERBOSE:
|
||||
print('! Namespace {namespace} unknown to PDC, project '
|
||||
'{namespace}/{name} ignored'.format(**proj))
|
||||
continue
|
||||
pdc_type = PDC_TYPES[proj['namespace']]
|
||||
pdc_type = env['pdc_types'][proj['namespace']]
|
||||
proj['branches'] = pdc_branches.get(pdc_type, {}).get(proj['name'], [])
|
||||
if not proj['branches'] and VERBOSE:
|
||||
print("! No PDC branch found for {namespace}/{name}".format(**proj))
|
||||
|
|
@ -733,7 +623,7 @@ def main():
|
|||
if re.match(r'^epel\d+$', branch):
|
||||
products.add('Fedora EPEL')
|
||||
else:
|
||||
products.add(NAMESPACE_TO_PRODUCT[project['namespace']])
|
||||
products.add(env['namespace_to_product'][project['namespace']])
|
||||
project['products'] = list(products)
|
||||
|
||||
# Now, we must transform the data we collected into something that PkgDB
|
||||
|
|
@ -751,10 +641,13 @@ def main():
|
|||
projects_dict[response['product']][response['project']] = response
|
||||
|
||||
# Initialize the connection to bugzilla
|
||||
bugzilla = BugzillaProxy(BZSERVER, BZUSER, BZPASS, projects_dict)
|
||||
bugzilla = BugzillaProxy(env['bugzilla']['url'],
|
||||
env['bugzilla']['user'],
|
||||
env['bugzilla']['password'],
|
||||
projects_dict)
|
||||
|
||||
for product, pkgs in projects_dict.items():
|
||||
if product not in PRODUCTS:
|
||||
if product not in env['products']:
|
||||
continue
|
||||
for pkgname, pkginfo in sorted(projects_dict[product].items(),
|
||||
key=lambda x: x[0]):
|
||||
|
|
@ -793,12 +686,13 @@ def main():
|
|||
else:
|
||||
notify_users(errors)
|
||||
send_email(
|
||||
EMAIL_FROM,
|
||||
NOTIFYEMAIL,
|
||||
env['email_from'],
|
||||
env['notify_emails'],
|
||||
'Errors while syncing bugzilla with the PackageDB',
|
||||
TMPL_EMAIL_ADMIN % ('\n'.join(errors),))
|
||||
env['tmpl_admin_email'] % ('\n'.join(errors),))
|
||||
)
|
||||
else:
|
||||
with open(DATA_CACHE, 'w') as stream:
|
||||
with open(env['data_cache'], 'w') as stream:
|
||||
json.dump({}, stream)
|
||||
|
||||
if VERBOSE:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue