Restructure the code
- Add an --env variable to change the environment used via the CLI - Drop the ProductCache object - Introduce the DistgitBugzillaSync object which allows dropping global variables to rely on attributes instead. - Drop the pkgdb compatibility layer that was once introduced, pkgdb is no longer a thing and it's time to move on This changes a little bit the order in which projects are updated, it used to be by products first (Fedora, Fedora EPEL...) then package, now it's the opposite: by package first then product. Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>
This commit is contained in:
parent
fd4c3b0714
commit
94d17b2c05
2 changed files with 371 additions and 380 deletions
|
@ -34,7 +34,7 @@ import datetime
|
||||||
from email.message import EmailMessage
|
from email.message import EmailMessage
|
||||||
import itertools
|
import itertools
|
||||||
import json
|
import json
|
||||||
import multiprocessing.pool
|
from operator import itemgetter
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import smtplib
|
import smtplib
|
||||||
|
@ -77,10 +77,6 @@ def retry_session():
|
||||||
return session
|
return session
|
||||||
|
|
||||||
|
|
||||||
VERBOSE = False
|
|
||||||
DRYRUN = False
|
|
||||||
|
|
||||||
|
|
||||||
def resilient_partial(fn, *initial, **kwargs):
|
def resilient_partial(fn, *initial, **kwargs):
|
||||||
""" A decorator that partially applies arguments.
|
""" A decorator that partially applies arguments.
|
||||||
|
|
||||||
|
@ -114,55 +110,9 @@ def segment(iterable, chunk, fill=None):
|
||||||
return itertools.zip_longest(*args, fillvalue=fill)
|
return itertools.zip_longest(*args, fillvalue=fill)
|
||||||
|
|
||||||
|
|
||||||
class ProductCache(dict):
|
|
||||||
def __init__(self, bz, acls):
|
|
||||||
self.bz = bz
|
|
||||||
self.acls = acls
|
|
||||||
|
|
||||||
# Ask bugzilla for a section of the pkglist.
|
|
||||||
# Save the information from the section that we want.
|
|
||||||
def __getitem__(self, key):
|
|
||||||
try:
|
|
||||||
return super(ProductCache, self).__getitem__(key)
|
|
||||||
except KeyError:
|
|
||||||
# We can only cache products we have pagure information for
|
|
||||||
if key not in self.acls:
|
|
||||||
raise
|
|
||||||
|
|
||||||
if env['bugzilla']['compat_api'] == 'getcomponentsdetails':
|
|
||||||
# Old API -- in python-bugzilla. But with current server, this
|
|
||||||
# gives ProxyError
|
|
||||||
products = self.bz.getcomponentsdetails(key)
|
|
||||||
elif env['bugzilla']['compat_api'] == 'component.get':
|
|
||||||
# Way that's undocumented in the partner-bugzilla api but works
|
|
||||||
# currently
|
|
||||||
pkglist = list(projects_dict[key])
|
|
||||||
products = {}
|
|
||||||
for pkg_segment in segment(pkglist, env['bugzilla']['req_segment']):
|
|
||||||
# Format that bugzilla will understand. Strip None's that
|
|
||||||
# segment() pads out the final data segment() with
|
|
||||||
query = [
|
|
||||||
dict(product=env['products'][key], component=p)
|
|
||||||
for p in pkg_segment if p is not None
|
|
||||||
]
|
|
||||||
raw_data = self.bz._proxy.Component.get(dict(names=query))
|
|
||||||
for package in raw_data['components']:
|
|
||||||
# Reformat data to be the same as what's returned from
|
|
||||||
# getcomponentsdetails
|
|
||||||
product = dict(
|
|
||||||
initialowner=package['default_assignee'],
|
|
||||||
description=package['description'],
|
|
||||||
initialqacontact=package['default_qa_contact'],
|
|
||||||
initialcclist=package['default_cc'])
|
|
||||||
products[package['name'].lower()] = product
|
|
||||||
self[key] = products
|
|
||||||
|
|
||||||
return super(ProductCache, self).__getitem__(key)
|
|
||||||
|
|
||||||
|
|
||||||
class BugzillaProxy:
|
class BugzillaProxy:
|
||||||
|
|
||||||
def __init__(self, bzServer, username, password, acls):
|
def __init__(self, bzServer, username, password, config):
|
||||||
self.bzXmlRpcServer = bzServer
|
self.bzXmlRpcServer = bzServer
|
||||||
self.username = username
|
self.username = username
|
||||||
self.password = password
|
self.password = password
|
||||||
|
@ -171,13 +121,15 @@ class BugzillaProxy:
|
||||||
url=self.bzXmlRpcServer,
|
url=self.bzXmlRpcServer,
|
||||||
user=self.username,
|
user=self.username,
|
||||||
password=self.password)
|
password=self.password)
|
||||||
self.productCache = ProductCache(self.server, acls)
|
self.productCache = {}
|
||||||
|
|
||||||
# Connect to the fedora account system
|
# Connect to the fedora account system
|
||||||
self.fas = AccountSystem(
|
self.fas = AccountSystem(
|
||||||
base_url=env['fas']['url'],
|
base_url=config['fas']['url'],
|
||||||
username=env['fas']['username'],
|
username=config['fas']['username'],
|
||||||
password=env['fas']['password'])
|
password=config['fas']['password'])
|
||||||
|
|
||||||
|
self.config = config
|
||||||
|
|
||||||
try:
|
try:
|
||||||
self.userCache = self.fas.people_by_key(
|
self.userCache = self.fas.people_by_key(
|
||||||
|
@ -188,6 +140,53 @@ class BugzillaProxy:
|
||||||
# It's ok, we build the cache as-needed later in the script.
|
# It's ok, we build the cache as-needed later in the script.
|
||||||
self.userCache = {}
|
self.userCache = {}
|
||||||
|
|
||||||
|
def build_product_cache(self, pagure_project):
|
||||||
|
""" Cache the bugzilla info about each package in each product.
|
||||||
|
"""
|
||||||
|
|
||||||
|
products = {}
|
||||||
|
if self.config['bugzilla']['compat_api'] == 'getcomponentsdetails':
|
||||||
|
# Old API -- in python-bugzilla. But with current server, this
|
||||||
|
# gives ProxyError
|
||||||
|
for collection, product in self.config["products"].items():
|
||||||
|
self.productCache[collection] = self.server.getcomponentsdetails(product)
|
||||||
|
elif self.config['bugzilla']['compat_api'] == 'component.get':
|
||||||
|
# Way that's undocumented in the partner-bugzilla api but works
|
||||||
|
# currently
|
||||||
|
products = {}
|
||||||
|
for collection, product in self.config["products"].items():
|
||||||
|
|
||||||
|
# restrict the list of info returned to only the packages of
|
||||||
|
# interest
|
||||||
|
pkglist = [
|
||||||
|
project["name"]
|
||||||
|
for project in pagure_project
|
||||||
|
if product in project["products"]
|
||||||
|
]
|
||||||
|
for pkg_segment in segment(pkglist, self.config['bugzilla']['req_segment']):
|
||||||
|
# Format that bugzilla will understand. Strip None's that
|
||||||
|
# segment() pads out the final data segment() with
|
||||||
|
query = [
|
||||||
|
dict(
|
||||||
|
product=self.config['products'][collection],
|
||||||
|
component=p
|
||||||
|
)
|
||||||
|
for p in pkg_segment
|
||||||
|
if p is not None
|
||||||
|
]
|
||||||
|
raw_data = self.server._proxy.Component.get(dict(names=query))
|
||||||
|
for package in raw_data['components']:
|
||||||
|
# Reformat data to be the same as what's returned from
|
||||||
|
# getcomponentsdetails
|
||||||
|
product = dict(
|
||||||
|
initialowner=package['default_assignee'],
|
||||||
|
description=package['description'],
|
||||||
|
initialqacontact=package['default_qa_contact'],
|
||||||
|
initialcclist=package['default_cc']
|
||||||
|
)
|
||||||
|
products[package['name'].lower()] = product
|
||||||
|
self.productCache[collection] = products
|
||||||
|
|
||||||
def _get_bugzilla_email(self, username):
|
def _get_bugzilla_email(self, username):
|
||||||
'''Return the bugzilla email address for a user.
|
'''Return the bugzilla email address for a user.
|
||||||
|
|
||||||
|
@ -283,10 +282,10 @@ class BugzillaProxy:
|
||||||
|
|
||||||
if data:
|
if data:
|
||||||
# Changes occurred. Submit a request to change via xmlrpc
|
# Changes occurred. Submit a request to change via xmlrpc
|
||||||
data['product'] = env['products'][collection]
|
data['product'] = self.config['products'][collection]
|
||||||
data['component'] = package
|
data['component'] = package
|
||||||
|
|
||||||
if VERBOSE:
|
if self.config["verbose"]:
|
||||||
print('[EDITCOMP] %s/%s' % (data["product"], data["component"]))
|
print('[EDITCOMP] %s/%s' % (data["product"], data["component"]))
|
||||||
for key in ["initialowner", "description", "initialqacontact", "initialcclist"]:
|
for key in ["initialowner", "description", "initialqacontact", "initialcclist"]:
|
||||||
if data.get(key):
|
if data.get(key):
|
||||||
|
@ -296,7 +295,7 @@ class BugzillaProxy:
|
||||||
# FIXME: initialowner has been made mandatory for some
|
# FIXME: initialowner has been made mandatory for some
|
||||||
# reason. Asking dkl why.
|
# reason. Asking dkl why.
|
||||||
data['initialowner'] = owner
|
data['initialowner'] = owner
|
||||||
if not DRYRUN:
|
if not self.config["dryrun"]:
|
||||||
try:
|
try:
|
||||||
self.server.editcomponent(data)
|
self.server.editcomponent(data)
|
||||||
except xmlrpc.client.Fault as e:
|
except xmlrpc.client.Fault as e:
|
||||||
|
@ -314,7 +313,7 @@ class BugzillaProxy:
|
||||||
qacontact = 'extras-qa@fedoraproject.org'
|
qacontact = 'extras-qa@fedoraproject.org'
|
||||||
|
|
||||||
data = {
|
data = {
|
||||||
'product': env['products'][collection],
|
'product': self.config['products'][collection],
|
||||||
'component': package,
|
'component': package,
|
||||||
'description': description or 'NA',
|
'description': description or 'NA',
|
||||||
'initialowner': owner,
|
'initialowner': owner,
|
||||||
|
@ -323,12 +322,12 @@ class BugzillaProxy:
|
||||||
if initialCCList:
|
if initialCCList:
|
||||||
data['initialcclist'] = initialCCList
|
data['initialcclist'] = initialCCList
|
||||||
|
|
||||||
if VERBOSE:
|
if self.config["verbose"]:
|
||||||
print('[ADDCOMP] %s/%s' % (data["product"], data["component"]))
|
print('[ADDCOMP] %s/%s' % (data["product"], data["component"]))
|
||||||
for key in ["initialowner", "description", "initialqacontact", "initialcclist"]:
|
for key in ["initialowner", "description", "initialqacontact", "initialcclist"]:
|
||||||
if data.get(key):
|
if data.get(key):
|
||||||
print(f" {key} set to {data.get(key)}")
|
print(f" {key} set to {data.get(key)}")
|
||||||
if not DRYRUN:
|
if not self.config["dryrun"]:
|
||||||
try:
|
try:
|
||||||
self.server.addcomponent(data)
|
self.server.addcomponent(data)
|
||||||
except xmlrpc.client.Fault as e:
|
except xmlrpc.client.Fault as e:
|
||||||
|
@ -359,14 +358,74 @@ def send_email(fromAddress, toAddress, subject, message, ccAddress=None):
|
||||||
smtp.quit()
|
smtp.quit()
|
||||||
|
|
||||||
|
|
||||||
def notify_users(errors):
|
@cache.cache_on_arguments()
|
||||||
|
def _get_override_yaml(project, session):
|
||||||
|
pagure_override_url = '{0}/{1}/raw/master/f/{2}/{3}'.format(
|
||||||
|
env['pagure_url'].rstrip('/'), env['bugzilla']['override_repo'], project['namespace'],
|
||||||
|
project['name'])
|
||||||
|
|
||||||
|
if config["verbose"]:
|
||||||
|
print('Querying {0}'.format(pagure_override_url))
|
||||||
|
override_rv = session.get(pagure_override_url, timeout=30)
|
||||||
|
if override_rv.status_code == 200:
|
||||||
|
override_yaml = yaml.safe_load(override_rv.text)
|
||||||
|
return override_yaml.get('bugzilla_contact', {})
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def _get_pdc_branches(session, repo):
|
||||||
|
"""
|
||||||
|
Gets the branches on a project. This function is used for mapping.
|
||||||
|
:param repo: the project dict
|
||||||
|
:return: a list of the repo's branches
|
||||||
|
"""
|
||||||
|
branches_url = '{0}component-branches/'.format(env['pdc_url'])
|
||||||
|
params = dict(
|
||||||
|
global_component=repo['name'],
|
||||||
|
type=env['pdc_types'][repo['namespace']]
|
||||||
|
)
|
||||||
|
if config["verbose"]:
|
||||||
|
print('Querying {0} {1}'.format(branches_url, params))
|
||||||
|
rv = session.get(branches_url, params=params, timeout=60)
|
||||||
|
|
||||||
|
# If the project's branches can't be reported, just return no branches and
|
||||||
|
# it will be skipped later on
|
||||||
|
if not rv.ok:
|
||||||
|
print(('The connection to "{0}" failed with the status code {1} and '
|
||||||
|
'output "{2}"'.format(branches_url, rv.status_code, rv.text)),
|
||||||
|
file=sys.stderr)
|
||||||
|
return []
|
||||||
|
|
||||||
|
data = rv.json()
|
||||||
|
return [branch['name'] for branch in data['results']]
|
||||||
|
|
||||||
|
|
||||||
|
def _is_retired(product, project):
|
||||||
|
branches = project['branches']
|
||||||
|
if product == 'Fedora EPEL':
|
||||||
|
for branch, active in branches:
|
||||||
|
if re.match(r'^epel\d+$', branch):
|
||||||
|
if active:
|
||||||
|
return False
|
||||||
|
# No active branches means it is retired.
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
for branch, active in branches:
|
||||||
|
if active:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
class DistgitBugzillaSync:
|
||||||
|
|
||||||
|
def notify_users(self, errors):
|
||||||
''' Browse the list of errors and when we can retrieve the email
|
''' Browse the list of errors and when we can retrieve the email
|
||||||
address, use it to notify the user about the issue.
|
address, use it to notify the user about the issue.
|
||||||
'''
|
'''
|
||||||
data = {}
|
data = {}
|
||||||
if os.path.exists(env['data_cache']):
|
if os.path.exists(self.env['data_cache']):
|
||||||
try:
|
try:
|
||||||
with open(env['data_cache']) as stream:
|
with open(self.env['data_cache']) as stream:
|
||||||
data = json.load(stream)
|
data = json.load(stream)
|
||||||
except Exception as err:
|
except Exception as err:
|
||||||
print('Could not read the json file at %s: \nError: %s' % (
|
print('Could not read the json file at %s: \nError: %s' % (
|
||||||
|
@ -402,11 +461,11 @@ def notify_users(errors):
|
||||||
|
|
||||||
if notify_user:
|
if notify_user:
|
||||||
send_email(
|
send_email(
|
||||||
env['email_from'],
|
self.env['email_from'],
|
||||||
[user_email],
|
[user_email],
|
||||||
subject='Please fix your bugzilla.redhat.com account',
|
subject='Please fix your bugzilla.redhat.com account',
|
||||||
message=env['tmpl_user_email'],
|
message=self.env['tmpl_user_email'],
|
||||||
ccAddress=env['notify_emails'],
|
ccAddress=self.env['notify_emails'],
|
||||||
)
|
)
|
||||||
|
|
||||||
new_data[user_email] = {
|
new_data[user_email] = {
|
||||||
|
@ -416,275 +475,206 @@ def notify_users(errors):
|
||||||
with open(env['data_cache'], 'w') as stream:
|
with open(env['data_cache'], 'w') as stream:
|
||||||
json.dump(new_data, stream)
|
json.dump(new_data, stream)
|
||||||
|
|
||||||
|
def get_cli_arguments(self):
|
||||||
@cache.cache_on_arguments()
|
""" Set the CLI argument parser and return the argument parsed.
|
||||||
def _get_override_yaml(project, session):
|
|
||||||
pagure_override_url = '{0}/{1}/raw/master/f/{2}/{3}'.format(
|
|
||||||
env['pagure_url'].rstrip('/'), env['bugzilla']['override_repo'], project['namespace'],
|
|
||||||
project['name'])
|
|
||||||
|
|
||||||
if VERBOSE:
|
|
||||||
print('Querying {0}'.format(pagure_override_url))
|
|
||||||
override_rv = session.get(pagure_override_url, timeout=30)
|
|
||||||
if override_rv.status_code == 200:
|
|
||||||
override_yaml = yaml.safe_load(override_rv.text)
|
|
||||||
return override_yaml.get('bugzilla_contact', {})
|
|
||||||
return {}
|
|
||||||
|
|
||||||
|
|
||||||
def _get_pdc_branches(session, repo):
|
|
||||||
"""
|
"""
|
||||||
Gets the branches on a project. This function is used for mapping.
|
|
||||||
:param repo: the project dict
|
|
||||||
:return: a list of the repo's branches
|
|
||||||
"""
|
|
||||||
branches_url = '{0}component-branches/'.format(env['pdc_url'])
|
|
||||||
params = dict(
|
|
||||||
global_component=repo['name'],
|
|
||||||
type=env['pdc_types'][repo['namespace']]
|
|
||||||
)
|
|
||||||
if VERBOSE:
|
|
||||||
print('Querying {0} {1}'.format(branches_url, params))
|
|
||||||
rv = session.get(branches_url, params=params, timeout=60)
|
|
||||||
|
|
||||||
# If the project's branches can't be reported, just return no branches and
|
|
||||||
# it will be skipped later on
|
|
||||||
if not rv.ok:
|
|
||||||
print(('The connection to "{0}" failed with the status code {1} and '
|
|
||||||
'output "{2}"'.format(branches_url, rv.status_code, rv.text)),
|
|
||||||
file=sys.stderr)
|
|
||||||
return []
|
|
||||||
|
|
||||||
data = rv.json()
|
|
||||||
return [branch['name'] for branch in data['results']]
|
|
||||||
|
|
||||||
|
|
||||||
def _is_retired(product, project):
|
|
||||||
branches = project['branches']
|
|
||||||
if product == 'Fedora EPEL':
|
|
||||||
for branch, active in branches:
|
|
||||||
if re.match(r'^epel\d+$', branch):
|
|
||||||
if active:
|
|
||||||
return False
|
|
||||||
# No active branches means it is retired.
|
|
||||||
return True
|
|
||||||
else:
|
|
||||||
for branch, active in branches:
|
|
||||||
if active:
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def _to_legacy_schema(product_and_project_and_summary, session=None):
|
|
||||||
"""
|
|
||||||
This function translates the JSON of a Pagure project to what PkgDB used to
|
|
||||||
output in the Bugzilla API. This function is used for mapping.
|
|
||||||
:param project_and_product: a tuple containing the dictionary of the JSON
|
|
||||||
of a Pagure project and a string of the product (e.g. "Fedora",
|
|
||||||
"Fedora EPEL")
|
|
||||||
:param session: a requests session object or None
|
|
||||||
:return: a dictionary of the content that the PkgDB Bugzilla API would
|
|
||||||
return
|
|
||||||
"""
|
|
||||||
product, project, rpm_summary = product_and_project_and_summary
|
|
||||||
|
|
||||||
if session is None:
|
|
||||||
session = retry_session()
|
|
||||||
|
|
||||||
owner = project['poc']
|
|
||||||
watchers = project['watchers']
|
|
||||||
|
|
||||||
summary = None
|
|
||||||
if project["namespace"] == "rpms":
|
|
||||||
summary = rpm_summary.get(project["name"])
|
|
||||||
|
|
||||||
# Check if the project is retired in PDC, and if so set assignee to orphan.
|
|
||||||
if _is_retired(product, project):
|
|
||||||
owner = 'orphan'
|
|
||||||
|
|
||||||
# Check if the Bugzilla ticket assignee has been manually overridden
|
|
||||||
override_yaml = _get_override_yaml(project, session)
|
|
||||||
if override_yaml.get(product) \
|
|
||||||
and isinstance(override_yaml[product], str):
|
|
||||||
owner = override_yaml[product]
|
|
||||||
|
|
||||||
return {
|
|
||||||
'cclist': {
|
|
||||||
# Groups is empty because you can't have groups watch projects.
|
|
||||||
# This is done only at the user level.
|
|
||||||
'groups': [],
|
|
||||||
'people': watchers,
|
|
||||||
},
|
|
||||||
'owner': owner,
|
|
||||||
# No package has this set in PkgDB's API, so it can be safely turned
|
|
||||||
# off and set to the defaults later on in the code
|
|
||||||
'qacontact': None,
|
|
||||||
'summary': summary,
|
|
||||||
# These two values are not part of original PkgDB RV, but they are
|
|
||||||
# useful
|
|
||||||
'product': product,
|
|
||||||
'project': project['name']
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
"""The entrypoint to the script."""
|
|
||||||
global envname, env, VERBOSE, DRYRUN, projects_dict
|
|
||||||
times = {
|
|
||||||
"start": time.time(),
|
|
||||||
}
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description='Script syncing information between Pagure and bugzilla'
|
description='Script syncing information between Pagure and bugzilla'
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--dry-run', dest='dryrun', action='store_true', default=False,
|
'--dry-run', dest='dryrun', action='store_true', default=False,
|
||||||
help='Do not actually make the changes')
|
help='Do not actually make any changes - Overrides the configuration')
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--verbose', dest='verbose', action='store_true', default=False,
|
'--verbose', dest='verbose', action='store_true', default=False,
|
||||||
help='Print actions verbosely')
|
help='Print actions verbosely - Overrides the configuration')
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--debug', dest='debug', action='store_true', default=False,
|
'--debug', dest='debug', action='store_true', default=False,
|
||||||
help='Combination of --verbose and --dry-run')
|
help='Combination of --verbose and --dry-run')
|
||||||
|
parser.add_argument(
|
||||||
|
'--env', dest='env',
|
||||||
|
help='Run the script for a specific environment, overrides the one '
|
||||||
|
'set in the configuration file')
|
||||||
|
|
||||||
args = parser.parse_args()
|
self.args = parser.parse_args()
|
||||||
|
|
||||||
load_configuration()
|
def get_pagure_project(self):
|
||||||
|
""" Builds a large list of all the projects on pagure.
|
||||||
envname = config['environment']
|
Each item in that list is a dict containing:
|
||||||
env = config['environments'][envname]
|
- the namespace of the project
|
||||||
|
- the name of the project
|
||||||
if args.debug:
|
- the point of contact of this project (ie: the default assignee
|
||||||
VERBOSE = True
|
in bugzilla)
|
||||||
DRYRUN = True
|
- the watchers of this project (ie: the initial CC list in bugzilla)
|
||||||
|
"""
|
||||||
if args.verbose:
|
|
||||||
VERBOSE = True
|
|
||||||
|
|
||||||
if args.dryrun:
|
|
||||||
DRYRUN = True
|
|
||||||
|
|
||||||
# Non-fatal errors to alert people about
|
|
||||||
errors = []
|
|
||||||
|
|
||||||
projects_dict = {
|
|
||||||
'Fedora': {},
|
|
||||||
'Fedora Container': {},
|
|
||||||
'Fedora Modules': {},
|
|
||||||
'Fedora EPEL': {},
|
|
||||||
}
|
|
||||||
|
|
||||||
session = retry_session()
|
|
||||||
pool = multiprocessing.pool.ThreadPool(8)
|
|
||||||
|
|
||||||
# Get the initial ownership and CC data from pagure
|
# Get the initial ownership and CC data from pagure
|
||||||
# This part is easy.
|
# This part is easy.
|
||||||
poc_url = env['distgit_url'] + '/extras/pagure_poc.json'
|
poc_url = self.env['distgit_url'] + '/extras/pagure_poc.json'
|
||||||
if VERBOSE:
|
if self.env["verbose"]:
|
||||||
print("Querying %r for points of contact." % poc_url)
|
print("Querying %r for points of contact." % poc_url)
|
||||||
pagure_namespace_to_poc = session.get(poc_url, timeout=120).json()
|
pagure_namespace_to_poc = self.session.get(poc_url, timeout=120).json()
|
||||||
cc_url = env['distgit_url'] + '/extras/pagure_bz.json'
|
|
||||||
if VERBOSE:
|
|
||||||
print("Querying %r for initial cc list." % cc_url)
|
|
||||||
pagure_namespace_to_cc = session.get(cc_url, timeout=120).json()
|
|
||||||
|
|
||||||
if VERBOSE:
|
cc_url = self.env['distgit_url'] + '/extras/pagure_bz.json'
|
||||||
print("Building a cache of the rpm packages' summary")
|
if self.env["verbose"]:
|
||||||
rpm_summary = package_summary.get_package_summary()
|
print("Querying %r for initial cc list." % cc_url)
|
||||||
|
pagure_namespace_to_cc = self.session.get(cc_url, timeout=120).json()
|
||||||
|
|
||||||
# Combine and collapse those two into a single list:
|
# Combine and collapse those two into a single list:
|
||||||
pagure_projects = []
|
self.pagure_projects = []
|
||||||
for namespace, entries in pagure_namespace_to_poc.items():
|
for namespace, entries in pagure_namespace_to_poc.items():
|
||||||
for name, poc in entries.items():
|
for name, poc in entries.items():
|
||||||
pagure_projects.append(dict(
|
self.pagure_projects.append(dict(
|
||||||
namespace=namespace,
|
namespace=namespace,
|
||||||
name=name,
|
name=name,
|
||||||
poc=poc,
|
poc=poc,
|
||||||
watchers=pagure_namespace_to_cc[namespace][name],
|
watchers=pagure_namespace_to_cc[namespace][name],
|
||||||
))
|
))
|
||||||
if env == 'staging':
|
# if self.env["environment"] == 'staging':
|
||||||
# Filter out any modules, since we don't have rights to create new
|
# # Filter out any modules, since we don't have rights to create new
|
||||||
# components in the "Fedora Modules" project yet
|
# # components in the "Fedora Modules" project yet
|
||||||
pagure_projects = [
|
# self.pagure_projects = [
|
||||||
p for p in pagure_projects if p['namespace'] != 'modules'
|
# p for p in self.pagure_projects if p['namespace'] != 'modules'
|
||||||
]
|
# ]
|
||||||
|
|
||||||
branches_url = env['pdc_url'].split('rest_api')[0] + 'extras/active_branches.json'
|
def add_branches_product_and_summary(self):
|
||||||
if VERBOSE:
|
""" For each project retrieved this method adds branches, products
|
||||||
|
and summary information.
|
||||||
|
|
||||||
|
The branches are retrieved from PDC
|
||||||
|
|
||||||
|
The products are determined based on the branches.
|
||||||
|
|
||||||
|
The summary is coming from the primary.xml file from the repodata
|
||||||
|
of the rawhide repository in koji.
|
||||||
|
"""
|
||||||
|
branches_url = "/".join([
|
||||||
|
self.env['pdc_url'].split('rest_api')[0].rstrip("/"),
|
||||||
|
'extras/active_branches.json',
|
||||||
|
])
|
||||||
|
if self.env["verbose"]:
|
||||||
print("Querying %r for EOL information." % branches_url)
|
print("Querying %r for EOL information." % branches_url)
|
||||||
pdc_branches = session.get(branches_url, timeout=120).json()
|
|
||||||
for proj in pagure_projects:
|
|
||||||
if proj['namespace'] not in env['pdc_types']:
|
|
||||||
proj['branches'] = []
|
|
||||||
if VERBOSE:
|
|
||||||
print('! Namespace {namespace} unknown to PDC, project '
|
|
||||||
'{namespace}/{name} ignored'.format(**proj))
|
|
||||||
continue
|
|
||||||
pdc_type = env['pdc_types'][proj['namespace']]
|
|
||||||
proj['branches'] = pdc_branches.get(pdc_type, {}).get(proj['name'], [])
|
|
||||||
if not proj['branches'] and VERBOSE:
|
|
||||||
print("! No PDC branch found for {namespace}/{name}".format(**proj))
|
|
||||||
|
|
||||||
# Determine what products each project maps to based on its branches.
|
pdc_branches = self.session.get(branches_url, timeout=120).json()
|
||||||
# pagure_rpms_project_products will be in the format of
|
for idx, project in enumerate(self.pagure_projects):
|
||||||
# [('python-requests': 'Fedora')...] which will be used my a mapping
|
# Summary
|
||||||
# function below
|
summary = None
|
||||||
for project in pagure_projects:
|
if project["namespace"] == "rpms":
|
||||||
|
summary = self.rpm_summary.get(project["name"])
|
||||||
|
project["summary"] = summary
|
||||||
|
|
||||||
|
# Branches
|
||||||
|
if project['namespace'] not in self.env['pdc_types']:
|
||||||
|
project['branches'] = []
|
||||||
|
project['products'] = []
|
||||||
|
if self.env["verbose"]:
|
||||||
|
print(
|
||||||
|
f'! Namespace {project["namespace"]} not found in the pdc_type '
|
||||||
|
f'configuration key, project {project["namespace"]}/{project["name"]} '
|
||||||
|
'ignored'
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
pdc_type = self.env['pdc_types'][project['namespace']]
|
||||||
|
project['branches'] = pdc_branches.get(pdc_type, {}).get(project['name'], [])
|
||||||
|
if not project['branches']:
|
||||||
|
if self.env["verbose"]:
|
||||||
|
print(f"! No PDC branch found for {project['namespace']}/{project['name']}")
|
||||||
|
|
||||||
|
# Products
|
||||||
products = set()
|
products = set()
|
||||||
for branch, active in project['branches']:
|
for branch, active in project.get('branches'):
|
||||||
if re.match(r'^epel\d+$', branch):
|
if re.match(r'^epel\d+$', branch):
|
||||||
products.add('Fedora EPEL')
|
products.add('Fedora EPEL')
|
||||||
else:
|
else:
|
||||||
products.add(env['namespace_to_product'][project['namespace']])
|
products.add(self.env['namespace_to_product'][project['namespace']])
|
||||||
project['products'] = list(products)
|
project['products'] = list(products)
|
||||||
|
|
||||||
# Now, we must transform the data we collected into something that PkgDB
|
self.pagure_projects[idx] = project
|
||||||
# would have returned
|
|
||||||
p_to_legacy_schema = resilient_partial(_to_legacy_schema, session=session)
|
|
||||||
items = [
|
|
||||||
(product, project, rpm_summary)
|
|
||||||
for project in pagure_projects
|
|
||||||
for product in project['products']
|
|
||||||
]
|
|
||||||
legacy_responses = pool.map(p_to_legacy_schema, items)
|
|
||||||
for response in legacy_responses:
|
|
||||||
if not response:
|
|
||||||
continue
|
|
||||||
projects_dict[response['product']][response['project']] = response
|
|
||||||
|
|
||||||
if VERBOSE:
|
def main(self):
|
||||||
|
"""The entrypoint to the script."""
|
||||||
|
global envname, env, projects_dict
|
||||||
|
times = {
|
||||||
|
"start": time.time(),
|
||||||
|
}
|
||||||
|
|
||||||
|
self.get_cli_arguments()
|
||||||
|
|
||||||
|
load_configuration()
|
||||||
|
self.config = config
|
||||||
|
|
||||||
|
envname = self.config['environment']
|
||||||
|
if self.args.env:
|
||||||
|
if self.args.env in self.config['environments']:
|
||||||
|
envname = self.args.env
|
||||||
|
else:
|
||||||
|
print(f"Invalid environment specified: {self.args.env}")
|
||||||
|
return 1
|
||||||
|
|
||||||
|
self.env = self.config['environments'][envname]
|
||||||
|
|
||||||
|
if self.args.debug:
|
||||||
|
self.env["verbose"] = True
|
||||||
|
self.env["dryrun"] = True
|
||||||
|
|
||||||
|
if self.args.verbose:
|
||||||
|
self.env["verbose"] = True
|
||||||
|
|
||||||
|
if self.args.dryrun:
|
||||||
|
self.env["dryrun"] = True
|
||||||
|
|
||||||
|
# Non-fatal errors to alert people about
|
||||||
|
errors = []
|
||||||
|
|
||||||
|
self.session = retry_session()
|
||||||
|
|
||||||
|
if self.env["verbose"]:
|
||||||
|
print("Building a cache of the rpm packages' summary")
|
||||||
|
self.rpm_summary = package_summary.get_package_summary()
|
||||||
|
|
||||||
|
self.get_pagure_project()
|
||||||
|
self.add_branches_product_and_summary()
|
||||||
|
|
||||||
|
if self.env["verbose"]:
|
||||||
|
print(f"{len(self.pagure_projects)} projects to consider")
|
||||||
|
|
||||||
|
if self.env["verbose"]:
|
||||||
times["data structure end"] = time.time()
|
times["data structure end"] = time.time()
|
||||||
delta = times["data structure end"] - times["start"]
|
delta = times["data structure end"] - times["start"]
|
||||||
print("Ran for %s seconds -- ie: %.2f minutes" % (delta, delta/60.0))
|
print("Ran for %s seconds -- ie: %.2f minutes" % (delta, delta/60.0))
|
||||||
print("Going to update bugzilla now (unless --dry-run)")
|
print("Building FAS' cache")
|
||||||
|
|
||||||
# Initialize the connection to bugzilla
|
# Initialize the connection to bugzilla
|
||||||
bugzilla = BugzillaProxy(env['bugzilla']['url'],
|
bugzilla = BugzillaProxy(self.env['bugzilla']['url'],
|
||||||
env['bugzilla']['user'],
|
self.env['bugzilla']['user'],
|
||||||
env['bugzilla']['password'],
|
self.env['bugzilla']['password'],
|
||||||
projects_dict)
|
self.env)
|
||||||
|
|
||||||
if VERBOSE:
|
if self.env["verbose"]:
|
||||||
times["FAS cache building end"] = time.time()
|
times["FAS cache building end"] = time.time()
|
||||||
delta = times["FAS cache building end"] - times["data structure end"]
|
delta = times["FAS cache building end"] - times["data structure end"]
|
||||||
print(f"Ran for {delta} seconds -- ie: {delta/60} minutes")
|
print(f"Ran for {delta} seconds -- ie: {delta/60} minutes")
|
||||||
print("bugzilla connection set!")
|
if self.env["dryrun"]:
|
||||||
|
print("Querying bugzilla but not doing anything")
|
||||||
|
else:
|
||||||
|
print("Updating bugzilla")
|
||||||
|
|
||||||
for product, pkgs in projects_dict.items():
|
bugzilla.build_product_cache(self.pagure_projects)
|
||||||
if product not in env['products']:
|
for project in sorted(self.pagure_projects, key=itemgetter('name')):
|
||||||
if VERBOSE:
|
for product in project["products"]:
|
||||||
print(f"Ignoring: {product}")
|
if product not in self.env['products']:
|
||||||
|
if self.env["verbose"]:
|
||||||
|
print(f"Ignoring: {product}/{project['name']}")
|
||||||
continue
|
continue
|
||||||
for pkgname, pkginfo in sorted(projects_dict[product].items(),
|
|
||||||
key=lambda x: x[0]):
|
|
||||||
try:
|
try:
|
||||||
bugzilla.add_edit_component(
|
bugzilla.add_edit_component(
|
||||||
pkgname,
|
package=project["name"],
|
||||||
product,
|
collection=product,
|
||||||
pkginfo['owner'],
|
owner=project['poc'],
|
||||||
pkginfo['summary'],
|
description=project['summary'],
|
||||||
pkginfo['qacontact'],
|
qacontact=None,
|
||||||
pkginfo['cclist']
|
cclist=project['watchers']
|
||||||
)
|
)
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
# A username didn't have a bugzilla address
|
# A username didn't have a bugzilla address
|
||||||
|
@ -701,25 +691,25 @@ def main():
|
||||||
except xmlrpc.client.Error as e:
|
except xmlrpc.client.Error as e:
|
||||||
# An error occurred in the xmlrpc call. Shouldn't happen but
|
# An error occurred in the xmlrpc call. Shouldn't happen but
|
||||||
# we better see what it is
|
# we better see what it is
|
||||||
errors.append('%s -- %s' % (pkgname, e.args[-1]))
|
errors.append('%s -- %s' % (project["name"], e.args[-1]))
|
||||||
|
|
||||||
# Send notification of errors
|
# Send notification of errors
|
||||||
if errors:
|
if errors:
|
||||||
if VERBOSE:
|
if self.env["verbose"] or self.env["dryrun"]:
|
||||||
print('[DEBUG]', '\n'.join(errors))
|
print('[DEBUG]', '\n'.join(errors))
|
||||||
else:
|
else:
|
||||||
notify_users(errors)
|
self.notify_users(errors)
|
||||||
send_email(
|
send_email(
|
||||||
env['email_from'],
|
self.env['email_from'],
|
||||||
env['notify_emails'],
|
self.env['notify_emails'],
|
||||||
'Errors while syncing bugzilla with the PackageDB',
|
'Errors while syncing bugzilla with the PackageDB',
|
||||||
env['tmpl_admin_email'].format(errors='\n'.join(errors))
|
self.env['tmpl_admin_email'].format(errors='\n'.join(errors))
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
with open(env['data_cache'], 'w') as stream:
|
with open(self.env['data_cache'], 'w') as stream:
|
||||||
json.dump({}, stream)
|
json.dump({}, stream)
|
||||||
|
|
||||||
if VERBOSE:
|
if self.env["verbose"]:
|
||||||
times["end"] = time.time()
|
times["end"] = time.time()
|
||||||
|
|
||||||
print(" ----------")
|
print(" ----------")
|
||||||
|
@ -739,8 +729,9 @@ def main():
|
||||||
delta = times["end"] - times["start"]
|
delta = times["end"] - times["start"]
|
||||||
print(f" Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")
|
print(f" Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")
|
||||||
|
|
||||||
sys.exit(0)
|
return 0
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
main()
|
dbs = DistgitBugzillaSync()
|
||||||
|
sys.exit(dbs.main())
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
#!/usr/bin/python3 -tt
|
#!/usr/bin/python3 -tt
|
||||||
|
|
||||||
from distgit_bugzilla_sync.script import main
|
from distgit_bugzilla_sync.script import DistgitBugzillaSync
|
||||||
|
|
||||||
main()
|
DistgitBugzillaSync().main()
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue