1206 lines
46 KiB
Python
1206 lines
46 KiB
Python
# -*- coding: utf-8 -*-
|
|
#
|
|
# Copyright © 2013-2019 Red Hat, Inc.
|
|
#
|
|
# This copyrighted material is made available to anyone wishing to use, modify,
|
|
# copy, or redistribute it subject to the terms and conditions of the GNU
|
|
# General Public License v.2, or (at your option) any later version. This
|
|
# program is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
# WARRANTY expressed or implied, including the implied warranties of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
|
|
# Public License for more details. You should have received a copy of the GNU
|
|
# General Public License along with this program; if not, write to the Free
|
|
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the source
|
|
# code or documentation are not subject to the GNU General Public License and
|
|
# may only be used or replicated with the express permission of Red Hat, Inc.
|
|
#
|
|
# Red Hat Author(s): Toshio Kuratomi <tkuratom@redhat.com>
|
|
# Author(s): Mike Watters <valholla75@fedoraproject.org>
|
|
# Author(s): Pierre-Yves Chibon <pingou@pingoured.fr>
|
|
# Author(s): Matt Prahl <mprahl@redhat.com>
|
|
# Author(s): Ralph Bean <rbean@redhat.com
|
|
# Author(s): Patrick Uiterwijk <puiterwijk@redhat.com>
|
|
#
|
|
"""
|
|
sync information from the Pagure into bugzilla
|
|
|
|
This ... script takes information about package onwership and imports it
|
|
into bugzilla.
|
|
"""
|
|
|
|
import argparse
|
|
import collections
|
|
import datetime
|
|
from email.message import EmailMessage
|
|
import itertools
|
|
import json
|
|
import math
|
|
from operator import itemgetter
|
|
import os
|
|
import re
|
|
import smtplib
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import xmlrpc.client
|
|
|
|
from bugzilla import Bugzilla
|
|
import dogpile.cache
|
|
import fedora.client
|
|
from fedora.client.fas2 import AccountSystem
|
|
import requests
|
|
from requests.adapters import HTTPAdapter
|
|
from urllib3.util import Retry
|
|
|
|
from . import package_summaries
|
|
from .config import config, email_overrides, load_configuration
|
|
|
|
|
|
cache = dogpile.cache.make_region().configure(
|
|
"dogpile.cache.memory", expiration_time=3600,
|
|
)
|
|
|
|
|
|
def retry_session():
|
|
session = requests.Session()
|
|
retry = Retry(
|
|
total=5,
|
|
read=5,
|
|
connect=5,
|
|
backoff_factor=0.3,
|
|
status_forcelist=(500, 502, 504),
|
|
)
|
|
adapter = HTTPAdapter(max_retries=retry)
|
|
session.mount("http://", adapter)
|
|
session.mount("https://", adapter)
|
|
return session
|
|
|
|
|
|
def resilient_partial(fn, *initial, **kwargs):
|
|
""" A decorator that partially applies arguments.
|
|
|
|
It additionally catches all raised exceptions, prints them, but then returns
|
|
None instead of propagating the failures.
|
|
|
|
This is used to protect functions used in a threadpool. If one fails, we
|
|
want to know about it, but we don't want it to kill the whole program. So
|
|
catch its error, log it, but proceed.
|
|
"""
|
|
|
|
def wrapper(*additional):
|
|
try:
|
|
full = initial + additional
|
|
return fn(*full, **kwargs)
|
|
except Exception:
|
|
traceback.print_exc()
|
|
return None
|
|
|
|
wrapper.__name__ = fn.__name__
|
|
wrapper.__doc__ = fn.__doc__
|
|
return wrapper
|
|
|
|
|
|
class DataChangedError(Exception):
|
|
"""Raised when data we are manipulating changes while we're modifying it."""
|
|
|
|
pass
|
|
|
|
|
|
def segment(iterable, chunk, fill=None):
|
|
"""Collect data into `chunk` sized block"""
|
|
args = [iter(iterable)] * chunk
|
|
return itertools.zip_longest(*args, fillvalue=fill)
|
|
|
|
|
|
class BugzillaProxy:
|
|
|
|
def __init__(self, bz_server, username, password, config, pre_cache_users=True):
|
|
self.bz_xmlrpc_server = bz_server
|
|
self.username = username
|
|
self.password = password
|
|
|
|
self.server = Bugzilla(
|
|
url=self.bz_xmlrpc_server, user=self.username, password=self.password
|
|
)
|
|
|
|
self.product_cache = {}
|
|
self.user_cache = {}
|
|
self.inverted_user_cache = {}
|
|
self.errors = []
|
|
|
|
# Connect to the fedora account system
|
|
self.fas = AccountSystem(
|
|
base_url=config["fas"]["url"],
|
|
username=config["fas"]["username"],
|
|
password=config["fas"]["password"],
|
|
)
|
|
|
|
self.config = config
|
|
|
|
if not pre_cache_users:
|
|
return
|
|
|
|
if config["verbose"]:
|
|
print("Pre-caching FAS users and their Bugzilla email addresses.")
|
|
|
|
try:
|
|
self.user_cache = self.fas.people_by_key(
|
|
key="username", fields=["email"]
|
|
)
|
|
except fedora.client.ServerError:
|
|
# Sometimes, building the user cache up front fails with a timeout.
|
|
# It's ok, we build the cache as-needed later in the script.
|
|
pass
|
|
else:
|
|
self.invert_user_cache()
|
|
|
|
def build_product_cache(self, pagure_projects):
|
|
""" Cache the bugzilla info about each package in each product.
|
|
"""
|
|
|
|
def _query_component(query, num_attempts=5):
|
|
for i in range(num_attempts):
|
|
try:
|
|
raw_data = self.server._proxy.Component.get({"names": query})
|
|
break
|
|
except Exception as e:
|
|
if i >= num_attempts - 1:
|
|
raise
|
|
if self.config["verbose"]:
|
|
print(f" ERROR {e}")
|
|
print(" - Query failed, going to try again in 10 seconds")
|
|
# Wait 10 seconds and try again
|
|
time.sleep(10)
|
|
return raw_data
|
|
|
|
if self.config["bugzilla"]["compat_api"] == "getcomponentsdetails":
|
|
# Old API -- in python-bugzilla. But with current server, this
|
|
# gives ProxyError
|
|
for collection, product in self.config["products"].items():
|
|
self.product_cache[collection] = self.server.getcomponentsdetails(
|
|
collection
|
|
)
|
|
elif self.config["bugzilla"]["compat_api"] == "component.get":
|
|
# Way that's undocumented in the partner-bugzilla api but works
|
|
# currently
|
|
chunk = self.config["bugzilla"]["req_segment"]
|
|
for collection, product in self.config["products"].items():
|
|
|
|
# restrict the list of info returned to only the packages of
|
|
# interest
|
|
pkglist = [
|
|
project["name"]
|
|
for project in pagure_projects
|
|
if collection in project["products"]
|
|
]
|
|
product_info_by_pkg = {}
|
|
estimated_size = math.ceil(len(pkglist) / chunk)
|
|
for cnt, pkg_segment in enumerate(segment(pkglist, chunk), start=1):
|
|
# Format that bugzilla will understand. Strip None's that
|
|
# segment() pads out the final data segment() with
|
|
query = [
|
|
{"product": collection, "component": p}
|
|
for p in pkg_segment
|
|
if p is not None
|
|
]
|
|
if self.config["verbose"]:
|
|
print(
|
|
f" - Querying product `{collection}`, "
|
|
f"query {cnt} of {estimated_size}"
|
|
)
|
|
raw_data = _query_component(query)
|
|
for package in raw_data["components"]:
|
|
# Reformat data to be the same as what's returned from
|
|
# getcomponentsdetails
|
|
product_info = {
|
|
"initialowner": package["default_assignee"],
|
|
"description": package["description"],
|
|
"initialqacontact": package["default_qa_contact"],
|
|
"initialcclist": package["default_cc"],
|
|
"is_active": package["is_active"],
|
|
}
|
|
product_info_by_pkg[package["name"].lower()] = product_info
|
|
self.product_cache[collection] = product_info_by_pkg
|
|
|
|
def invert_user_cache(self):
|
|
""" Takes the user_cache built when querying FAS and invert it so
|
|
that the bugzilla_email is the key and the username the value.
|
|
"""
|
|
for username in self.user_cache:
|
|
bz_email = self.user_cache[username]["email"].lower()
|
|
self.inverted_user_cache[bz_email] = username
|
|
|
|
def _get_bugzilla_email(self, username):
|
|
"""Return the bugzilla email address for a user.
|
|
|
|
First looks in a cache for a username => bugzilla email. If not found,
|
|
reloads the cache from fas and tries again.
|
|
"""
|
|
try:
|
|
bz_email = self.user_cache[username]["email"].lower()
|
|
except KeyError:
|
|
if username.startswith("@"):
|
|
group = self.fas.group_by_name(username[1:])
|
|
bz_email = group.mailing_list
|
|
if bz_email is None:
|
|
return
|
|
self.user_cache[username] = {"email": bz_email}
|
|
else:
|
|
person = self.fas.person_by_username(username)
|
|
bz_email = person.get("email")
|
|
if bz_email is None:
|
|
return
|
|
self.user_cache[username] = {"email": bz_email}
|
|
bz_email = bz_email.lower()
|
|
self.inverted_user_cache[bz_email] = username
|
|
bz_email = email_overrides.get(bz_email, bz_email)
|
|
return bz_email.strip()
|
|
|
|
def update_open_bugs(self, new_poc, prev_poc, product, name, print_fas_names=False):
|
|
"""Change the package owner
|
|
:arg new_poc: email of the new point of contact.
|
|
:arg prev_poc: Username of the previous point of contact
|
|
:arg product: The product of the package to change in bugzilla
|
|
:arg name: Name of the package to change the owner.
|
|
:kwarg print_fas_names: Boolean specifying wether to print email or FAS names
|
|
(if these could be found).
|
|
"""
|
|
bz_query = {}
|
|
bz_query["product"] = product
|
|
bz_query["component"] = name
|
|
bz_query["bug_status"] = [
|
|
"NEW",
|
|
"ASSIGNED",
|
|
"ON_DEV",
|
|
"ON_QA",
|
|
"MODIFIED",
|
|
"POST",
|
|
"FAILS_QA",
|
|
"PASSES_QA",
|
|
"RELEASE_PENDING",
|
|
]
|
|
# Update only maintained releases
|
|
bz_query["version"] = self.config["products"][product]["versions"]
|
|
|
|
def _query_bz(query, num_attempts=5):
|
|
for i in range(num_attempts):
|
|
try:
|
|
raw_data = self.server.query(bz_query)
|
|
break
|
|
except Exception as e:
|
|
if i >= num_attempts - 1:
|
|
raise
|
|
if self.config["verbose"]:
|
|
print(f" ERROR {e}")
|
|
print(" - Query failed, going to try again in 20 seconds")
|
|
# Wait 20 seconds and try again
|
|
time.sleep(20)
|
|
return raw_data
|
|
|
|
query_results = _query_bz(bz_query)
|
|
|
|
for bug in query_results:
|
|
if bug.assigned_to == prev_poc and bug.assigned_to != new_poc:
|
|
if self.config["verbose"]:
|
|
old_poc = bug.assigned_to
|
|
if print_fas_names:
|
|
if old_poc in self.inverted_user_cache:
|
|
old_poc = self.inverted_user_cache[old_poc]
|
|
else:
|
|
old_poc = old_poc.split("@", 1)[0] + "@..."
|
|
if new_poc in self.inverted_user_cache:
|
|
new_poc = self.inverted_user_cache[new_poc]
|
|
else:
|
|
new_poc = new_poc.split("@", 1)[0] + "@..."
|
|
print(
|
|
f"[UPDATEBUG] {product}/{name} reassigning bug #{bug.bug_id} "
|
|
f"from {old_poc} to {new_poc}"
|
|
)
|
|
if not self.config["dryrun"]:
|
|
try:
|
|
bug.setassignee(
|
|
assigned_to=new_poc,
|
|
comment=self.config["bz_maintainer_change_comment"],
|
|
)
|
|
except xmlrpc.client.Fault as e:
|
|
# Output something useful in args
|
|
e.args = (new_poc, e.faultCode, e.faultString)
|
|
raise
|
|
except xmlrpc.client.ProtocolError as e:
|
|
e.args = ("ProtocolError", e.errcode, e.errmsg)
|
|
raise
|
|
|
|
def add_edit_component(
|
|
self,
|
|
package,
|
|
collection,
|
|
owner,
|
|
description=None,
|
|
qacontact=None,
|
|
cclist=None,
|
|
print_fas_names=False,
|
|
retired=False,
|
|
):
|
|
"""Add or update a component to have the values specified.
|
|
"""
|
|
# Turn the cclist into something usable by bugzilla
|
|
initial_cc_emails = []
|
|
initial_cc_emails_lower = []
|
|
initial_cc_fasnames = []
|
|
for watcher in cclist:
|
|
bz_email = self._get_bugzilla_email(watcher)
|
|
if bz_email:
|
|
initial_cc_emails.append(bz_email)
|
|
initial_cc_emails_lower.append(bz_email.lower())
|
|
initial_cc_fasnames.append(watcher)
|
|
else:
|
|
self.errors.append(
|
|
f"{watcher} has no bugzilla_email or mailing_list set on "
|
|
f"({collection}/{package})"
|
|
)
|
|
if self.config["verbose"]:
|
|
print(
|
|
f"** {watcher} has no bugzilla_email or mailing_list set "
|
|
f"({collection}/{package}) **"
|
|
)
|
|
|
|
# Add owner to the cclist so comaintainers taking over a bug don't
|
|
# have to do this manually
|
|
owner_email = self._get_bugzilla_email(owner)
|
|
if owner_email.lower() not in initial_cc_emails_lower:
|
|
initial_cc_emails.append(owner_email)
|
|
initial_cc_emails_lower.append(owner_email.lower())
|
|
initial_cc_fasnames.append(owner)
|
|
|
|
# Lookup product
|
|
try:
|
|
product = self.product_cache[collection]
|
|
except xmlrpc.client.Fault as e:
|
|
# Output something useful in args
|
|
e.args = (e.faultCode, e.faultString)
|
|
raise
|
|
except xmlrpc.client.ProtocolError as e:
|
|
e.args = ("ProtocolError", e.errcode, e.errmsg)
|
|
raise
|
|
|
|
|
|
# Set the qacontact_email and name
|
|
default_qa_contact_email = self.config["default_qa_contact_email"]
|
|
default_qa_contact = (
|
|
f"<default: {default_qa_contact_email.split('@', 1)[0]}@...>"
|
|
)
|
|
if qacontact:
|
|
qacontact_email = self._get_bugzilla_email(qacontact).strip()
|
|
else:
|
|
qacontact = default_qa_contact
|
|
qacontact_email = default_qa_contact_email.strip()
|
|
|
|
pkg_key = package.lower()
|
|
if pkg_key in product:
|
|
# edit the package information
|
|
data = {}
|
|
|
|
# Check for changes to the owner, qacontact, or description
|
|
if product[pkg_key]["initialowner"].lower() != owner_email.lower():
|
|
data["initialowner"] = owner_email
|
|
|
|
if description and product[pkg_key]["description"] != description:
|
|
data["description"] = description
|
|
|
|
if (
|
|
qacontact
|
|
and product[pkg_key]["initialqacontact"].lower()
|
|
!= qacontact_email.lower()
|
|
):
|
|
data["initialqacontact"] = qacontact_email
|
|
|
|
if len(product[pkg_key]["initialcclist"]) != len(initial_cc_emails):
|
|
data["initialcclist"] = initial_cc_emails
|
|
else:
|
|
for cc_member in product[pkg_key]["initialcclist"]:
|
|
if cc_member.lower() not in initial_cc_emails_lower:
|
|
data["initialcclist"] = initial_cc_emails
|
|
break
|
|
|
|
if product[pkg_key]["is_active"] != (not retired):
|
|
data["is_active"] = not retired
|
|
|
|
if data:
|
|
# Changes occurred. Submit a request to change via xmlrpc
|
|
data["product"] = collection
|
|
data["component"] = package
|
|
|
|
if self.config["verbose"]:
|
|
for key in [
|
|
"initialowner",
|
|
"description",
|
|
"initialqacontact",
|
|
"initialcclist",
|
|
"is_active",
|
|
]:
|
|
if data.get(key) is not None:
|
|
|
|
old_value = product[pkg_key][key]
|
|
if isinstance(old_value, list):
|
|
old_value = sorted(old_value)
|
|
new_value = data.get(key)
|
|
if isinstance(new_value, list):
|
|
new_value = sorted(new_value)
|
|
|
|
if print_fas_names and key in (
|
|
"initialowner",
|
|
"initialqacontact",
|
|
"initialcclist",
|
|
):
|
|
if key == "initialowner":
|
|
new_value = owner
|
|
elif key == "initialqacontact":
|
|
new_value = qacontact
|
|
else:
|
|
new_value = sorted(initial_cc_fasnames)
|
|
|
|
from_fas_names = []
|
|
for email in product[pkg_key][key]:
|
|
if email in self.inverted_user_cache:
|
|
from_fas_names.append(
|
|
self.inverted_user_cache[email]
|
|
)
|
|
elif email == default_qa_contact_email:
|
|
from_fas_names.append(default_qa_contact)
|
|
if from_fas_names:
|
|
if len(from_fas_names) < len(product[pkg_key][key]):
|
|
x = len(product[pkg_key][key]) - len(
|
|
from_fas_names
|
|
)
|
|
from_fas_names.append(f"And {x} more")
|
|
old_value = f"from `{from_fas_names}`"
|
|
else:
|
|
old_value = ""
|
|
print(
|
|
f"[EDITCOMP] {data['product']}/{data['component']}"
|
|
f" {key} changed {old_value} to FAS name(s) `{new_value}`"
|
|
)
|
|
else:
|
|
print(
|
|
f"[EDITCOMP] {data['product']}/{data['component']}"
|
|
f" {key} changed from `{old_value}` to `{new_value}`"
|
|
)
|
|
|
|
owner_changed = "initialowner" in data
|
|
|
|
# FIXME: initialowner has been made mandatory for some
|
|
# reason. Asking dkl why.
|
|
data["initialowner"] = owner_email
|
|
|
|
def edit_component(data, num_attempts=5):
|
|
for i in range(num_attempts):
|
|
try:
|
|
self.server.editcomponent(data)
|
|
break
|
|
except Exception as e:
|
|
if (
|
|
isinstance(e, xmlrpc.client.Fault)
|
|
and e.faultCode == 504
|
|
):
|
|
if self.config["verbose"]:
|
|
print(f" ERROR {e}")
|
|
raise
|
|
if i >= num_attempts - 1:
|
|
raise
|
|
if self.config["verbose"]:
|
|
print(f" ERROR {e}")
|
|
print(
|
|
" - Query failed, going to try again in 20 seconds"
|
|
)
|
|
# Wait 20 seconds and try again
|
|
time.sleep(20)
|
|
|
|
if not self.config["dryrun"]:
|
|
try:
|
|
edit_component(data)
|
|
except xmlrpc.client.Fault as e:
|
|
# Output something useful in args
|
|
e.args = (data, e.faultCode, e.faultString)
|
|
raise
|
|
except xmlrpc.client.ProtocolError as e:
|
|
e.args = ("ProtocolError", e.errcode, e.errmsg)
|
|
raise
|
|
|
|
if owner_changed:
|
|
self.update_open_bugs(
|
|
new_poc=owner_email,
|
|
prev_poc=product[pkg_key]["initialowner"],
|
|
name=package,
|
|
product=collection,
|
|
print_fas_names=print_fas_names,
|
|
)
|
|
else:
|
|
if self.config.get("print-no-change"):
|
|
print(f"[NOCHANGE] {package}/{collection}")
|
|
else:
|
|
if retired:
|
|
if self.config["verbose"]:
|
|
print(f"[NOADD] {collection}/{package} (is retired)")
|
|
return
|
|
|
|
# Add component
|
|
data = {
|
|
"product": collection,
|
|
"component": package,
|
|
"description": description or "NA",
|
|
"initialowner": owner_email,
|
|
"initialqacontact": qacontact_email,
|
|
"is_active": not retired,
|
|
}
|
|
if initial_cc_emails:
|
|
data["initialcclist"] = initial_cc_emails
|
|
|
|
if self.config["verbose"]:
|
|
for key in [
|
|
"initialowner",
|
|
"description",
|
|
"initialqacontact",
|
|
"initialcclist",
|
|
"is_active",
|
|
]:
|
|
if print_fas_names and key in (
|
|
"initialowner",
|
|
"initialqacontact",
|
|
"initialcclist",
|
|
):
|
|
if key == "initialowner":
|
|
value = owner
|
|
elif key == "initialqacontact":
|
|
value = qacontact
|
|
else:
|
|
value = initial_cc_fasnames
|
|
print(
|
|
f"[ADDCOMP] {collection}/{package}"
|
|
f" {key} set to FAS name(s) `{value}`"
|
|
)
|
|
else:
|
|
print(
|
|
f"[ADDCOMP] {collection}/{package}"
|
|
f" {key} set to {data.get(key)}"
|
|
)
|
|
|
|
def add_component(data, num_attempts=5):
|
|
for i in range(num_attempts):
|
|
try:
|
|
self.server.addcomponent(data)
|
|
break
|
|
except Exception as e:
|
|
if isinstance(e, xmlrpc.client.Fault) and e.faultCode in [
|
|
504,
|
|
1200,
|
|
]:
|
|
# error 504: user <email> is not a valid username
|
|
# error 1200: Product <product> has already a component named <name>
|
|
if self.config["verbose"]:
|
|
print(f" ERROR {e}")
|
|
raise
|
|
if i >= num_attempts - 1:
|
|
raise
|
|
if self.config["verbose"]:
|
|
print(f" ERROR {e}")
|
|
print(
|
|
" - Query failed, going to try again in 20 seconds"
|
|
)
|
|
# Wait 20 seconds and try again
|
|
time.sleep(20)
|
|
|
|
if not self.config["dryrun"]:
|
|
try:
|
|
add_component(data)
|
|
except xmlrpc.client.Fault as e:
|
|
# Output something useful in args
|
|
e.args = (data, e.faultCode, e.faultString)
|
|
raise
|
|
|
|
|
|
def _get_pdc_branches(session, repo):
|
|
"""
|
|
Gets the branches on a project. This function is used for mapping.
|
|
:param repo: the project dict
|
|
:return: a list of the repo's branches
|
|
"""
|
|
branches_url = "{0}component-branches/".format(env["pdc_url"])
|
|
params = {
|
|
"global_component": repo["name"],
|
|
"type": env["pdc_types"][repo["namespace"]],
|
|
}
|
|
if config["verbose"]:
|
|
print("Querying {0} {1}".format(branches_url, params))
|
|
rv = session.get(branches_url, params=params, timeout=60)
|
|
|
|
# If the project's branches can't be reported, just return no branches and
|
|
# it will be skipped later on
|
|
if not rv.ok:
|
|
print(
|
|
(
|
|
'The connection to "{0}" failed with the status code {1} and '
|
|
'output "{2}"'.format(branches_url, rv.status_code, rv.text)
|
|
),
|
|
file=sys.stderr,
|
|
)
|
|
return []
|
|
|
|
data = rv.json()
|
|
return [branch["name"] for branch in data["results"]]
|
|
|
|
|
|
class ScriptExecError(RuntimeError):
|
|
def __init__(self, *args, **kwargs):
|
|
self.errorcode = kwargs.pop("errorcode", 1)
|
|
super().__init__(*args, **kwargs)
|
|
|
|
|
|
class DistgitBugzillaSync:
|
|
|
|
# cache placeholders for properties which are computed once
|
|
_namespace_to_product = None
|
|
_product_to_branch_regex = None
|
|
_branch_regex_to_product = None
|
|
|
|
def send_email(self, from_address, to_address, subject, message, cc_address=None):
|
|
"""Send an email if there's an error.
|
|
|
|
This will be replaced by sending messages to a log later.
|
|
"""
|
|
if not self.env["email"]["send_mails"]:
|
|
return
|
|
|
|
msg = EmailMessage()
|
|
msg.add_header("To", ",".join(to_address))
|
|
msg.add_header("From", from_address)
|
|
msg.add_header("Subject", subject)
|
|
if cc_address is not None:
|
|
msg.add_header("Cc", ",".join(cc_address))
|
|
to_address += cc_address
|
|
msg.set_payload(message)
|
|
smtp = smtplib.SMTP(self.env["email"]["smtp_host"])
|
|
smtp.sendmail(from_address, to_address, msg.as_string())
|
|
smtp.quit()
|
|
|
|
def notify_users(self, errors):
|
|
""" Browse the list of errors and when we can retrieve the email
|
|
address, use it to notify the user about the issue.
|
|
"""
|
|
data = {}
|
|
if os.path.exists(self.env["data_cache"]):
|
|
try:
|
|
with open(self.env["data_cache"]) as stream:
|
|
data = json.load(stream)
|
|
except Exception as err:
|
|
print(
|
|
"Could not read the json file at %s: \nError: %s"
|
|
% (env["data_cache"], err)
|
|
)
|
|
|
|
new_data = {}
|
|
seen = []
|
|
for error in errors:
|
|
notify_user = False
|
|
if "The name " in error and " is not a valid username" in error:
|
|
user_email = (
|
|
error.split(" is not a valid username")[0]
|
|
.split("The name ")[1]
|
|
.strip()
|
|
)
|
|
now = datetime.datetime.utcnow()
|
|
|
|
# See if we already know about this user
|
|
if user_email in data and data[user_email]["last_update"]:
|
|
last_update = datetime.datetime.fromtimestamp(
|
|
int(data[user_email]["last_update"])
|
|
)
|
|
# Only notify users once per hour
|
|
if (now - last_update).seconds >= 3600:
|
|
notify_user = True
|
|
else:
|
|
new_data[user_email] = data[user_email]
|
|
elif not data or user_email not in data:
|
|
notify_user = True
|
|
|
|
# Ensure we notify the user only once, no matter how many errors we
|
|
# got concerning them.
|
|
if user_email not in seen:
|
|
seen.append(user_email)
|
|
else:
|
|
notify_user = False
|
|
|
|
if notify_user:
|
|
self.send_email(
|
|
self.env["email"]["from"],
|
|
[user_email],
|
|
subject="Please fix your bugzilla.redhat.com account",
|
|
message=self.env["email"]["templates"]["user_notification"],
|
|
cc_address=self.env["email"]["notify_admins"],
|
|
)
|
|
|
|
new_data[user_email] = {"last_update": time.mktime(now.timetuple())}
|
|
|
|
with open(env["data_cache"], "w") as stream:
|
|
json.dump(new_data, stream)
|
|
|
|
def get_cli_arguments(self):
|
|
""" Set the CLI argument parser and return the argument parsed.
|
|
"""
|
|
parser = argparse.ArgumentParser(
|
|
description="Script syncing information between Pagure and bugzilla"
|
|
)
|
|
parser.add_argument(
|
|
"--dry-run",
|
|
dest="dryrun",
|
|
action="store_true",
|
|
default=False,
|
|
help="Do not actually make any changes - Overrides the configuration",
|
|
)
|
|
parser.add_argument(
|
|
"--verbose",
|
|
dest="verbose",
|
|
action="store_true",
|
|
default=False,
|
|
help="Print actions verbosely - Overrides the configuration",
|
|
)
|
|
parser.add_argument(
|
|
"--debug",
|
|
dest="debug",
|
|
action="store_true",
|
|
default=False,
|
|
help="Combination of --verbose and --dry-run",
|
|
)
|
|
parser.add_argument(
|
|
"--env",
|
|
dest="env",
|
|
help="Run the script for a specific environment, overrides the one "
|
|
"set in the configuration file",
|
|
)
|
|
parser.add_argument(
|
|
"--add-config-file",
|
|
metavar="CONFIG_FILE",
|
|
dest="addl_config_files",
|
|
action="append",
|
|
help="File(s) from which to read overriding configuration",
|
|
)
|
|
parser.add_argument(
|
|
"--add-email-overrides-file",
|
|
metavar="EMAIL_OVERRIDES_FILE",
|
|
dest="addl_email_overrides_files",
|
|
action="append",
|
|
help="File(s) from which to read additional email overrides",
|
|
)
|
|
parser.add_argument(
|
|
"-p",
|
|
"--project",
|
|
dest="projects",
|
|
nargs="+",
|
|
help="Update one or more projects (provided as namespace/name), "
|
|
"in all of its products",
|
|
)
|
|
parser.add_argument(
|
|
"--print-fas-names",
|
|
action="store_true",
|
|
default=False,
|
|
help="Print FAS names rather than email addresses in output, useful when pasting into "
|
|
"public fora",
|
|
)
|
|
parser.add_argument(
|
|
"--print-no-change",
|
|
action="store_true",
|
|
default=False,
|
|
help="Print elements that are not being changed as they are checked",
|
|
)
|
|
parser.add_argument(
|
|
"--no-user-notifications",
|
|
dest="user_notifications",
|
|
action="store_false",
|
|
default=True,
|
|
help="Do not notify every packager whose account is wrongly set-up, but do send the "
|
|
"overall report to the admins",
|
|
)
|
|
|
|
self.args = parser.parse_args()
|
|
|
|
def get_pagure_projects(self, project_list=None):
|
|
""" Builds a large list of all the projects on pagure.
|
|
Each item in that list is a dict containing:
|
|
- the namespace of the project
|
|
- the name of the project
|
|
- the point of contact of this project (ie: the default assignee
|
|
in bugzilla)
|
|
- the watchers of this project (ie: the initial CC list in bugzilla)
|
|
"""
|
|
|
|
# Get the initial ownership and CC data from pagure
|
|
# This part is easy.
|
|
poc_url = self.env["distgit_url"] + "/extras/pagure_poc.json"
|
|
if self.env["verbose"]:
|
|
print("Querying %r for points of contact." % poc_url)
|
|
pagure_namespace_to_poc = self.session.get(poc_url, timeout=120).json()
|
|
|
|
cc_url = self.env["distgit_url"] + "/extras/pagure_bz.json"
|
|
if self.env["verbose"]:
|
|
print("Querying %r for initial cc list." % cc_url)
|
|
pagure_namespace_to_cc = self.session.get(cc_url, timeout=120).json()
|
|
|
|
# Combine and collapse those two into a single list:
|
|
self.pagure_projects = []
|
|
if project_list:
|
|
project_list = {tuple(p.split("/", 1)) for p in project_list}
|
|
for namespace, entries in pagure_namespace_to_poc.items():
|
|
for name, poc in entries.items():
|
|
if not project_list or (namespace, name) in project_list:
|
|
self.pagure_projects.append(
|
|
{
|
|
"namespace": namespace,
|
|
"name": name,
|
|
"poc": poc["fedora"],
|
|
"epelpoc": poc["epel"],
|
|
"watchers": pagure_namespace_to_cc[namespace][name],
|
|
}
|
|
)
|
|
|
|
@property
|
|
def namespace_to_product(self):
|
|
if self._namespace_to_product is None:
|
|
self._namespace_to_product = {
|
|
p["namespace"]: n
|
|
for n, p in self.env["products"].items()
|
|
if "namespace" in p
|
|
}
|
|
return self._namespace_to_product
|
|
|
|
@property
|
|
def product_to_branch_regex(self):
|
|
if self._product_to_branch_regex is None:
|
|
self._product_to_branch_regex = {
|
|
n: re.compile(p["branch_regex"])
|
|
for n, p in self.env["products"].items()
|
|
if "branch_regex" in p
|
|
}
|
|
return self._product_to_branch_regex
|
|
|
|
@property
|
|
def branch_regex_to_product(self):
|
|
if self._branch_regex_to_product is None:
|
|
self._branch_regex_to_product = {
|
|
n: r for r, n in self.product_to_branch_regex.items()
|
|
}
|
|
return self._branch_regex_to_product
|
|
|
|
def _is_retired(self, product, project):
|
|
branches = project["branches"]
|
|
branch_regex = self.product_to_branch_regex.get(product)
|
|
if branch_regex:
|
|
for branch, active in branches:
|
|
if branch_regex.match(branch) and active:
|
|
return False
|
|
# No active branches means it is retired.
|
|
return True
|
|
else:
|
|
for branch, active in branches:
|
|
if active:
|
|
return False
|
|
return True
|
|
|
|
def add_branches_products_and_summaries(self):
|
|
""" For each project retrieved, this method adds branches, products
|
|
and summary information.
|
|
|
|
The branches are retrieved from PDC.
|
|
|
|
The products are determined based on the branches.
|
|
|
|
The summaries are coming from the primary.xml file of the Rawhide repodata
|
|
in Koji.
|
|
"""
|
|
branches_url = "/".join(
|
|
[
|
|
self.env["pdc_url"].split("rest_api")[0].rstrip("/"),
|
|
"extras/active_branches.json",
|
|
]
|
|
)
|
|
if self.env["verbose"]:
|
|
print("Querying %r for EOL information." % branches_url)
|
|
|
|
pdc_branches = self.session.get(branches_url, timeout=120).json()
|
|
for idx, project in enumerate(self.pagure_projects):
|
|
# Summary
|
|
summary = None
|
|
if project["namespace"] == "rpms":
|
|
summary = self.rpm_summaries.get(project["name"])
|
|
project["summary"] = summary
|
|
|
|
# Branches
|
|
if project["namespace"] not in self.env["pdc_types"]:
|
|
project["branches"] = []
|
|
project["products"] = []
|
|
error = f'Namespace `{project["namespace"]}` not found in the pdc_type ' \
|
|
f'configuration key -- ignoring it'
|
|
if not error in self.errors["configuration"]:
|
|
self.errors["configuration"].append(error)
|
|
if self.env["verbose"]:
|
|
print(
|
|
f'Namespace `{project["namespace"]}` not found in the pdc_type '
|
|
f'configuration key, project {project["namespace"]}/{project["name"]} '
|
|
"ignored"
|
|
)
|
|
continue
|
|
|
|
pdc_type = self.env["pdc_types"][project["namespace"]]
|
|
project["branches"] = pdc_branches.get(pdc_type, {}).get(
|
|
project["name"], []
|
|
)
|
|
if not project["branches"]:
|
|
self.errors["PDC"].append(
|
|
f"No PDC branch found for {project['namespace']}/{project['name']}"
|
|
)
|
|
|
|
# Products
|
|
products = set()
|
|
for branch, active in project.get("branches"):
|
|
for regex, product in self.branch_regex_to_product.items():
|
|
if regex.match(branch):
|
|
products.add(product)
|
|
break
|
|
else:
|
|
products.add(self.namespace_to_product[project["namespace"]])
|
|
project["products"] = list(products)
|
|
|
|
products_poc = {}
|
|
products_retired = {}
|
|
for product in products:
|
|
owner = project["poc"]
|
|
# Check if the project is retired in PDC, and if so set assignee to orphan.
|
|
if self._is_retired(product, project):
|
|
owner = "orphan"
|
|
products_retired[product] = True
|
|
else:
|
|
products_retired[product] = False
|
|
|
|
# Check if the Bugzilla ticket assignee has been manually overridden
|
|
if product == "Fedora EPEL":
|
|
products_poc[product] = project["epelpoc"]
|
|
else:
|
|
products_poc[product] = owner
|
|
|
|
project["products_poc"] = products_poc
|
|
project["products_retired"] = products_retired
|
|
|
|
# Clean up the watchers we never want to sync to bugzilla
|
|
# If these users are POC for a project, things will not work, which
|
|
# is expected/desired.
|
|
for user in (self.config.get("ignorable_accounts") or []):
|
|
if user in (project.get("watchers") or []):
|
|
project["watchers"].remove(user)
|
|
|
|
self.pagure_projects[idx] = project
|
|
|
|
@classmethod
|
|
def main(cls):
|
|
"""The entrypoint for running the script."""
|
|
dbs = cls()
|
|
try:
|
|
dbs.run()
|
|
except ScriptExecError as e:
|
|
print(str(e), file=sys.stderr)
|
|
sys.exit(e.errorcode)
|
|
else:
|
|
sys.exit(0)
|
|
|
|
def run(self):
|
|
"""Run the script."""
|
|
global envname, env, projects_dict
|
|
times = {
|
|
"start": time.time(),
|
|
}
|
|
|
|
self.get_cli_arguments()
|
|
|
|
load_configuration(
|
|
addl_config_files=self.args.addl_config_files,
|
|
addl_email_overrides_files=self.args.addl_email_overrides_files,
|
|
)
|
|
self.config = config
|
|
|
|
envname = self.config["environment"]
|
|
if self.args.env:
|
|
if self.args.env in self.config["environments"]:
|
|
envname = self.args.env
|
|
else:
|
|
raise ScriptExecError(f"Invalid environment specified: {self.args.env}")
|
|
|
|
self.env = self.config["environments"][envname]
|
|
|
|
if self.args.debug:
|
|
self.env["verbose"] = True
|
|
self.env["dryrun"] = True
|
|
|
|
if self.args.verbose:
|
|
self.env["verbose"] = True
|
|
|
|
if self.args.dryrun:
|
|
self.env["dryrun"] = True
|
|
|
|
self.env["print-no-change"] = self.args.print_no_change
|
|
|
|
# Non-fatal errors to alert people about
|
|
self.errors = collections.defaultdict(list)
|
|
|
|
self.session = retry_session()
|
|
|
|
if self.env["verbose"]:
|
|
print("Building a cache of the rpm package summaries")
|
|
self.rpm_summaries = package_summaries.get_package_summaries()
|
|
|
|
self.get_pagure_projects(self.args.projects)
|
|
self.add_branches_products_and_summaries()
|
|
|
|
if self.env["verbose"]:
|
|
print(f"{len(self.pagure_projects)} projects to consider")
|
|
|
|
if not self.pagure_projects:
|
|
return
|
|
|
|
times["data structure end"] = time.time()
|
|
|
|
# Initialize the connection to bugzilla
|
|
bugzilla = BugzillaProxy(
|
|
bz_server=self.env["bugzilla"]["url"],
|
|
username=self.env["bugzilla"]["user"],
|
|
password=self.env["bugzilla"]["password"],
|
|
config=self.env,
|
|
pre_cache_users=not self.args.projects or self.args.print_fas_names,
|
|
)
|
|
|
|
times["FAS cache building end"] = time.time()
|
|
if self.env["verbose"]:
|
|
print("Building bugzilla's products in-memory cache")
|
|
|
|
bugzilla.build_product_cache(self.pagure_projects)
|
|
|
|
times["BZ cache building end"] = time.time()
|
|
if self.env["verbose"]:
|
|
if self.env["dryrun"]:
|
|
print("Querying bugzilla but not doing anything")
|
|
else:
|
|
print("Updating bugzilla")
|
|
|
|
for project in sorted(self.pagure_projects, key=itemgetter("name")):
|
|
for product in project["products"]:
|
|
if product not in self.env["products"]:
|
|
if self.env["verbose"]:
|
|
print(f"Ignoring: {product}/{project['name']}")
|
|
continue
|
|
|
|
try:
|
|
bugzilla.add_edit_component(
|
|
package=project["name"],
|
|
collection=product,
|
|
owner=project["products_poc"][product],
|
|
description=project["summary"],
|
|
qacontact=None,
|
|
cclist=project["watchers"],
|
|
print_fas_names=self.args.print_fas_names,
|
|
retired=project["products_retired"][product],
|
|
)
|
|
except ValueError as e:
|
|
# A username didn't have a bugzilla address
|
|
self.errors["bugzilla_raw"].append(str(e.args))
|
|
self.errors["bugzilla"].append(
|
|
f"Failed to update: `{product}/{project['name']}`:"
|
|
f"\n {e}"
|
|
f"\n {e.args}"
|
|
)
|
|
except DataChangedError as e:
|
|
# A Package or Collection was returned via xmlrpc but wasn't
|
|
# present when we tried to change it
|
|
self.errors["bugzilla_raw"].append(str(e.args))
|
|
self.errors["bugzilla"].append(
|
|
f"Failed to update: `{product}/{project['name']}`: "
|
|
f"\n {e}"
|
|
f"\n {e.args}"
|
|
)
|
|
except xmlrpc.client.ProtocolError as e:
|
|
# Unrecoverable and likely means that nothing is going to
|
|
# succeed.
|
|
self.errors["bugzilla_raw"].append(str(e.args))
|
|
self.errors["bugzilla"].append(
|
|
f"Failed to update: `{product}/{project['name']}`: "
|
|
f"\n {e}"
|
|
f"\n {e.args}"
|
|
)
|
|
break
|
|
except xmlrpc.client.Error as e:
|
|
# An error occurred in the xmlrpc call. Shouldn't happen but
|
|
# we better see what it is
|
|
self.errors["bugzilla_raw"].append(
|
|
"%s -- %s" % (project["name"], e.args[-1])
|
|
)
|
|
self.errors["bugzilla"].append(
|
|
f"Failed to update: `{product}/{project['name']}`: "
|
|
f"\n {e}"
|
|
f"\n {e.args}"
|
|
)
|
|
|
|
# Send notification of errors
|
|
if self.errors:
|
|
if not self.env["dryrun"] and self.args.user_notifications:
|
|
self.notify_users(self.errors["bugzilla"])
|
|
|
|
# Build the report for the admins
|
|
report = ["ERROR REPORT"]
|
|
for key in ["configuration", "PDC", "SCM overrides", "bugzilla"]:
|
|
if self.errors[key]:
|
|
report.append(key)
|
|
report.append(" - {}".format("\n - ".join(self.errors[key])))
|
|
report.append("")
|
|
|
|
if self.env["verbose"] or self.env["dryrun"]:
|
|
print("*" * 80)
|
|
print("\n".join(report))
|
|
|
|
# Do not send the email in dryrun or when the error only relates to
|
|
# configuration (which will always happen for flatpaks and tests)
|
|
if not self.env["dryrun"] and tuple(self.errors) != ("configuration",):
|
|
self.send_email(
|
|
self.env["email"]["from"],
|
|
self.env["email"]["notify_admins"],
|
|
"Errors while syncing bugzilla with the PackageDB",
|
|
self.env["email"]["templates"]["admin_notification"].format(
|
|
errors="\n".join(report)
|
|
),
|
|
)
|
|
else:
|
|
with open(self.env["data_cache"], "w") as stream:
|
|
json.dump({}, stream)
|
|
|
|
if self.env["verbose"]:
|
|
times["end"] = time.time()
|
|
|
|
print(" ----------")
|
|
print("Building the data structure")
|
|
delta = times["data structure end"] - times["start"]
|
|
print(f" Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")
|
|
|
|
print("Building the FAS cache")
|
|
delta = times["FAS cache building end"] - times["data structure end"]
|
|
print(f" Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")
|
|
|
|
print("Building the bugzilla cache")
|
|
delta = times["BZ cache building end"] - times["FAS cache building end"]
|
|
print(f" Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")
|
|
|
|
print("Interacting with bugzilla")
|
|
delta = times["end"] - times["BZ cache building end"]
|
|
print(f" Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")
|
|
|
|
print("Total")
|
|
delta = times["end"] - times["start"]
|
|
print(f" Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
DistgitBugzillaSync.main()
|