# -*- coding: utf-8 -*-
#
# Copyright © 2013-2019  Red Hat, Inc.
#
# This copyrighted material is made available to anyone wishing to use, modify,
# copy, or redistribute it subject to the terms and conditions of the GNU
# General Public License v.2, or (at your option) any later version.  This
# program is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY expressed or implied, including the implied warranties of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.  You should have received a copy of the GNU
# General Public License along with this program; if not, write to the Free
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the source
# code or documentation are not subject to the GNU General Public License and
# may only be used or replicated with the express permission of Red Hat, Inc.
#
# Red Hat Author(s): Toshio Kuratomi <tkuratom@redhat.com>
#         Author(s): Mike Watters <valholla75@fedoraproject.org>
#         Author(s): Pierre-Yves Chibon <pingou@pingoured.fr>
#         Author(s): Matt Prahl <mprahl@redhat.com>
#         Author(s): Ralph Bean <rbean@redhat.com
#         Author(s): Patrick Uiterwijk <puiterwijk@redhat.com>
#
"""
sync information from the Pagure into bugzilla

This ... script takes information about package onwership and imports it
into bugzilla.
"""

import argparse
import collections
import datetime
from email.message import EmailMessage
import itertools
import json
import math
from operator import itemgetter
import os
import re
import smtplib
import sys
import time
import traceback
import xmlrpc.client

from bugzilla import Bugzilla
import dogpile.cache
import fedora.client
from fedora.client.fas2 import AccountSystem
import requests
from requests.adapters import HTTPAdapter
from urllib3.util import Retry

from . import package_summaries
from .config import config, email_overrides, load_configuration


cache = dogpile.cache.make_region().configure(
    "dogpile.cache.memory", expiration_time=3600,
)


def retry_session():
    session = requests.Session()
    retry = Retry(
        total=5,
        read=5,
        connect=5,
        backoff_factor=0.3,
        status_forcelist=(500, 502, 504),
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount("http://", adapter)
    session.mount("https://", adapter)
    return session


def resilient_partial(fn, *initial, **kwargs):
    """ A decorator that partially applies arguments.

    It additionally catches all raised exceptions, prints them, but then returns
    None instead of propagating the failures.

    This is used to protect functions used in a threadpool.  If one fails, we
    want to know about it, but we don't want it to kill the whole program.  So
    catch its error, log it, but proceed.
    """

    def wrapper(*additional):
        try:
            full = initial + additional
            return fn(*full, **kwargs)
        except Exception:
            traceback.print_exc()
            return None

    wrapper.__name__ = fn.__name__
    wrapper.__doc__ = fn.__doc__
    return wrapper


class DataChangedError(Exception):
    """Raised when data we are manipulating changes while we're modifying it."""

    pass


def segment(iterable, chunk, fill=None):
    """Collect data into `chunk` sized block"""
    args = [iter(iterable)] * chunk
    return itertools.zip_longest(*args, fillvalue=fill)


class BugzillaProxy:

    def __init__(self, bz_server, username, password, config, pre_cache_users=True):
        self.bz_xmlrpc_server = bz_server
        self.username = username
        self.password = password

        self.server = Bugzilla(
            url=self.bz_xmlrpc_server, user=self.username, password=self.password
        )

        self.product_cache = {}
        self.user_cache = {}
        self.inverted_user_cache = {}
        self.errors = []

        # Connect to the fedora account system
        self.fas = AccountSystem(
            base_url=config["fas"]["url"],
            username=config["fas"]["username"],
            password=config["fas"]["password"],
        )

        self.config = config

        if not pre_cache_users:
            return

        if config["verbose"]:
            print("Pre-caching FAS users and their Bugzilla email addresses.")

        try:
            self.user_cache = self.fas.people_by_key(
                key="username", fields=["email"]
            )
        except fedora.client.ServerError:
            # Sometimes, building the user cache up front fails with a timeout.
            # It's ok, we build the cache as-needed later in the script.
            pass
        else:
            self.invert_user_cache()

    def build_product_cache(self, pagure_projects):
        """ Cache the bugzilla info about each package in each product.
        """

        def _query_component(query, num_attempts=5):
            for i in range(num_attempts):
                try:
                    raw_data = self.server._proxy.Component.get({"names": query})
                    break
                except Exception as e:
                    if i >= num_attempts - 1:
                        raise
                    if self.config["verbose"]:
                        print(f"    ERROR {e}")
                        print("    - Query failed, going to try again in 10 seconds")
                    # Wait 10 seconds and try again
                    time.sleep(10)
            return raw_data

        if self.config["bugzilla"]["compat_api"] == "getcomponentsdetails":
            # Old API -- in python-bugzilla.  But with current server, this
            # gives ProxyError
            for collection, product in self.config["products"].items():
                self.product_cache[collection] = self.server.getcomponentsdetails(
                    collection
                )
        elif self.config["bugzilla"]["compat_api"] == "component.get":
            # Way that's undocumented in the partner-bugzilla api but works
            # currently
            chunk = self.config["bugzilla"]["req_segment"]
            for collection, product in self.config["products"].items():

                # restrict the list of info returned to only the packages of
                # interest
                pkglist = [
                    project["name"]
                    for project in pagure_projects
                    if collection in project["products"]
                ]
                product_info_by_pkg = {}
                estimated_size = math.ceil(len(pkglist) / chunk)
                for cnt, pkg_segment in enumerate(segment(pkglist, chunk), start=1):
                    # Format that bugzilla will understand.  Strip None's that
                    # segment() pads out the final data segment() with
                    query = [
                        {"product": collection, "component": p}
                        for p in pkg_segment
                        if p is not None
                    ]
                    if self.config["verbose"]:
                        print(
                            f"  - Querying product `{collection}`, "
                            f"query {cnt} of {estimated_size}"
                        )
                    raw_data = _query_component(query)
                    for package in raw_data["components"]:
                        # Reformat data to be the same as what's returned from
                        # getcomponentsdetails
                        product_info = {
                            "initialowner": package["default_assignee"],
                            "description": package["description"],
                            "initialqacontact": package["default_qa_contact"],
                            "initialcclist": package["default_cc"],
                            "is_active": package["is_active"],
                        }
                        product_info_by_pkg[package["name"].lower()] = product_info
                self.product_cache[collection] = product_info_by_pkg

    def invert_user_cache(self):
        """ Takes the user_cache built when querying FAS and invert it so
        that the bugzilla_email is the key and the username the value.
        """
        for username in self.user_cache:
            bz_email = self.user_cache[username]["email"].lower()
            self.inverted_user_cache[bz_email] = username

    def _get_bugzilla_email(self, username):
        """Return the bugzilla email address for a user.

        First looks in a cache for a username => bugzilla email.  If not found,
        reloads the cache from fas and tries again.
        """
        try:
            bz_email = self.user_cache[username]["email"].lower()
        except KeyError:
            if username.startswith("@"):
                group = self.fas.group_by_name(username[1:])
                bz_email = group.mailing_list
                if bz_email is None:
                    return
                self.user_cache[username] = {"email": bz_email}
            else:
                person = self.fas.person_by_username(username)
                bz_email = person.get("email")
                if bz_email is None:
                    return
                self.user_cache[username] = {"email": bz_email}
            bz_email = bz_email.lower()
            self.inverted_user_cache[bz_email] = username
        bz_email = email_overrides.get(bz_email, bz_email)
        return bz_email.strip()

    def update_open_bugs(self, new_poc, prev_poc, product, name, print_fas_names=False):
        """Change the package owner
        :arg new_poc: email of the new point of contact.
        :arg prev_poc: Username of the previous point of contact
        :arg product: The product of the package to change in bugzilla
        :arg name: Name of the package to change the owner.
        :kwarg print_fas_names: Boolean specifying wether to print email or FAS names
            (if these could be found).
        """
        bz_query = {}
        bz_query["product"] = product
        bz_query["component"] = name
        bz_query["bug_status"] = [
            "NEW",
            "ASSIGNED",
            "ON_DEV",
            "ON_QA",
            "MODIFIED",
            "POST",
            "FAILS_QA",
            "PASSES_QA",
            "RELEASE_PENDING",
        ]
        # Update only maintained releases
        bz_query["version"] = self.config["products"][product]["versions"]

        def _query_bz(query, num_attempts=5):
            for i in range(num_attempts):
                try:
                    raw_data = self.server.query(bz_query)
                    break
                except Exception as e:
                    if i >= num_attempts - 1:
                        raise
                    if self.config["verbose"]:
                        print(f"    ERROR {e}")
                        print("    - Query failed, going to try again in 20 seconds")
                    # Wait 20 seconds and try again
                    time.sleep(20)
            return raw_data

        query_results = _query_bz(bz_query)

        for bug in query_results:
            if bug.assigned_to == prev_poc and bug.assigned_to != new_poc:
                if self.config["verbose"]:
                    old_poc = bug.assigned_to
                    if print_fas_names:
                        if old_poc in self.inverted_user_cache:
                            old_poc = self.inverted_user_cache[old_poc]
                        else:
                            old_poc = old_poc.split("@", 1)[0] + "@..."
                        if new_poc in self.inverted_user_cache:
                            new_poc = self.inverted_user_cache[new_poc]
                        else:
                            new_poc = new_poc.split("@", 1)[0] + "@..."
                    print(
                        f"[UPDATEBUG] {product}/{name}  reassigning bug #{bug.bug_id} "
                        f"from {old_poc} to {new_poc}"
                    )
                if not self.config["dryrun"]:
                    try:
                        bug.setassignee(
                            assigned_to=new_poc,
                            comment=self.config["bz_maintainer_change_comment"],
                        )
                    except xmlrpc.client.Fault as e:
                        # Output something useful in args
                        e.args = (new_poc, e.faultCode, e.faultString)
                        raise
                    except xmlrpc.client.ProtocolError as e:
                        e.args = ("ProtocolError", e.errcode, e.errmsg)
                        raise

    def add_edit_component(
        self,
        package,
        collection,
        owner,
        description=None,
        qacontact=None,
        cclist=None,
        print_fas_names=False,
        retired=False,
    ):
        """Add or update a component to have the values specified.
        """
        # Turn the cclist into something usable by bugzilla
        initial_cc_emails = []
        initial_cc_emails_lower = []
        initial_cc_fasnames = []
        for watcher in cclist:
            bz_email = self._get_bugzilla_email(watcher)
            if bz_email:
                initial_cc_emails.append(bz_email)
                initial_cc_emails_lower.append(bz_email.lower())
                initial_cc_fasnames.append(watcher)
            else:
                self.errors.append(
                    f"{watcher} has no bugzilla_email or mailing_list set on "
                    f"({collection}/{package})"
                )
                if self.config["verbose"]:
                    print(
                        f"** {watcher} has no bugzilla_email or mailing_list set "
                        f"({collection}/{package}) **"
                    )

        # Add owner to the cclist so comaintainers taking over a bug don't
        # have to do this manually
        owner_email = self._get_bugzilla_email(owner)
        if owner_email.lower() not in initial_cc_emails_lower:
            initial_cc_emails.append(owner_email)
            initial_cc_emails_lower.append(owner_email.lower())
            initial_cc_fasnames.append(owner)

        # Lookup product
        try:
            product = self.product_cache[collection]
        except xmlrpc.client.Fault as e:
            # Output something useful in args
            e.args = (e.faultCode, e.faultString)
            raise
        except xmlrpc.client.ProtocolError as e:
            e.args = ("ProtocolError", e.errcode, e.errmsg)
            raise


        # Set the qacontact_email and name
        default_qa_contact_email = self.config["default_qa_contact_email"]
        default_qa_contact = (
            f"<default: {default_qa_contact_email.split('@', 1)[0]}@...>"
        )
        if qacontact:
            qacontact_email = self._get_bugzilla_email(qacontact).strip()
        else:
            qacontact = default_qa_contact
            qacontact_email = default_qa_contact_email.strip()

        pkg_key = package.lower()
        if pkg_key in product:
            # edit the package information
            data = {}

            # Check for changes to the owner, qacontact, or description
            if product[pkg_key]["initialowner"].lower() != owner_email.lower():
                data["initialowner"] = owner_email

            if description and product[pkg_key]["description"] != description:
                data["description"] = description

            if (
                qacontact
                and product[pkg_key]["initialqacontact"].lower()
                != qacontact_email.lower()
            ):
                data["initialqacontact"] = qacontact_email

            if len(product[pkg_key]["initialcclist"]) != len(initial_cc_emails):
                data["initialcclist"] = initial_cc_emails
            else:
                for cc_member in product[pkg_key]["initialcclist"]:
                    if cc_member.lower() not in initial_cc_emails_lower:
                        data["initialcclist"] = initial_cc_emails
                        break

            if product[pkg_key]["is_active"] != (not retired):
                data["is_active"] = not retired

            if data:
                # Changes occurred.  Submit a request to change via xmlrpc
                data["product"] = collection
                data["component"] = package

                if self.config["verbose"]:
                    for key in [
                        "initialowner",
                        "description",
                        "initialqacontact",
                        "initialcclist",
                        "is_active",
                    ]:
                        if data.get(key) is not None:

                            old_value = product[pkg_key][key]
                            if isinstance(old_value, list):
                                old_value = sorted(old_value)
                            new_value = data.get(key)
                            if isinstance(new_value, list):
                                new_value = sorted(new_value)

                            if print_fas_names and key in (
                                "initialowner",
                                "initialqacontact",
                                "initialcclist",
                            ):
                                if key == "initialowner":
                                    new_value = owner
                                elif key == "initialqacontact":
                                    new_value = qacontact
                                else:
                                    new_value = sorted(initial_cc_fasnames)

                                from_fas_names = []
                                for email in product[pkg_key][key]:
                                    if email in self.inverted_user_cache:
                                        from_fas_names.append(
                                            self.inverted_user_cache[email]
                                        )
                                    elif email == default_qa_contact_email:
                                        from_fas_names.append(default_qa_contact)
                                if from_fas_names:
                                    if len(from_fas_names) < len(product[pkg_key][key]):
                                        x = len(product[pkg_key][key]) - len(
                                            from_fas_names
                                        )
                                        from_fas_names.append(f"And {x} more")
                                    old_value = f"from `{from_fas_names}`"
                                else:
                                    old_value = ""
                                print(
                                    f"[EDITCOMP] {data['product']}/{data['component']}"
                                    f"  {key} changed {old_value} to FAS name(s) `{new_value}`"
                                )
                            else:
                                print(
                                    f"[EDITCOMP] {data['product']}/{data['component']}"
                                    f"  {key} changed from `{old_value}` to `{new_value}`"
                                )

                owner_changed = "initialowner" in data

                # FIXME: initialowner has been made mandatory for some
                # reason.  Asking dkl why.
                data["initialowner"] = owner_email

                def edit_component(data, num_attempts=5):
                    for i in range(num_attempts):
                        try:
                            self.server.editcomponent(data)
                            break
                        except Exception as e:
                            if (
                                isinstance(e, xmlrpc.client.Fault)
                                and e.faultCode == 504
                            ):
                                if self.config["verbose"]:
                                    print(f"    ERROR {e}")
                                raise
                            if i >= num_attempts - 1:
                                raise
                            if self.config["verbose"]:
                                print(f"    ERROR {e}")
                                print(
                                    "    - Query failed, going to try again in 20 seconds"
                                )
                            # Wait 20 seconds and try again
                            time.sleep(20)

                if not self.config["dryrun"]:
                    try:
                        edit_component(data)
                    except xmlrpc.client.Fault as e:
                        # Output something useful in args
                        e.args = (data, e.faultCode, e.faultString)
                        raise
                    except xmlrpc.client.ProtocolError as e:
                        e.args = ("ProtocolError", e.errcode, e.errmsg)
                        raise

                if owner_changed:
                    self.update_open_bugs(
                        new_poc=owner_email,
                        prev_poc=product[pkg_key]["initialowner"],
                        name=package,
                        product=collection,
                        print_fas_names=print_fas_names,
                    )
            else:
                if self.config.get("print-no-change"):
                    print(f"[NOCHANGE] {package}/{collection}")
        else:
            if retired:
                if self.config["verbose"]:
                    print(f"[NOADD] {collection}/{package} (is retired)")
                return

            # Add component
            data = {
                "product": collection,
                "component": package,
                "description": description or "NA",
                "initialowner": owner_email,
                "initialqacontact": qacontact_email,
                "is_active": not retired,
            }
            if initial_cc_emails:
                data["initialcclist"] = initial_cc_emails

            if self.config["verbose"]:
                for key in [
                    "initialowner",
                    "description",
                    "initialqacontact",
                    "initialcclist",
                    "is_active",
                ]:
                    if print_fas_names and key in (
                        "initialowner",
                        "initialqacontact",
                        "initialcclist",
                    ):
                        if key == "initialowner":
                            value = owner
                        elif key == "initialqacontact":
                            value = qacontact
                        else:
                            value = initial_cc_fasnames
                        print(
                            f"[ADDCOMP] {collection}/{package}"
                            f"  {key} set to FAS name(s) `{value}`"
                        )
                    else:
                        print(
                            f"[ADDCOMP] {collection}/{package}"
                            f"  {key} set to {data.get(key)}"
                        )

            def add_component(data, num_attempts=5):
                for i in range(num_attempts):
                    try:
                        self.server.addcomponent(data)
                        break
                    except Exception as e:
                        if isinstance(e, xmlrpc.client.Fault) and e.faultCode in [
                            504,
                            1200,
                        ]:
                            # error  504: user <email> is not a valid username
                            # error 1200: Product <product> has already a component named <name>
                            if self.config["verbose"]:
                                print(f"    ERROR {e}")
                            raise
                        if i >= num_attempts - 1:
                            raise
                        if self.config["verbose"]:
                            print(f"    ERROR {e}")
                            print(
                                "    - Query failed, going to try again in 20 seconds"
                            )
                        # Wait 20 seconds and try again
                        time.sleep(20)

            if not self.config["dryrun"]:
                try:
                    add_component(data)
                except xmlrpc.client.Fault as e:
                    # Output something useful in args
                    e.args = (data, e.faultCode, e.faultString)
                    raise


def _get_pdc_branches(session, repo):
    """
    Gets the branches on a project. This function is used for mapping.
    :param repo: the project dict
    :return: a list of the repo's branches
    """
    branches_url = "{0}component-branches/".format(env["pdc_url"])
    params = {
        "global_component": repo["name"],
        "type": env["pdc_types"][repo["namespace"]],
    }
    if config["verbose"]:
        print("Querying {0} {1}".format(branches_url, params))
    rv = session.get(branches_url, params=params, timeout=60)

    # If the project's branches can't be reported, just return no branches and
    # it will be skipped later on
    if not rv.ok:
        print(
            (
                'The connection to "{0}" failed with the status code {1} and '
                'output "{2}"'.format(branches_url, rv.status_code, rv.text)
            ),
            file=sys.stderr,
        )
        return []

    data = rv.json()
    return [branch["name"] for branch in data["results"]]


class ScriptExecError(RuntimeError):
    def __init__(self, *args, **kwargs):
        self.errorcode = kwargs.pop("errorcode", 1)
        super().__init__(*args, **kwargs)


class DistgitBugzillaSync:

    # cache placeholders for properties which are computed once
    _namespace_to_product = None
    _product_to_branch_regex = None
    _branch_regex_to_product = None

    def send_email(self, from_address, to_address, subject, message, cc_address=None):
        """Send an email if there's an error.

        This will be replaced by sending messages to a log later.
        """
        if not self.env["email"]["send_mails"]:
            return

        msg = EmailMessage()
        msg.add_header("To", ",".join(to_address))
        msg.add_header("From", from_address)
        msg.add_header("Subject", subject)
        if cc_address is not None:
            msg.add_header("Cc", ",".join(cc_address))
            to_address += cc_address
        msg.set_payload(message)
        smtp = smtplib.SMTP(self.env["email"]["smtp_host"])
        smtp.sendmail(from_address, to_address, msg.as_string())
        smtp.quit()

    def notify_users(self, errors):
        """ Browse the list of errors and when we can retrieve the email
        address, use it to notify the user about the issue.
        """
        data = {}
        if os.path.exists(self.env["data_cache"]):
            try:
                with open(self.env["data_cache"]) as stream:
                    data = json.load(stream)
            except Exception as err:
                print(
                    "Could not read the json file at %s: \nError:  %s"
                    % (env["data_cache"], err)
                )

        new_data = {}
        seen = []
        for error in errors:
            notify_user = False
            if "The name " in error and " is not a valid username" in error:
                user_email = (
                    error.split(" is not a valid username")[0]
                    .split("The name ")[1]
                    .strip()
                )
                now = datetime.datetime.utcnow()

                # See if we already know about this user
                if user_email in data and data[user_email]["last_update"]:
                    last_update = datetime.datetime.fromtimestamp(
                        int(data[user_email]["last_update"])
                    )
                    # Only notify users once per hour
                    if (now - last_update).seconds >= 3600:
                        notify_user = True
                    else:
                        new_data[user_email] = data[user_email]
                elif not data or user_email not in data:
                    notify_user = True

                # Ensure we notify the user only once, no matter how many errors we
                # got concerning them.
                if user_email not in seen:
                    seen.append(user_email)
                else:
                    notify_user = False

                if notify_user:
                    self.send_email(
                        self.env["email"]["from"],
                        [user_email],
                        subject="Please fix your bugzilla.redhat.com account",
                        message=self.env["email"]["templates"]["user_notification"],
                        cc_address=self.env["email"]["notify_admins"],
                    )

                    new_data[user_email] = {"last_update": time.mktime(now.timetuple())}

        with open(env["data_cache"], "w") as stream:
            json.dump(new_data, stream)

    def get_cli_arguments(self):
        """ Set the CLI argument parser and return the argument parsed.
        """
        parser = argparse.ArgumentParser(
            description="Script syncing information between Pagure and bugzilla"
        )
        parser.add_argument(
            "--dry-run",
            dest="dryrun",
            action="store_true",
            default=False,
            help="Do not actually make any changes - Overrides the configuration",
        )
        parser.add_argument(
            "--verbose",
            dest="verbose",
            action="store_true",
            default=False,
            help="Print actions verbosely - Overrides the configuration",
        )
        parser.add_argument(
            "--debug",
            dest="debug",
            action="store_true",
            default=False,
            help="Combination of --verbose and --dry-run",
        )
        parser.add_argument(
            "--env",
            dest="env",
            help="Run the script for a specific environment, overrides the one "
            "set in the configuration file",
        )
        parser.add_argument(
            "--add-config-file",
            metavar="CONFIG_FILE",
            dest="addl_config_files",
            action="append",
            help="File(s) from which to read overriding configuration",
        )
        parser.add_argument(
            "--add-email-overrides-file",
            metavar="EMAIL_OVERRIDES_FILE",
            dest="addl_email_overrides_files",
            action="append",
            help="File(s) from which to read additional email overrides",
        )
        parser.add_argument(
            "-p",
            "--project",
            dest="projects",
            nargs="+",
            help="Update one or more projects (provided as namespace/name), "
            "in all of its products",
        )
        parser.add_argument(
            "--print-fas-names",
            action="store_true",
            default=False,
            help="Print FAS names rather than email addresses in output, useful when pasting into "
            "public fora",
        )
        parser.add_argument(
            "--print-no-change",
            action="store_true",
            default=False,
            help="Print elements that are not being changed as they are checked",
        )
        parser.add_argument(
            "--no-user-notifications",
            dest="user_notifications",
            action="store_false",
            default=True,
            help="Do not notify every packager whose account is wrongly set-up, but do send the "
            "overall report to the admins",
        )

        self.args = parser.parse_args()

    def get_pagure_projects(self, project_list=None):
        """ Builds a large list of all the projects on pagure.
        Each item in that list is a dict containing:
        - the namespace of the project
        - the name of the project
        - the point of contact of this project (ie: the default assignee
          in bugzilla)
        - the watchers of this project (ie: the initial CC list in bugzilla)
        """

        # Get the initial ownership and CC data from pagure
        # This part is easy.
        poc_url = self.env["distgit_url"] + "/extras/pagure_poc.json"
        if self.env["verbose"]:
            print("Querying %r for points of contact." % poc_url)
        pagure_namespace_to_poc = self.session.get(poc_url, timeout=120).json()

        cc_url = self.env["distgit_url"] + "/extras/pagure_bz.json"
        if self.env["verbose"]:
            print("Querying %r for initial cc list." % cc_url)
        pagure_namespace_to_cc = self.session.get(cc_url, timeout=120).json()

        # Combine and collapse those two into a single list:
        self.pagure_projects = []
        if project_list:
            project_list = {tuple(p.split("/", 1)) for p in project_list}
        for namespace, entries in pagure_namespace_to_poc.items():
            for name, poc in entries.items():
                if not project_list or (namespace, name) in project_list:
                    self.pagure_projects.append(
                        {
                            "namespace": namespace,
                            "name": name,
                            "poc": poc["fedora"],
                            "epelpoc": poc["epel"],
                            "watchers": pagure_namespace_to_cc[namespace][name],
                        }
                    )

    @property
    def namespace_to_product(self):
        if self._namespace_to_product is None:
            self._namespace_to_product = {
                p["namespace"]: n
                for n, p in self.env["products"].items()
                if "namespace" in p
            }
        return self._namespace_to_product

    @property
    def product_to_branch_regex(self):
        if self._product_to_branch_regex is None:
            self._product_to_branch_regex = {
                n: re.compile(p["branch_regex"])
                for n, p in self.env["products"].items()
                if "branch_regex" in p
            }
        return self._product_to_branch_regex

    @property
    def branch_regex_to_product(self):
        if self._branch_regex_to_product is None:
            self._branch_regex_to_product = {
                n: r for r, n in self.product_to_branch_regex.items()
            }
        return self._branch_regex_to_product

    def _is_retired(self, product, project):
        branches = project["branches"]
        branch_regex = self.product_to_branch_regex.get(product)
        if branch_regex:
            for branch, active in branches:
                if branch_regex.match(branch) and active:
                    return False
            # No active branches means it is retired.
            return True
        else:
            for branch, active in branches:
                if active:
                    return False
            return True

    def add_branches_products_and_summaries(self):
        """ For each project retrieved, this method adds branches, products
        and summary information.

        The branches are retrieved from PDC.

        The products are determined based on the branches.

        The summaries are coming from the primary.xml file of the Rawhide repodata
        in Koji.
        """
        branches_url = "/".join(
            [
                self.env["pdc_url"].split("rest_api")[0].rstrip("/"),
                "extras/active_branches.json",
            ]
        )
        if self.env["verbose"]:
            print("Querying %r for EOL information." % branches_url)

        pdc_branches = self.session.get(branches_url, timeout=120).json()
        for idx, project in enumerate(self.pagure_projects):
            # Summary
            summary = None
            if project["namespace"] == "rpms":
                summary = self.rpm_summaries.get(project["name"])
            project["summary"] = summary

            # Branches
            if project["namespace"] not in self.env["pdc_types"]:
                project["branches"] = []
                project["products"] = []
                error = f'Namespace `{project["namespace"]}` not found in the pdc_type ' \
                    f'configuration key -- ignoring it'
                if not error in self.errors["configuration"]:
                    self.errors["configuration"].append(error)
                if self.env["verbose"]:
                    print(
                        f'Namespace `{project["namespace"]}` not found in the pdc_type '
                        f'configuration key, project {project["namespace"]}/{project["name"]} '
                        "ignored"
                    )
                continue

            pdc_type = self.env["pdc_types"][project["namespace"]]
            project["branches"] = pdc_branches.get(pdc_type, {}).get(
                project["name"], []
            )
            if not project["branches"]:
                self.errors["PDC"].append(
                    f"No PDC branch found for {project['namespace']}/{project['name']}"
                )

            # Products
            products = set()
            for branch, active in project.get("branches"):
                for regex, product in self.branch_regex_to_product.items():
                    if regex.match(branch):
                        products.add(product)
                        break
                else:
                    products.add(self.namespace_to_product[project["namespace"]])
            project["products"] = list(products)

            products_poc = {}
            products_retired = {}
            for product in products:
                owner = project["poc"]
                # Check if the project is retired in PDC, and if so set assignee to orphan.
                if self._is_retired(product, project):
                    owner = "orphan"
                    products_retired[product] = True
                else:
                    products_retired[product] = False

                # Check if the Bugzilla ticket assignee has been manually overridden
                if product == "Fedora EPEL":
                    products_poc[product] = project["epelpoc"]
                else:
                    products_poc[product] = owner

            project["products_poc"] = products_poc
            project["products_retired"] = products_retired

            # Clean up the watchers we never want to sync to bugzilla
            # If these users are POC for a project, things will not work, which
            # is expected/desired.
            for user in (self.config.get("ignorable_accounts") or []):
                if user in (project.get("watchers") or []):
                    project["watchers"].remove(user)

            self.pagure_projects[idx] = project

    @classmethod
    def main(cls):
        """The entrypoint for running the script."""
        dbs = cls()
        try:
            dbs.run()
        except ScriptExecError as e:
            print(str(e), file=sys.stderr)
            sys.exit(e.errorcode)
        else:
            sys.exit(0)

    def run(self):
        """Run the script."""
        global envname, env, projects_dict
        times = {
            "start": time.time(),
        }

        self.get_cli_arguments()

        load_configuration(
            addl_config_files=self.args.addl_config_files,
            addl_email_overrides_files=self.args.addl_email_overrides_files,
        )
        self.config = config

        envname = self.config["environment"]
        if self.args.env:
            if self.args.env in self.config["environments"]:
                envname = self.args.env
            else:
                raise ScriptExecError(f"Invalid environment specified: {self.args.env}")

        self.env = self.config["environments"][envname]

        if self.args.debug:
            self.env["verbose"] = True
            self.env["dryrun"] = True

        if self.args.verbose:
            self.env["verbose"] = True

        if self.args.dryrun:
            self.env["dryrun"] = True

        self.env["print-no-change"] = self.args.print_no_change

        # Non-fatal errors to alert people about
        self.errors = collections.defaultdict(list)

        self.session = retry_session()

        if self.env["verbose"]:
            print("Building a cache of the rpm package summaries")
        self.rpm_summaries = package_summaries.get_package_summaries()

        self.get_pagure_projects(self.args.projects)
        self.add_branches_products_and_summaries()

        if self.env["verbose"]:
            print(f"{len(self.pagure_projects)} projects to consider")

        if not self.pagure_projects:
            return

        times["data structure end"] = time.time()

        # Initialize the connection to bugzilla
        bugzilla = BugzillaProxy(
            bz_server=self.env["bugzilla"]["url"],
            username=self.env["bugzilla"]["user"],
            password=self.env["bugzilla"]["password"],
            config=self.env,
            pre_cache_users=not self.args.projects or self.args.print_fas_names,
        )

        times["FAS cache building end"] = time.time()
        if self.env["verbose"]:
            print("Building bugzilla's products in-memory cache")

        bugzilla.build_product_cache(self.pagure_projects)

        times["BZ cache building end"] = time.time()
        if self.env["verbose"]:
            if self.env["dryrun"]:
                print("Querying bugzilla but not doing anything")
            else:
                print("Updating bugzilla")

        for project in sorted(self.pagure_projects, key=itemgetter("name")):
            for product in project["products"]:
                if product not in self.env["products"]:
                    if self.env["verbose"]:
                        print(f"Ignoring: {product}/{project['name']}")
                    continue

                try:
                    bugzilla.add_edit_component(
                        package=project["name"],
                        collection=product,
                        owner=project["products_poc"][product],
                        description=project["summary"],
                        qacontact=None,
                        cclist=project["watchers"],
                        print_fas_names=self.args.print_fas_names,
                        retired=project["products_retired"][product],
                    )
                except ValueError as e:
                    # A username didn't have a bugzilla address
                    self.errors["bugzilla_raw"].append(str(e.args))
                    self.errors["bugzilla"].append(
                        f"Failed to update: `{product}/{project['name']}`:"
                        f"\n   {e}"
                        f"\n   {e.args}"
                    )
                except DataChangedError as e:
                    # A Package or Collection was returned via xmlrpc but wasn't
                    # present when we tried to change it
                    self.errors["bugzilla_raw"].append(str(e.args))
                    self.errors["bugzilla"].append(
                        f"Failed to update: `{product}/{project['name']}`: "
                        f"\n   {e}"
                        f"\n   {e.args}"
                    )
                except xmlrpc.client.ProtocolError as e:
                    # Unrecoverable and likely means that nothing is going to
                    # succeed.
                    self.errors["bugzilla_raw"].append(str(e.args))
                    self.errors["bugzilla"].append(
                        f"Failed to update: `{product}/{project['name']}`: "
                        f"\n   {e}"
                        f"\n   {e.args}"
                    )
                    break
                except xmlrpc.client.Error as e:
                    # An error occurred in the xmlrpc call.  Shouldn't happen but
                    # we better see what it is
                    self.errors["bugzilla_raw"].append(
                        "%s -- %s" % (project["name"], e.args[-1])
                    )
                    self.errors["bugzilla"].append(
                        f"Failed to update: `{product}/{project['name']}`: "
                        f"\n   {e}"
                        f"\n   {e.args}"
                    )

        # Send notification of errors
        if self.errors:
            if not self.env["dryrun"] and self.args.user_notifications:
                self.notify_users(self.errors["bugzilla"])

            # Build the report for the admins
            report = ["ERROR REPORT"]
            for key in ["configuration", "PDC", "SCM overrides", "bugzilla"]:
                if self.errors[key]:
                    report.append(key)
                    report.append("  - {}".format("\n  - ".join(self.errors[key])))
                    report.append("")

            if self.env["verbose"] or self.env["dryrun"]:
                print("*" * 80)
                print("\n".join(report))

            # Do not send the email in dryrun or when the error only relates to
            # configuration (which will always happen for flatpaks and tests)
            if not self.env["dryrun"] and tuple(self.errors) != ("configuration",):
                self.send_email(
                    self.env["email"]["from"],
                    self.env["email"]["notify_admins"],
                    "Errors while syncing bugzilla with the PackageDB",
                    self.env["email"]["templates"]["admin_notification"].format(
                        errors="\n".join(report)
                    ),
                )
        else:
            with open(self.env["data_cache"], "w") as stream:
                json.dump({}, stream)

        if self.env["verbose"]:
            times["end"] = time.time()

            print("   ----------")
            print("Building the data structure")
            delta = times["data structure end"] - times["start"]
            print(f"  Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")

            print("Building the FAS cache")
            delta = times["FAS cache building end"] - times["data structure end"]
            print(f"  Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")

            print("Building the bugzilla cache")
            delta = times["BZ cache building end"] - times["FAS cache building end"]
            print(f"  Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")

            print("Interacting with bugzilla")
            delta = times["end"] - times["BZ cache building end"]
            print(f"  Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")

            print("Total")
            delta = times["end"] - times["start"]
            print(f"  Ran on {delta:.2f} seconds -- ie: {delta/60:.2f} minutes")


if __name__ == "__main__":
    DistgitBugzillaSync.main()