From 92b3805fe1ef687b946905dd49c04ad4f17b7088 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Wed, 8 Jul 2020 16:24:23 +0200 Subject: [PATCH] Introduce config defaults, valid for all toddlers This introduces a default section which can be overridden for any toddler individually. The dictionaries will be merged at depth, i.e. sub-dictionaries don't have to be copied wholesale just to override a single value. Additionally, it consolidates the configuration keys for Pagure-as-dist-git to `dist_git_*`, this was inconsistent between different toddlers. The old `pagure_*` keys will be used as a fallback for the time being, if `dist_git_*` keys are missing, to support deployed configuration. Signed-off-by: Nils Philippsen --- tests/plugins/test_flag_ci_pr.py | 12 ++-- tests/plugins/test_flag_commit_build.py | 16 ++--- toddlers.toml.example | 78 ++++++++++++++----------- toddlers/plugins/flag_ci_pr.py | 11 ++-- toddlers/plugins/flag_commit_build.py | 11 ++-- toddlers/runner.py | 26 +++++++-- 6 files changed, 91 insertions(+), 63 deletions(-) diff --git a/tests/plugins/test_flag_ci_pr.py b/tests/plugins/test_flag_ci_pr.py index c5c90ab..c2474c6 100644 --- a/tests/plugins/test_flag_ci_pr.py +++ b/tests/plugins/test_flag_ci_pr.py @@ -105,9 +105,9 @@ class TestFlagCIPRToddler: "run": {"url": "https://example.com/testing"}, } config = { - "pagure_token_seed": "example_seed", - "pagure_url": "https://pagure.io", - "pagure_token": "ahah", + "dist_git_token_seed": "example_seed", + "dist_git_url": "https://pagure.io", + "dist_git_token": "ahah", } assert toddler.process(config=config, message=msg) == 1 @@ -135,9 +135,9 @@ class TestFlagCIPRToddler: "run": {"url": "https://example.com/testing"}, } config = { - "pagure_token_seed": "example_seed", - "pagure_url": "https://pagure.io", - "pagure_token": "ahah", + "dist_git_token_seed": "example_seed", + "dist_git_url": "https://pagure.io", + "dist_git_token": "ahah", } assert toddler.process(config=config, message=msg) is None diff --git a/tests/plugins/test_flag_commit_build.py b/tests/plugins/test_flag_commit_build.py index cfd3fcf..5c7a779 100644 --- a/tests/plugins/test_flag_commit_build.py +++ b/tests/plugins/test_flag_commit_build.py @@ -149,8 +149,8 @@ class TestFlagCommitBuildToddler: } config = { "koji_url": "https://koji.fedoraproject.org", - "pagure_url": "https://src.fedoraproject.org", - "pagure_token": "ahah", + "dist_git_url": "https://src.fedoraproject.org", + "dist_git_token": "ahah", } assert toddler.process(config=config, message=msg) is None assert ( @@ -193,8 +193,8 @@ class TestFlagCommitBuildToddler: } config = { "koji_url": "https://koji.fedoraproject.org", - "pagure_url": "https://src.fedoraproject.org", - "pagure_token": "ahah", + "dist_git_url": "https://src.fedoraproject.org", + "dist_git_token": "ahah", } assert toddler.process(config=config, message=msg) is None assert ( @@ -239,8 +239,8 @@ class TestFlagCommitBuildToddler: } config = { "koji_url": "https://koji.fedoraproject.org", - "pagure_url": "https://src.fedoraproject.org", - "pagure_token": "ahah", + "dist_git_url": "https://src.fedoraproject.org", + "dist_git_token": "ahah", } assert toddler.process(config=config, message=msg) is None assert ( @@ -282,8 +282,8 @@ class TestFlagCommitBuildToddler: } config = { "koji_url": "https://koji.fedoraproject.org", - "pagure_url": "https://src.fedoraproject.org", - "pagure_token": "ahah", + "dist_git_url": "https://src.fedoraproject.org", + "dist_git_token": "ahah", } assert toddler.process(config=config, message=msg) is None assert ( diff --git a/toddlers.toml.example b/toddlers.toml.example index 7f6b178..35d78a6 100644 --- a/toddlers.toml.example +++ b/toddlers.toml.example @@ -45,32 +45,25 @@ routing_keys = ["#"] # This is dynamically generated in the code # more of them. blocked_toddlers = ["debug"] - -[consumer_config.flag_ci_pr] -# flag_ci_pr -pagure_token_seed = "private random string to change" -pagure_token = "private random string to change" -pagure_url = "https://src.fedoraproject.org" - - -[consumer_config.flag_commit_build] -# flag_commit_build -pagure_token = "private random string to change" -pagure_url = "https://src.fedoraproject.org" -koji_url = "https://koji.fedoraproject.org" - - -[consumer_config.packager_bugzilla_sync] -# Configuration file storing all the email overrides in the form of: -# "foo@bar.com" = "bar@foo.org" -# This is the same format as used by the distgit_bugzilla_sync cron/app -email_overrides_file = "/path/to/email_overrides.toml" - -# Base url of dist-git -dist_git_url = "https://src.fedoraproject.org" - -# List of accounts we do not want to report about -ignorable_accounts = ["packagerbot", "zuul"] +[consumer_config.default] +# Configuration common to all toddlers. +# +# You can override any of these in the section of a particular toddler, e.g.: +# +# [consumer_config.default] +# somekey = "somevalue" +# +# [consumer_config.default.subsection] +# key1 = "value1" +# key2 = "value2" +# +# ... +# +# [consumer_config.sometoddler] +# somekey = "someothervalue" +# +# [consumer_config.sometoddler.subsection] +# key2 = "othervalue2" # Configuration used when sending notifications: mail_server = "bastion.fedoraproject.org" @@ -87,10 +80,18 @@ bugzilla_username = "Bugzilla username (ie: email)" bugzilla_password = "Bugzilla password for that account" bugzilla_group = "fedora_contrib" +# Base URL for the Koji build system +koji_url = "https://koji.fedoraproject.org" -[consumer_config.pdc_retired_packages] -file_check_url = "https://src.fedoraproject.org/%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s" +# Account to use to connect to Pagure-as-dist-git +dist_git_url = "https://src.fedoraproject.org" +dist_git_token_seed = "private random string to change" +dist_git_token = "private random string to change" +[consumer_config.default.pdc_config] +# Configuration to talk to PDC, as understood by pdc-client. +server = "https://pdc.fedoraproject.org/rest_api/v1/" +ssl_verify = false # Enable if using a self-signed cert # XXX - getting the token is a bit of a pain, but here's a walk through # 1) go to https://pdc.fedoraproject.org/ in your browser and login. # 2) go to https://pdc.fedoraproject.org/rest_api/v1/auth/token/obtain/ @@ -100,13 +101,24 @@ file_check_url = "https://src.fedoraproject.org/%(namespace)s/%(repo)s/blob/%(br # 6) before hitting enter, edit the command to add the following option # -H 'Accept: application/json' # to tell the API you want data # 7) the command should print out your token. - -# Credentials to talk to PDC -[consumer_config.pdc_retired_packages.pdc_config] -server = "https://pdc.fedoraproject.org/rest_api/v1/" -ssl_verify = false # Enable if using a self-signed cert token = "PUT_HERE_THE_TOKEN_OBTAINED_MANUALLY" +[consumer_config.flag_ci_pr] + +[consumer_config.flag_commit_build] + +[consumer_config.packager_bugzilla_sync] +# Configuration file storing all the email overrides in the form of: +# "foo@bar.com" = "bar@foo.org" +# This is the same format as used by the distgit_bugzilla_sync cron/app +email_overrides_file = "/path/to/email_overrides.toml" + +# List of accounts we do not want to report about +ignorable_accounts = ["packagerbot", "zuul"] + +[consumer_config.pdc_retired_packages] +file_check_url = "https://src.fedoraproject.org/%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s" + [qos] prefetch_size = 0 prefetch_count = 25 diff --git a/toddlers/plugins/flag_ci_pr.py b/toddlers/plugins/flag_ci_pr.py index 3a0a91f..c4e9043 100644 --- a/toddlers/plugins/flag_ci_pr.py +++ b/toddlers/plugins/flag_ci_pr.py @@ -106,7 +106,7 @@ class FlagCIPR(ToddlerBase): pr_id = str(msg["artifact"]["id"]) commit_hash_text = " for %s" % msg["artifact"]["commit_hash"][:8] - seed = config.get("pagure_token_seed") or "" + seed = config.get("dist_git_token_seed", config.get("pagure_token_seed")) or "" data = { "username": "Fedora CI", @@ -116,7 +116,7 @@ class FlagCIPR(ToddlerBase): "uid": hashlib.md5((pr_id + seed).encode("utf-8")).hexdigest(), } - pagure_url = config["pagure_url"] + dist_git_url = config.get("dist_git_url") or config["pagure_url"] namespace = msg["artifact"]["repository"].split("/")[-2] repo = msg["artifact"]["repository"].split("/")[-1] @@ -124,11 +124,12 @@ class FlagCIPR(ToddlerBase): target_url = "/".join( ["api", "0", namespace, repo, "pull-request", pr_id, "flag"] ) - flag_url = pagure_url + "/" + target_url + flag_url = dist_git_url + "/" + target_url _log.info("Flagging commit at: %s" % flag_url) headers = { - "Authorization": "token " + config["pagure_token"], + "Authorization": "token " + config.get("dist_git_token") + or config["pagure_token"], "User-Agent": "loopabull@fedora-infra", } @@ -137,7 +138,7 @@ class FlagCIPR(ToddlerBase): req = requests_session.request( method="POST", url=flag_url, headers=headers, data=data, ) - _log.info("Request to %s returned: %s" % (pagure_url, req.status_code)) + _log.info("Request to %s returned: %s" % (dist_git_url, req.status_code)) if not req.ok: _log.debug(req.text) return 1 diff --git a/toddlers/plugins/flag_commit_build.py b/toddlers/plugins/flag_commit_build.py index 1f75a09..76239da 100644 --- a/toddlers/plugins/flag_commit_build.py +++ b/toddlers/plugins/flag_commit_build.py @@ -127,14 +127,15 @@ class FlagCommitBuild(ToddlerBase): "url": _koji_link(config, msg), } - pagure_url = config["pagure_url"] + dist_git_url = config.get("dist_git_url") or config["pagure_url"] target_url = "/".join(["api", "0", "rpms", msg["name"], "c", commit, "flag"]) - flag_url = pagure_url + "/" + target_url + flag_url = dist_git_url + "/" + target_url _log.info("Flagging commit at: %s" % flag_url) headers = { - "Authorization": "token " + config["pagure_token"], + "Authorization": "token " + config.get("dist_git_token") + or config["pagure_token"], "User-Agent": "toddlers@fedora-infra", } @@ -143,7 +144,7 @@ class FlagCommitBuild(ToddlerBase): req = requests_session.request( method="POST", url=flag_url, headers=headers, data=data, ) - _log.info("Request to %s returned: %s" % (pagure_url, req.status_code)) + _log.info("Request to %s returned: %s" % (dist_git_url, req.status_code)) if not req.ok: _log.debug(" -- FAILED -- ") _log.debug(req.text) @@ -151,7 +152,7 @@ class FlagCommitBuild(ToddlerBase): else: _log.info("All clear") _log.info( - "User-URL: %s" % pagure_url + "User-URL: %s" % dist_git_url + "/" + "/".join(["rpms", msg["name"], "c", commit]) ) diff --git a/toddlers/runner.py b/toddlers/runner.py index f5ab0c3..fb9558c 100644 --- a/toddlers/runner.py +++ b/toddlers/runner.py @@ -5,6 +5,7 @@ import fedora_messaging.exceptions from . import plugins # noqa: F401 from .base import ToddlerBase +from .utils.misc import merge_dicts _log = logging.getLogger(__name__) @@ -39,6 +40,15 @@ class RunningToddler(object): "No toddlers found to run :(" ) + self.toddler_config = {} + for toddler in self.toddlers: + name = toddler.name + # Bail out if two toddlers have the same name. + if name in self.toddler_config: + raise ValueError(f"Toddler named {name} already exists.") + + self.toddler_config[name] = self._get_toddler_config(name) + topics = set() for toddler in self.toddlers: _log.debug( @@ -57,6 +67,15 @@ class RunningToddler(object): ) binding["routing_keys"] = topics + def _get_toddler_config(self, toddler_name: str) -> dict: + """Merge default and private configuration for a toddler.""" + base_config = fedora_messaging.config.conf["consumer_config"] + + default_config = base_config.get("default", {}) + private_config = base_config.get(toddler_name, {}) + + return merge_dicts(default_config, private_config) + def __call__(self, message): """ Invoked when a message is received by the consumer. @@ -74,12 +93,7 @@ class RunningToddler(object): toddler.name, message.id, ) - toddler.process( - fedora_messaging.config.conf["consumer_config"].get( - toddler.name - ), - message, - ) + toddler.process(self.toddler_config[toddler.name], message) except Exception: _log.exception( "Toddler '%s' failed to process message id: %s -- putting it back in the queue",