Introduce config defaults, valid for all toddlers
This introduces a default section which can be overridden for any toddler individually. The dictionaries will be merged at depth, i.e. sub-dictionaries don't have to be copied wholesale just to override a single value. Additionally, it consolidates the configuration keys for Pagure-as-dist-git to `dist_git_*`, this was inconsistent between different toddlers. The old `pagure_*` keys will be used as a fallback for the time being, if `dist_git_*` keys are missing, to support deployed configuration. Signed-off-by: Nils Philippsen <nils@redhat.com>
This commit is contained in:
Nils Philippsen
parent
c753e54947
commit
92b3805fe1
6 changed files with 91 additions and 63 deletions
|
|
@ -105,9 +105,9 @@ class TestFlagCIPRToddler:
|
|||
"run": {"url": "https://example.com/testing"},
|
||||
}
|
||||
config = {
|
||||
"pagure_token_seed": "example_seed",
|
||||
"pagure_url": "https://pagure.io",
|
||||
"pagure_token": "ahah",
|
||||
"dist_git_token_seed": "example_seed",
|
||||
"dist_git_url": "https://pagure.io",
|
||||
"dist_git_token": "ahah",
|
||||
}
|
||||
|
||||
assert toddler.process(config=config, message=msg) == 1
|
||||
|
|
@ -135,9 +135,9 @@ class TestFlagCIPRToddler:
|
|||
"run": {"url": "https://example.com/testing"},
|
||||
}
|
||||
config = {
|
||||
"pagure_token_seed": "example_seed",
|
||||
"pagure_url": "https://pagure.io",
|
||||
"pagure_token": "ahah",
|
||||
"dist_git_token_seed": "example_seed",
|
||||
"dist_git_url": "https://pagure.io",
|
||||
"dist_git_token": "ahah",
|
||||
}
|
||||
|
||||
assert toddler.process(config=config, message=msg) is None
|
||||
|
|
|
|||
|
|
@ -149,8 +149,8 @@ class TestFlagCommitBuildToddler:
|
|||
}
|
||||
config = {
|
||||
"koji_url": "https://koji.fedoraproject.org",
|
||||
"pagure_url": "https://src.fedoraproject.org",
|
||||
"pagure_token": "ahah",
|
||||
"dist_git_url": "https://src.fedoraproject.org",
|
||||
"dist_git_token": "ahah",
|
||||
}
|
||||
assert toddler.process(config=config, message=msg) is None
|
||||
assert (
|
||||
|
|
@ -193,8 +193,8 @@ class TestFlagCommitBuildToddler:
|
|||
}
|
||||
config = {
|
||||
"koji_url": "https://koji.fedoraproject.org",
|
||||
"pagure_url": "https://src.fedoraproject.org",
|
||||
"pagure_token": "ahah",
|
||||
"dist_git_url": "https://src.fedoraproject.org",
|
||||
"dist_git_token": "ahah",
|
||||
}
|
||||
assert toddler.process(config=config, message=msg) is None
|
||||
assert (
|
||||
|
|
@ -239,8 +239,8 @@ class TestFlagCommitBuildToddler:
|
|||
}
|
||||
config = {
|
||||
"koji_url": "https://koji.fedoraproject.org",
|
||||
"pagure_url": "https://src.fedoraproject.org",
|
||||
"pagure_token": "ahah",
|
||||
"dist_git_url": "https://src.fedoraproject.org",
|
||||
"dist_git_token": "ahah",
|
||||
}
|
||||
assert toddler.process(config=config, message=msg) is None
|
||||
assert (
|
||||
|
|
@ -282,8 +282,8 @@ class TestFlagCommitBuildToddler:
|
|||
}
|
||||
config = {
|
||||
"koji_url": "https://koji.fedoraproject.org",
|
||||
"pagure_url": "https://src.fedoraproject.org",
|
||||
"pagure_token": "ahah",
|
||||
"dist_git_url": "https://src.fedoraproject.org",
|
||||
"dist_git_token": "ahah",
|
||||
}
|
||||
assert toddler.process(config=config, message=msg) is None
|
||||
assert (
|
||||
|
|
|
|||
|
|
@ -45,32 +45,25 @@ routing_keys = ["#"] # This is dynamically generated in the code
|
|||
# more of them.
|
||||
blocked_toddlers = ["debug"]
|
||||
|
||||
|
||||
[consumer_config.flag_ci_pr]
|
||||
# flag_ci_pr
|
||||
pagure_token_seed = "private random string to change"
|
||||
pagure_token = "private random string to change"
|
||||
pagure_url = "https://src.fedoraproject.org"
|
||||
|
||||
|
||||
[consumer_config.flag_commit_build]
|
||||
# flag_commit_build
|
||||
pagure_token = "private random string to change"
|
||||
pagure_url = "https://src.fedoraproject.org"
|
||||
koji_url = "https://koji.fedoraproject.org"
|
||||
|
||||
|
||||
[consumer_config.packager_bugzilla_sync]
|
||||
# Configuration file storing all the email overrides in the form of:
|
||||
# "foo@bar.com" = "bar@foo.org"
|
||||
# This is the same format as used by the distgit_bugzilla_sync cron/app
|
||||
email_overrides_file = "/path/to/email_overrides.toml"
|
||||
|
||||
# Base url of dist-git
|
||||
dist_git_url = "https://src.fedoraproject.org"
|
||||
|
||||
# List of accounts we do not want to report about
|
||||
ignorable_accounts = ["packagerbot", "zuul"]
|
||||
[consumer_config.default]
|
||||
# Configuration common to all toddlers.
|
||||
#
|
||||
# You can override any of these in the section of a particular toddler, e.g.:
|
||||
#
|
||||
# [consumer_config.default]
|
||||
# somekey = "somevalue"
|
||||
#
|
||||
# [consumer_config.default.subsection]
|
||||
# key1 = "value1"
|
||||
# key2 = "value2"
|
||||
#
|
||||
# ...
|
||||
#
|
||||
# [consumer_config.sometoddler]
|
||||
# somekey = "someothervalue"
|
||||
#
|
||||
# [consumer_config.sometoddler.subsection]
|
||||
# key2 = "othervalue2"
|
||||
|
||||
# Configuration used when sending notifications:
|
||||
mail_server = "bastion.fedoraproject.org"
|
||||
|
|
@ -87,10 +80,18 @@ bugzilla_username = "Bugzilla username (ie: email)"
|
|||
bugzilla_password = "Bugzilla password for that account"
|
||||
bugzilla_group = "fedora_contrib"
|
||||
|
||||
# Base URL for the Koji build system
|
||||
koji_url = "https://koji.fedoraproject.org"
|
||||
|
||||
[consumer_config.pdc_retired_packages]
|
||||
file_check_url = "https://src.fedoraproject.org/%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s"
|
||||
# Account to use to connect to Pagure-as-dist-git
|
||||
dist_git_url = "https://src.fedoraproject.org"
|
||||
dist_git_token_seed = "private random string to change"
|
||||
dist_git_token = "private random string to change"
|
||||
|
||||
[consumer_config.default.pdc_config]
|
||||
# Configuration to talk to PDC, as understood by pdc-client.
|
||||
server = "https://pdc.fedoraproject.org/rest_api/v1/"
|
||||
ssl_verify = false # Enable if using a self-signed cert
|
||||
# XXX - getting the token is a bit of a pain, but here's a walk through
|
||||
# 1) go to https://pdc.fedoraproject.org/ in your browser and login.
|
||||
# 2) go to https://pdc.fedoraproject.org/rest_api/v1/auth/token/obtain/
|
||||
|
|
@ -100,13 +101,24 @@ file_check_url = "https://src.fedoraproject.org/%(namespace)s/%(repo)s/blob/%(br
|
|||
# 6) before hitting enter, edit the command to add the following option
|
||||
# -H 'Accept: application/json' # to tell the API you want data
|
||||
# 7) the command should print out your token.
|
||||
|
||||
# Credentials to talk to PDC
|
||||
[consumer_config.pdc_retired_packages.pdc_config]
|
||||
server = "https://pdc.fedoraproject.org/rest_api/v1/"
|
||||
ssl_verify = false # Enable if using a self-signed cert
|
||||
token = "PUT_HERE_THE_TOKEN_OBTAINED_MANUALLY"
|
||||
|
||||
[consumer_config.flag_ci_pr]
|
||||
|
||||
[consumer_config.flag_commit_build]
|
||||
|
||||
[consumer_config.packager_bugzilla_sync]
|
||||
# Configuration file storing all the email overrides in the form of:
|
||||
# "foo@bar.com" = "bar@foo.org"
|
||||
# This is the same format as used by the distgit_bugzilla_sync cron/app
|
||||
email_overrides_file = "/path/to/email_overrides.toml"
|
||||
|
||||
# List of accounts we do not want to report about
|
||||
ignorable_accounts = ["packagerbot", "zuul"]
|
||||
|
||||
[consumer_config.pdc_retired_packages]
|
||||
file_check_url = "https://src.fedoraproject.org/%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s"
|
||||
|
||||
[qos]
|
||||
prefetch_size = 0
|
||||
prefetch_count = 25
|
||||
|
|
|
|||
|
|
@ -106,7 +106,7 @@ class FlagCIPR(ToddlerBase):
|
|||
pr_id = str(msg["artifact"]["id"])
|
||||
|
||||
commit_hash_text = " for %s" % msg["artifact"]["commit_hash"][:8]
|
||||
seed = config.get("pagure_token_seed") or ""
|
||||
seed = config.get("dist_git_token_seed", config.get("pagure_token_seed")) or ""
|
||||
|
||||
data = {
|
||||
"username": "Fedora CI",
|
||||
|
|
@ -116,7 +116,7 @@ class FlagCIPR(ToddlerBase):
|
|||
"uid": hashlib.md5((pr_id + seed).encode("utf-8")).hexdigest(),
|
||||
}
|
||||
|
||||
pagure_url = config["pagure_url"]
|
||||
dist_git_url = config.get("dist_git_url") or config["pagure_url"]
|
||||
|
||||
namespace = msg["artifact"]["repository"].split("/")[-2]
|
||||
repo = msg["artifact"]["repository"].split("/")[-1]
|
||||
|
|
@ -124,11 +124,12 @@ class FlagCIPR(ToddlerBase):
|
|||
target_url = "/".join(
|
||||
["api", "0", namespace, repo, "pull-request", pr_id, "flag"]
|
||||
)
|
||||
flag_url = pagure_url + "/" + target_url
|
||||
flag_url = dist_git_url + "/" + target_url
|
||||
_log.info("Flagging commit at: %s" % flag_url)
|
||||
|
||||
headers = {
|
||||
"Authorization": "token " + config["pagure_token"],
|
||||
"Authorization": "token " + config.get("dist_git_token")
|
||||
or config["pagure_token"],
|
||||
"User-Agent": "loopabull@fedora-infra",
|
||||
}
|
||||
|
||||
|
|
@ -137,7 +138,7 @@ class FlagCIPR(ToddlerBase):
|
|||
req = requests_session.request(
|
||||
method="POST", url=flag_url, headers=headers, data=data,
|
||||
)
|
||||
_log.info("Request to %s returned: %s" % (pagure_url, req.status_code))
|
||||
_log.info("Request to %s returned: %s" % (dist_git_url, req.status_code))
|
||||
if not req.ok:
|
||||
_log.debug(req.text)
|
||||
return 1
|
||||
|
|
|
|||
|
|
@ -127,14 +127,15 @@ class FlagCommitBuild(ToddlerBase):
|
|||
"url": _koji_link(config, msg),
|
||||
}
|
||||
|
||||
pagure_url = config["pagure_url"]
|
||||
dist_git_url = config.get("dist_git_url") or config["pagure_url"]
|
||||
|
||||
target_url = "/".join(["api", "0", "rpms", msg["name"], "c", commit, "flag"])
|
||||
flag_url = pagure_url + "/" + target_url
|
||||
flag_url = dist_git_url + "/" + target_url
|
||||
_log.info("Flagging commit at: %s" % flag_url)
|
||||
|
||||
headers = {
|
||||
"Authorization": "token " + config["pagure_token"],
|
||||
"Authorization": "token " + config.get("dist_git_token")
|
||||
or config["pagure_token"],
|
||||
"User-Agent": "toddlers@fedora-infra",
|
||||
}
|
||||
|
||||
|
|
@ -143,7 +144,7 @@ class FlagCommitBuild(ToddlerBase):
|
|||
req = requests_session.request(
|
||||
method="POST", url=flag_url, headers=headers, data=data,
|
||||
)
|
||||
_log.info("Request to %s returned: %s" % (pagure_url, req.status_code))
|
||||
_log.info("Request to %s returned: %s" % (dist_git_url, req.status_code))
|
||||
if not req.ok:
|
||||
_log.debug(" -- FAILED -- ")
|
||||
_log.debug(req.text)
|
||||
|
|
@ -151,7 +152,7 @@ class FlagCommitBuild(ToddlerBase):
|
|||
else:
|
||||
_log.info("All clear")
|
||||
_log.info(
|
||||
"User-URL: %s" % pagure_url
|
||||
"User-URL: %s" % dist_git_url
|
||||
+ "/"
|
||||
+ "/".join(["rpms", msg["name"], "c", commit])
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import fedora_messaging.exceptions
|
|||
|
||||
from . import plugins # noqa: F401
|
||||
from .base import ToddlerBase
|
||||
from .utils.misc import merge_dicts
|
||||
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
|
@ -39,6 +40,15 @@ class RunningToddler(object):
|
|||
"No toddlers found to run :("
|
||||
)
|
||||
|
||||
self.toddler_config = {}
|
||||
for toddler in self.toddlers:
|
||||
name = toddler.name
|
||||
# Bail out if two toddlers have the same name.
|
||||
if name in self.toddler_config:
|
||||
raise ValueError(f"Toddler named {name} already exists.")
|
||||
|
||||
self.toddler_config[name] = self._get_toddler_config(name)
|
||||
|
||||
topics = set()
|
||||
for toddler in self.toddlers:
|
||||
_log.debug(
|
||||
|
|
@ -57,6 +67,15 @@ class RunningToddler(object):
|
|||
)
|
||||
binding["routing_keys"] = topics
|
||||
|
||||
def _get_toddler_config(self, toddler_name: str) -> dict:
|
||||
"""Merge default and private configuration for a toddler."""
|
||||
base_config = fedora_messaging.config.conf["consumer_config"]
|
||||
|
||||
default_config = base_config.get("default", {})
|
||||
private_config = base_config.get(toddler_name, {})
|
||||
|
||||
return merge_dicts(default_config, private_config)
|
||||
|
||||
def __call__(self, message):
|
||||
"""
|
||||
Invoked when a message is received by the consumer.
|
||||
|
|
@ -74,12 +93,7 @@ class RunningToddler(object):
|
|||
toddler.name,
|
||||
message.id,
|
||||
)
|
||||
toddler.process(
|
||||
fedora_messaging.config.conf["consumer_config"].get(
|
||||
toddler.name
|
||||
),
|
||||
message,
|
||||
)
|
||||
toddler.process(self.toddler_config[toddler.name], message)
|
||||
except Exception:
|
||||
_log.exception(
|
||||
"Toddler '%s' failed to process message id: %s -- putting it back in the queue",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue