diff --git a/requirements.txt b/requirements.txt index 165e12c..a4cf597 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ bs4 defusedxml fasjson-client fedora-messaging +fedora-messaging-git-hook-messages GitPython koji requests diff --git a/tests/plugins/test_distgit_commit_processor.py b/tests/plugins/test_distgit_commit_processor.py new file mode 100644 index 0000000..2d090e3 --- /dev/null +++ b/tests/plugins/test_distgit_commit_processor.py @@ -0,0 +1,214 @@ +from contextlib import nullcontext +import datetime as dt +import random +from typing import ContextManager, Optional +from unittest import mock + +from fedora_messaging.message import Message +from fedora_messaging_git_hook_messages import CommitV1 +import pytest + +from toddlers.exceptions import ConfigError +from toddlers.plugins import distgit_commit_processor + + +class TestDistGitCommitProcessor: + """Test the DistGitCommitProcessor toddler plugin.""" + + toddler_cls = distgit_commit_processor.DistGitCommitProcessor + + @pytest.mark.parametrize( + "topic, valid", + ( + pytest.param("org.fedoraproject.prod.git.receive", True, id="prod-valid"), + pytest.param("org.fedoraproject.stg.git.receive", True, id="stg-valid"), + pytest.param( + "org.fedoraproject.prod.pagure.git.receive", + False, + id="pagure-prod-invalid", + ), + pytest.param( + "org.fedoraproject.stg.pagure.git.receive", + False, + id="pagure-stg-invalid", + ), + pytest.param( + "pagure.io.prod.pagure.git.receive", False, id="pagure-io-invalid" + ), + ), + ) + def test_accepts_topic( + self, + topic: str, + valid: bool, + toddler: distgit_commit_processor.DistGitCommitProcessor, + ) -> None: + assert toddler.accepts_topic(topic) is valid + + @pytest.mark.parametrize( + "namespace, repo, path, result", + ( + ("rpms", "kernel", "/some/root/rpms/kernel.git/", False), + ("rpms", "kernel", "/some/root/rpms/kernel/", False), + ("frobozzniks", "blomp", "/some/root/rpms/kernel.git/", False), + ("rpms", "kernel", "/some/root/fork/foo/rpms/kernel.git/", True), + ("rpms", "kernel", "/some/root/fork/foo/rpms/kernel", True), + (None, "project", "/some/root/project.git/", False), + ), + ) + def test_ignore_commit( + self, + namespace: Optional[str], + repo: str, + path: str, + result: bool, + toddler: distgit_commit_processor.DistGitCommitProcessor, + caplog: pytest.LogCaptureFixture, + ): + if namespace: + repo_with_ns = namespace + "/" + repo + else: + repo_with_ns = repo + + commit = { + "namespace": namespace, + "repo": repo, + "path": path, + } + body = {"id": "BOO", "agent": "m0rk", "commit": commit} + + message = CommitV1(body=body) + + assert toddler.ignore_commit(message) is result + + if repo_with_ns in path: + assert f"Message {message.id} mismatch" not in caplog.text + else: + assert f"Message {message.id} mismatch" in caplog.text + + @pytest.mark.parametrize( + "testcase", + ( + "success", + "success-loglevel-info", + "success-default-subject", + "success-default-content", + "failure-wrong-msg-type", + "failure-wrong-msg-type-loglevel-info", + "failure-fork-commit", + "failure-config-error", + ), + ) + def test_process( + self, + testcase: str, + toddler: distgit_commit_processor.DistGitCommitProcessor, + caplog: pytest.LogCaptureFixture, + ) -> None: + success = "success" in testcase + wrong_msg_type = "wrong-msg-type" in testcase + fork_commit = "fork-commit" in testcase + config_error = "config-error" in testcase + loglevel_info = "loglevel-info" in testcase + default_subject = "default-subject" in testcase + default_content = "default-content" in testcase + + # Appease mypy + exception_ctx: ContextManager + + now = dt.datetime.now(tz=dt.timezone.utc).isoformat() + + commit = { + "namespace": "rpms", + "repo": "kernel", + "path": "/some/root/rpms/kernel.git/", + "branch": "rawhide", + "rev": "deadbeef", + "name": "Mork", + "email": "mork@ork.org", + "date": now, + "summary": "Did the thing", + "message": "Did the thing\n\nAmazing.", + "patch": "No, I won’t fake a diff here.", + } + body = {"agent": "m0rk", "commit": commit} + + if fork_commit: + commit["path"] = "/some/root/fork/foo/rpms/kernel.git/" + + msg = CommitV1(body=body) + if wrong_msg_type: + msg = Message(body=body) + + # Config items which must be set + config = { + "mail_server": "bastion.fedoraproject.org", + "mail_from": "notifications@fedoraproject.org", + "mail_to": "scm-commits@lists.fedoraproject.org", + } + if config_error: + # Nuke a random configuration item + del config[list(config)[random.randint(0, len(config) - 1)]] + exception_ctx = pytest.raises(ConfigError) + else: + exception_ctx = nullcontext() + + if not default_subject: + config["mail_subject_tmpl"] = "SUBJECT-IS-SET: {message.summary}" + + if not default_content: + config["mail_content_tmpl"] = "CONTENT-IS-SET\n{message}" + + with caplog.at_level( + "INFO" if loglevel_info else "DEBUG" + ), exception_ctx, mock.patch.object( + distgit_commit_processor, "send_email" + ) as send_email: + toddler.process(config, msg) + + if not loglevel_info: + assert "Processing message:" in caplog.text + + if success: + send_email.assert_called_with( + to_addresses=[config["mail_to"]], + from_address=config["mail_from"], + subject=mock.ANY, + content=mock.ANY, + mail_server=config["mail_server"], + ) + + subject = send_email.call_args.kwargs["subject"] + assert ( + f"{body['agent']} pushed to {commit['namespace']}/{commit['repo']}" + in subject + ) + assert f"({commit['branch']})" in subject + assert f"\"{commit['summary']}\"" in subject + if default_subject: + assert "SUBJECT-IS-SET" not in subject + else: + assert "SUBJECT-IS-SET" in subject + + content = send_email.call_args.kwargs["content"] + assert f"From {commit['rev']}" in content + assert f"From: {commit['name']} <{commit['email']}>" in content + assert f"Date: {now}" in content + assert f"Subject: {commit['summary']}" in content + assert commit["message"] in content + if default_content: + assert "CONTENT-IS-SET" not in content + else: + assert "CONTENT-IS-SET" in content + else: + send_email.assert_not_called() + + if wrong_msg_type and not loglevel_info: + assert "Skipping message" in caplog.text + else: + assert "Skipping message" not in caplog.text + + if fork_commit: + assert "Ignoring message" in caplog.text + else: + assert "Ignoring message" not in caplog.text diff --git a/toddlers.toml.example b/toddlers.toml.example index 1ec00ff..1987d5e 100644 --- a/toddlers.toml.example +++ b/toddlers.toml.example @@ -342,3 +342,15 @@ exclude_users = [] notify_emails = [ "root@localhost.localdomain", ] + +# Configuration section for distgit_commit_processor +[consumer_config.distgit_commit_processor] +mail_from = "notifications@fedoraproject.org" +mail_to = "scm-commits@lists.fedoraproject.org" +mail_subject_tmpl = "{message.summary}" +mail_content_tmpl = """Notification time stamped {headers['sent-at']} + +{message} + + {commit['url']} +""" diff --git a/toddlers/exceptions/__init__.py b/toddlers/exceptions/__init__.py index 2ed8d0c..b49908f 100644 --- a/toddlers/exceptions/__init__.py +++ b/toddlers/exceptions/__init__.py @@ -1,2 +1,3 @@ +from .config_error import ConfigError # noqa: F401 from .pagure_error import PagureError # noqa: F401 from .validation_error import ValidationError # noqa: F401 diff --git a/toddlers/exceptions/config_error.py b/toddlers/exceptions/config_error.py new file mode 100644 index 0000000..3b3acaf --- /dev/null +++ b/toddlers/exceptions/config_error.py @@ -0,0 +1,7 @@ +""" +Exception that is raised when configuration is broken. +""" + + +class ConfigError(Exception): + pass diff --git a/toddlers/plugins/distgit_commit_processor.py b/toddlers/plugins/distgit_commit_processor.py new file mode 100644 index 0000000..6a7d1ab --- /dev/null +++ b/toddlers/plugins/distgit_commit_processor.py @@ -0,0 +1,137 @@ +""" +This plugin takes as input the Fedora messages published under the topic +``pagure.git.receive`` (for dist-git) and sends out emails to scm-commits-list. +""" + +import json +import logging +from typing import Any, Optional + +from fedora_messaging.api import Message +from fedora_messaging_git_hook_messages import CommitV1 + +from toddlers.base import ToddlerBase +from toddlers.exceptions import ConfigError +from toddlers.utils.notify import send_email + +log = logging.getLogger(__name__) + + +class DistGitCommitProcessor(ToddlerBase): + """A processor which sends emails about commits to dist-git repositories to + scm-commits-list. + """ + + name: str = "distgit_commit_processor" + + amqp_topics: list[str] = ["org.fedoraproject.*.git.receive"] + + def accepts_topic(self, topic: str) -> bool: + """Check whether this toddler is interested in messages of a topic. + + :arg topic: Topic to check. + + :returns: Whether or not this toddler accepts the topic. + """ + return ( + topic.startswith("org.fedoraproject.") + and topic.endswith(".git.receive") + and not topic.endswith(".pagure.git.receive") # different type of message + ) + + def ignore_commit(self, message: CommitV1) -> bool: + """Determine if a message pertains to a forked repo. + + :arg message: a CommitV1 message from the bus + :returns: True if the message is about a fork, not upstream + """ + commit = message.body["commit"] + namespace = commit["namespace"] + repo = commit["repo"] + + if namespace: + repo_with_ns = namespace + "/" + repo + else: + repo_with_ns = repo + + orig_path = commit["path"] + path = orig_path.rstrip("/") + + if path.endswith(".git"): + path = path[:-4] + + path = path.rstrip("/") + + if path.endswith(f"/{repo_with_ns}"): + return "/fork/" in path[: -len(repo_with_ns)] + + log.warning( + "Message %s mismatch: path=%r namespace=%r repo=%r", + message.id, + orig_path, + namespace, + repo, + ) + + # Something is off, rather notify superfluously. + return False + + def process(self, config: dict[str, Any], message: Message) -> None: + """Process a given message. + + :arg config: Toddlers configuration + :arg message: Message to process + """ + if log.getEffectiveLevel() <= logging.DEBUG: + log.debug("Processing message:\n%s", json.dumps(message.body, indent=2)) + + if not isinstance(message, CommitV1): + log.debug( + "Skipping message %s of incompatible type: %s", + message.id, + type(message), + ) + return + + if self.ignore_commit(message): + log.debug("Ignoring message %r", message) + return + + # These must be set + mail_from: Optional[str] = config.get("mail_from") + mail_to: Optional[str] = config.get("mail_to") + mail_server: Optional[str] = config.get("mail_server") + + if not mail_from or not mail_to or not mail_server: + raise ConfigError( + "Invalid toddler configuration: mail_from, mail_to and mail_server have to be set." + ) + + # These have good default fallbacks + mail_subject_tmpl: Optional[str] = config.get("mail_subject_tmpl") + mail_content_tmpl: Optional[str] = config.get("mail_content_tmpl") + + tmpl_params = { + "message": message, + "headers": message._headers, + "body": message.body, + "commit": message.body["commit"], + } + + if mail_subject_tmpl: + subject = mail_subject_tmpl.format(**tmpl_params) + else: + subject = message.summary + + if mail_content_tmpl: + content = mail_content_tmpl.format(**tmpl_params) + else: + content = str(message) + + send_email( + to_addresses=[mail_to], + from_address=mail_from, + subject=subject, + content=content, + mail_server=mail_server, + )