From b5ed2ce4e2d5737b9338ca2af6fa773e132d6013 Mon Sep 17 00:00:00 2001 From: Adam Williamson Date: Mon, 17 Jun 2019 11:23:22 -0700 Subject: [PATCH] openqa/dispatcher: set up email error reporting We had this for fedmsgs and it's pretty important, so let's try and do it for fedora-messaging. The role has a generic setup, infra-appropriate values are set in the group vars. Signed-off-by: Adam Williamson --- inventory/group_vars/openqa_common | 4 +++ roles/openqa/dispatcher/defaults/main.yml | 2 ++ roles/openqa/dispatcher/tasks/main.yml | 9 ++++++ .../fedora_openqa_resultsdb_reporter.toml.j2 | 28 +++++++++++++++++++ .../templates/fedora_openqa_scheduler.toml.j2 | 28 +++++++++++++++++++ .../fedora_openqa_wiki_reporter.toml.j2 | 28 +++++++++++++++++++ 6 files changed, 99 insertions(+) diff --git a/inventory/group_vars/openqa_common b/inventory/group_vars/openqa_common index fa117b83b5..7de554a709 100644 --- a/inventory/group_vars/openqa_common +++ b/inventory/group_vars/openqa_common @@ -47,6 +47,10 @@ openqa_amqp_resultsdb_reporter_routing_keys: ["org.fedoraproject.{{ deployment_t openqa_amqp_wiki_reporter_queue: "openqa{{ openqa_env_suffix }}_wiki_reporter" openqa_amqp_wiki_reporter_routing_keys: ["org.fedoraproject.{{ deployment_type }}.openqa.job.done"] +# fedora-messaging email error reporting settings +openqa_amqp_mailto: ["adamwill@fedoraproject.org", "lruzicka@fedoraproject.org"] +openqa_amqp_smtp: bastion + # http and NFS tcp_ports: [80, 2049] diff --git a/roles/openqa/dispatcher/defaults/main.yml b/roles/openqa/dispatcher/defaults/main.yml index c4221fde2c..002dc07670 100644 --- a/roles/openqa/dispatcher/defaults/main.yml +++ b/roles/openqa/dispatcher/defaults/main.yml @@ -17,3 +17,5 @@ openqa_amqp_reporter_key: /etc/fedora-messaging/fedora-key.pem openqa_amqp_reporter_cert: /etc/fedora-messaging/fedora-cert.pem openqa_amqp_resultsdb_reporter_routing_keys: ["org.fedoraproject.prod.openqa.job.done"] openqa_amqp_wiki_reporter_routing_keys: ["org.fedoraproject.prod.openqa.job.done"] +openqa_amqp_mailfrom: "root@{{ external_hostname }}" +openqa_amqp_smtp: localhost diff --git a/roles/openqa/dispatcher/tasks/main.yml b/roles/openqa/dispatcher/tasks/main.yml index d6007acfb0..cf319bb720 100644 --- a/roles/openqa/dispatcher/tasks/main.yml +++ b/roles/openqa/dispatcher/tasks/main.yml @@ -153,6 +153,15 @@ ## wiki reporter to subscribe to. The role default ## for this is the appropriate keys for the Fedora ## production broker. +# - openqa_amqp_mailto +## list - List of email addresses to email errors to. If set, +## the email log handler will be configured. +# - openqa_amqp_mailfrom +## string - From email address for error report emails. Defaults +## to "root@{{ external_hostname }}". +# - openqa_amqp_smtp +## string - Hostname of SMTP server to use for sending error +## emails. Defaults to 'localhost'. # # NOTE: There are still currently a couple of assumptions that the # openQA server boxes will always act as their own dispatchers, but diff --git a/roles/openqa/dispatcher/templates/fedora_openqa_resultsdb_reporter.toml.j2 b/roles/openqa/dispatcher/templates/fedora_openqa_resultsdb_reporter.toml.j2 index 069b7a44bb..acab96d872 100644 --- a/roles/openqa/dispatcher/templates/fedora_openqa_resultsdb_reporter.toml.j2 +++ b/roles/openqa/dispatcher/templates/fedora_openqa_resultsdb_reporter.toml.j2 @@ -64,20 +64,44 @@ class = "logging.StreamHandler" formatter = "simple" stream = "ext://sys.stdout" +{% if openqa_amqp_mailto is defined %} +[log_config.handlers.email] +class = "logging.handlers.SMTPHandler" +formatter = "simple" +level = "ERROR" +filters = ["rate_limit"] +mailhost = "{{ openqa_amqp_smtp }}" +fromaddr = "{{ openqa_amqp_mailfrom }}" +toaddrs = [{% for key in openqa_amqp_mailto %}"{{ key }}",{% endfor %}] +subject = "openQA ResultsDB reporter error" +{% endif %} + [log_config.loggers.OpenQAResultsDBReporter] level = "INFO" propagate = false +{% if openqa_amqp_mailto is defined %} +handlers = ["console", "email"] +{% else %} handlers = ["console"] +{% endif %} [log_config.loggers.fedora_messaging] level = "INFO" propagate = false +{% if openqa_amqp_mailto is defined %} +handlers = ["console", "email"] +{% else %} handlers = ["console"] +{% endif %} [log_config.loggers.twisted] level = "INFO" propagate = false +{% if openqa_amqp_mailto is defined %} +handlers = ["console", "email"] +{% else %} handlers = ["console"] +{% endif %} [log_config.loggers.pika] level = "WARNING" @@ -86,4 +110,8 @@ handlers = ["console"] [log_config.root] level = "ERROR" +{% if openqa_amqp_mailto is defined %} +handlers = ["console", "email"] +{% else %} handlers = ["console"] +{% endif %} diff --git a/roles/openqa/dispatcher/templates/fedora_openqa_scheduler.toml.j2 b/roles/openqa/dispatcher/templates/fedora_openqa_scheduler.toml.j2 index bf78953560..0ad20842a4 100644 --- a/roles/openqa/dispatcher/templates/fedora_openqa_scheduler.toml.j2 +++ b/roles/openqa/dispatcher/templates/fedora_openqa_scheduler.toml.j2 @@ -61,20 +61,44 @@ class = "logging.StreamHandler" formatter = "simple" stream = "ext://sys.stdout" +{% if openqa_amqp_mailto is defined %} +[log_config.handlers.email] +class = "logging.handlers.SMTPHandler" +formatter = "simple" +level = "ERROR" +filters = ["rate_limit"] +mailhost = "{{ openqa_amqp_smtp }}" +fromaddr = "{{ openqa_amqp_mailfrom }}" +toaddrs = [{% for key in openqa_amqp_mailto %}"{{ key }}",{% endfor %}] +subject = "openQA scheduler error" +{% endif %} + [log_config.loggers.OpenQAScheduler] level = "INFO" propagate = false +{% if openqa_amqp_mailto is defined %} +handlers = ["console", "email"] +{% else %} handlers = ["console"] +{% endif %} [log_config.loggers.fedora_messaging] level = "INFO" propagate = false +{% if openqa_amqp_mailto is defined %} +handlers = ["console", "email"] +{% else %} handlers = ["console"] +{% endif %} [log_config.loggers.twisted] level = "INFO" propagate = false +{% if openqa_amqp_mailto is defined %} +handlers = ["console", "email"] +{% else %} handlers = ["console"] +{% endif %} [log_config.loggers.pika] level = "WARNING" @@ -83,4 +107,8 @@ handlers = ["console"] [log_config.root] level = "ERROR" +{% if openqa_amqp_mailto is defined %} +handlers = ["console", "email"] +{% else %} handlers = ["console"] +{% endif %} diff --git a/roles/openqa/dispatcher/templates/fedora_openqa_wiki_reporter.toml.j2 b/roles/openqa/dispatcher/templates/fedora_openqa_wiki_reporter.toml.j2 index 37c284340b..113efb4ca5 100644 --- a/roles/openqa/dispatcher/templates/fedora_openqa_wiki_reporter.toml.j2 +++ b/roles/openqa/dispatcher/templates/fedora_openqa_wiki_reporter.toml.j2 @@ -64,20 +64,44 @@ class = "logging.StreamHandler" formatter = "simple" stream = "ext://sys.stdout" +{% if openqa_amqp_mailto is defined %} +[log_config.handlers.email] +class = "logging.handlers.SMTPHandler" +formatter = "simple" +level = "ERROR" +filters = ["rate_limit"] +mailhost = "{{ openqa_amqp_smtp }}" +fromaddr = "{{ openqa_amqp_mailfrom }}" +toaddrs = [{% for key in openqa_amqp_mailto %}"{{ key }}",{% endfor %}] +subject = "openQA wiki reporter error" +{% endif %} + [log_config.loggers.OpenQAWikiReporter] level = "INFO" propagate = false +{% if openqa_amqp_mailto is defined %} +handlers = ["console", "email"] +{% else %} handlers = ["console"] +{% endif %} [log_config.loggers.fedora_messaging] level = "INFO" propagate = false +{% if openqa_amqp_mailto is defined %} +handlers = ["console", "email"] +{% else %} handlers = ["console"] +{% endif %} [log_config.loggers.twisted] level = "INFO" propagate = false +{% if openqa_amqp_mailto is defined %} +handlers = ["console", "email"] +{% else %} handlers = ["console"] +{% endif %} [log_config.loggers.pika] level = "WARNING" @@ -86,4 +110,8 @@ handlers = ["console"] [log_config.root] level = "ERROR" +{% if openqa_amqp_mailto is defined %} +handlers = ["console", "email"] +{% else %} handlers = ["console"] +{% endif %}