openqa/dispatcher: set up email error reporting

We had this for fedmsgs and it's pretty important, so let's try
and do it for fedora-messaging. The role has a generic setup,
infra-appropriate values are set in the group vars.

Signed-off-by: Adam Williamson <awilliam@redhat.com>

inventory/group_vars/openqa_common

@@ -47,6 +47,10 @@ openqa_amqp_resultsdb_reporter_routing_keys: ["org.fedoraproject.{{ deployment_t
 openqa_amqp_wiki_reporter_queue: "openqa{{ openqa_env_suffix }}_wiki_reporter"
 openqa_amqp_wiki_reporter_routing_keys: ["org.fedoraproject.{{ deployment_type }}.openqa.job.done"]
 
+# fedora-messaging email error reporting settings
+openqa_amqp_mailto: ["adamwill@fedoraproject.org", "lruzicka@fedoraproject.org"]
+openqa_amqp_smtp: bastion
+
 # http and NFS
 tcp_ports: [80, 2049]
 

roles/openqa/dispatcher/defaults/main.yml

@@ -17,3 +17,5 @@ openqa_amqp_reporter_key: /etc/fedora-messaging/fedora-key.pem
 openqa_amqp_reporter_cert: /etc/fedora-messaging/fedora-cert.pem
 openqa_amqp_resultsdb_reporter_routing_keys: ["org.fedoraproject.prod.openqa.job.done"]
 openqa_amqp_wiki_reporter_routing_keys: ["org.fedoraproject.prod.openqa.job.done"]
+openqa_amqp_mailfrom: "root@{{ external_hostname }}"
+openqa_amqp_smtp: localhost

roles/openqa/dispatcher/tasks/main.yml

@@ -153,6 +153,15 @@
 ##               wiki reporter to subscribe to. The role default
 ##               for this is the appropriate keys for the Fedora
 ##               production broker.
+# - openqa_amqp_mailto
+##        list - List of email addresses to email errors to. If set,
+##               the email log handler will be configured.
+# - openqa_amqp_mailfrom
+##      string - From email address for error report emails. Defaults
+##               to "root@{{ external_hostname }}".
+# - openqa_amqp_smtp
+##      string - Hostname of SMTP server to use for sending error
+##               emails. Defaults to 'localhost'.
 #
 # NOTE: There are still currently a couple of assumptions that the
 # openQA server boxes will always act as their own dispatchers, but

roles/openqa/dispatcher/templates/fedora_openqa_resultsdb_reporter.toml.j2

@@ -64,20 +64,44 @@ class = "logging.StreamHandler"
 formatter = "simple"
 stream = "ext://sys.stdout"
 
+{% if openqa_amqp_mailto is defined %}
+[log_config.handlers.email]
+class = "logging.handlers.SMTPHandler"
+formatter = "simple"
+level = "ERROR"
+filters = ["rate_limit"]
+mailhost = "{{ openqa_amqp_smtp }}"
+fromaddr = "{{ openqa_amqp_mailfrom }}"
+toaddrs = [{% for key in openqa_amqp_mailto %}"{{ key }}",{% endfor %}]
+subject = "openQA ResultsDB reporter error"
+{% endif %}
+
 [log_config.loggers.OpenQAResultsDBReporter]
 level = "INFO"
 propagate = false
+{% if openqa_amqp_mailto is defined %}
+handlers = ["console", "email"]
+{% else %}
 handlers = ["console"]
+{% endif %}
 
 [log_config.loggers.fedora_messaging]
 level = "INFO"
 propagate = false
+{% if openqa_amqp_mailto is defined %}
+handlers = ["console", "email"]
+{% else %}
 handlers = ["console"]
+{% endif %}
 
 [log_config.loggers.twisted]
 level = "INFO"
 propagate = false
+{% if openqa_amqp_mailto is defined %}
+handlers = ["console", "email"]
+{% else %}
 handlers = ["console"]
+{% endif %}
 
 [log_config.loggers.pika]
 level = "WARNING"
@@ -86,4 +110,8 @@ handlers = ["console"]
 
 [log_config.root]
 level = "ERROR"
+{% if openqa_amqp_mailto is defined %}
+handlers = ["console", "email"]
+{% else %}
 handlers = ["console"]
+{% endif %}

roles/openqa/dispatcher/templates/fedora_openqa_scheduler.toml.j2

@@ -61,20 +61,44 @@ class = "logging.StreamHandler"
 formatter = "simple"
 stream = "ext://sys.stdout"
 
+{% if openqa_amqp_mailto is defined %}
+[log_config.handlers.email]
+class = "logging.handlers.SMTPHandler"
+formatter = "simple"
+level = "ERROR"
+filters = ["rate_limit"]
+mailhost = "{{ openqa_amqp_smtp }}"
+fromaddr = "{{ openqa_amqp_mailfrom }}"
+toaddrs = [{% for key in openqa_amqp_mailto %}"{{ key }}",{% endfor %}]
+subject = "openQA scheduler error"
+{% endif %}
+
 [log_config.loggers.OpenQAScheduler]
 level = "INFO"
 propagate = false
+{% if openqa_amqp_mailto is defined %}
+handlers = ["console", "email"]
+{% else %}
 handlers = ["console"]
+{% endif %}
 
 [log_config.loggers.fedora_messaging]
 level = "INFO"
 propagate = false
+{% if openqa_amqp_mailto is defined %}
+handlers = ["console", "email"]
+{% else %}
 handlers = ["console"]
+{% endif %}
 
 [log_config.loggers.twisted]
 level = "INFO"
 propagate = false
+{% if openqa_amqp_mailto is defined %}
+handlers = ["console", "email"]
+{% else %}
 handlers = ["console"]
+{% endif %}
 
 [log_config.loggers.pika]
 level = "WARNING"
@@ -83,4 +107,8 @@ handlers = ["console"]
 
 [log_config.root]
 level = "ERROR"
+{% if openqa_amqp_mailto is defined %}
+handlers = ["console", "email"]
+{% else %}
 handlers = ["console"]
+{% endif %}

roles/openqa/dispatcher/templates/fedora_openqa_wiki_reporter.toml.j2

@@ -64,20 +64,44 @@ class = "logging.StreamHandler"
 formatter = "simple"
 stream = "ext://sys.stdout"
 
+{% if openqa_amqp_mailto is defined %}
+[log_config.handlers.email]
+class = "logging.handlers.SMTPHandler"
+formatter = "simple"
+level = "ERROR"
+filters = ["rate_limit"]
+mailhost = "{{ openqa_amqp_smtp }}"
+fromaddr = "{{ openqa_amqp_mailfrom }}"
+toaddrs = [{% for key in openqa_amqp_mailto %}"{{ key }}",{% endfor %}]
+subject = "openQA wiki reporter error"
+{% endif %}
+
 [log_config.loggers.OpenQAWikiReporter]
 level = "INFO"
 propagate = false
+{% if openqa_amqp_mailto is defined %}
+handlers = ["console", "email"]
+{% else %}
 handlers = ["console"]
+{% endif %}
 
 [log_config.loggers.fedora_messaging]
 level = "INFO"
 propagate = false
+{% if openqa_amqp_mailto is defined %}
+handlers = ["console", "email"]
+{% else %}
 handlers = ["console"]
+{% endif %}
 
 [log_config.loggers.twisted]
 level = "INFO"
 propagate = false
+{% if openqa_amqp_mailto is defined %}
+handlers = ["console", "email"]
+{% else %}
 handlers = ["console"]
+{% endif %}
 
 [log_config.loggers.pika]
 level = "WARNING"
@@ -86,4 +110,8 @@ handlers = ["console"]
 
 [log_config.root]
 level = "ERROR"
+{% if openqa_amqp_mailto is defined %}
+handlers = ["console", "email"]
+{% else %}
 handlers = ["console"]
+{% endif %}