openqa/dispatcher: first cut at supporting fedora-messaging

This is a big change to support fedora-messaging based consumers
instead of fedmsg ones in the openqa/dispatcher role. This needs
the fedora-messaging branch of fedora_openqa, until that is
merged to stable. For now this is all conditionalized so prod
will continue to use fedmsg. This also improves some variable
usage and documentation in the fedmsg path too.

Signed-off-by: Adam Williamson <awilliam@redhat.com>

inventory/group_vars/openqa

@@ -22,6 +22,7 @@ openqa_key: "{{ prod_openqa_apikey }}"
 openqa_secret: "{{ prod_openqa_apisecret }}"
 
 wikitcms_token: "{{ private }}/files/openidc/production/wikitcms.json"
+openqa_resultsdb_url: http://resultsdb01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
 
 # The checkcompose settings below cause system(s) in this group to
 # send out check-compose reports. This could cause duplicate reports

inventory/group_vars/openqa_stg

@@ -41,6 +41,8 @@ openqa_secret: "{{ stg_openqa_apisecret }}"
 openqa_extraarches: true
 
 wikitcms_token: "{{ private }}/files/openidc/staging/wikitcms.json"
+openqa_resultsdb_url: http://resultsdb-stg01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
+openqa_fedoramessaging: true
 
 checkcompose_url: "https://{{ external_hostname }}"
 checkcompose_greenwaveurl: https://greenwave-web-greenwave.app.os.stg.fedoraproject.org

roles/openqa/dispatcher/defaults/main.yml

@@ -1,3 +1,4 @@
 openqa_hostname: localhost
 openqa_consumer: true
 openqa_extraarches: false
+openqa_fedoramessaging: false

roles/openqa/dispatcher/tasks/main.yml

@@ -1,41 +1,54 @@
 # Required vars with defaults
 # - openqa_hostname
-##      string - hostname of openQA server to run jobs for
+##      string - hostname of openQA server to run jobs for and query
+##               job results from
 ##      default - localhost
 # - openqa_consumer
 ##      bool - whether to enable the fedmsg consumer and run whenever
 ##             a compose completes
 ##      default - true
 # - external_hostname
-##       string - The public hostname of the openQA server (used in ResultsDB links)
-##       default - ansible_nodename
+##      string - The public hostname of the openQA server (used in
+##               links back from ResultsDB and wiki results)
+##      default - ansible_nodename
 # - openqa_extraarches
 ##      bool - when this is set, this instance will be configured to
 ##             create jobs for 'extra' arches beyond x86_64 (currently
 ##             ppc64 and aarch64). Only set it if this instance will
 ##             have worker hosts for those arches
 ##      default - false
+# - openqa_fedoramessaging
+##      bool - if true, configure and deploy the fedora-messaging
+##             consumers; if false, configure and deploy the fedmsg
+##             consumers
+##      default - false
 #
 # Optional vars
 # - wikitcms_token
 ##      string - a token file to install for unattended reporting to
-#                a wiki using openidc auth
+#                a wiki using openidc auth. If set, wiki reporting
+#                will be enabled, if unset, it will not. Note you must
+##               also set openqa_wiki_reporter_uuid (no
+##               deployment_type), openqa_wiki_reporter_stg_uuid
+##               (staging), or openqa_wiki_reporter_prod_uuid
+##               (production) if both this and openqa_fedoramessaging
+##               are set. You should probably NOT set this unless you
+##               are maintaining the Fedora infrastructure deployment
+# - openqa_resultsdb_url
+##      string - a ResultsDB API URL to report results to. If set,
+##               ResultsDB reporting will be enabled. Note you must
+##               also set openqa_resultsdb_reporter_uuid (no
+##               deployment_type), openqa_resultsdb_reporter_stg_uuid
+##               (staging), or openqa_resultsdb_reporter_prod_uuid
+##               (production) if both this and openqa_fedoramessaging
+##               are set. You should probably NOT set this unless you
+##               are maintaining the Fedora infrastructure deployment
 # - deployment_type
 ##      string - Fedora Infrastructure thing; for this role, the
 ##               fedora_openqa config file will be set appropriately
 ##               for infra deployments if this is set, so don't set
 ##               it for private deployments
 #
-# When all of the above are set, a wikitcms 'token' file will be
-# installed and result submission to the wiki will be enabled. If
-# deployment_type is set to 'prod', results will be submitted to the
-# production wiki in response to openQA production 'job complete'
-# fedmsgs; if set to 'stg', results will be submitted to the staging
-# wiki in response to openQA staging 'job complete' fedmsgs. Result
-# reporting to ResultsDB will also be enabled for the production
-# deployment. You probably should NOT set these unless you're
-# maintaining the Fedora infrastructure deployments.
-#
 # NOTE: There are still currently a couple of assumptions that the
 # openQA server boxes will always act as their own dispatchers, but
 # there is no longer any fundamental reason why this must be the case,
@@ -53,13 +66,29 @@
 
 - name: Install required packages
   dnf:
-    name: ['python3-fedfind', 'python3-wikitcms', 'python3-fedmsg',
-           'python3-openqa_client', 'python3-requests', 'python3-resultsdb_api',
-           'python3-resultsdb_conventions-fedora', 'python3-setuptools', 'python3-six']
+    name: ['python3-fedfind', 'python3-wikitcms', 'python3-openqa_client', 'python3-requests',
+           'python3-resultsdb_api', 'python3-resultsdb_conventions-fedora', 'python3-setuptools',
+           'python3-six']
     state: present
   tags:
   - packages
 
+- name: Install required packages (fedmsg)
+  dnf:
+    name: python3-fedmsg
+    state: present
+  when: "not openqa_fedoramessaging"
+  tags:
+  - packages
+
+- name: Install required packages (fedora-messaging)
+  dnf:
+    name: fedora-messaging
+    state: present
+  when: "openqa_fedoramessaging"
+  tags:
+  - packages
+
 - name: Install required packages (wiki oidc auth)
   dnf:
     name: python3-openidc-client
@@ -152,13 +181,45 @@
 #  tags:
 #  - config
 
-- name: Enable fedmsg consumer
+# We do this even if fedora-messaging is true, because in that case,
+# we write this file empty to ensure no fedmsg consumers are enabled
+- name: Configure fedmsg consumers
   template: src=openqa_consumer.py.j2 dest=/etc/fedmsg.d/openqa_consumer.py owner=root group=root mode=0644
   notify:
   - restart fedmsg-hub
   tags:
   - config
 
+- name: Configure fedora-messaging scheduler
+  template: src=fedora_openqa_scheduler.toml.j2 dest=/etc/fedora-messaging/fedora_openqa_scheduler.toml owner=root group=root mode=0640
+  when: "openqa_fedoramessaging"
+  tags:
+  - config
+
+- name: Configure fedora-messaging wiki reporter
+  template: src=fedora_openqa_wiki_reporter.toml.j2 dest=/etc/fedora-messaging/fedora_openqa_wiki_reporter.toml owner=root group=root mode=0640
+  when: "wikitcms_token is defined and openqa_fedoramessaging"
+  tags:
+  - config
+
+  - name: Configure fedora-messaging ResultsDB reporter
+  template: src=fedora_openqa_resultsdb_reporter.toml.j2 dest=/etc/fedora-messaging/fedora_openqa_resultsdb_reporter.toml owner=root group=root mode=0640
+  when: "openqa_resultsdb_url is defined and openqa_fedoramessaging"
+  tags:
+  - config
+
+- name: Enable and start fedora-messaging scheduler service
+  service: name=fm-consumer@fedora_openqa_scheduler enabled=yes state=started
+  when: "openqa_fedoramessaging"
+
+- name: Enable and start fedora-messaging wiki reporter service
+  service: name=fm-consumer@fedora_openqa_wiki_reporter enabled=yes state=started
+  when: "wikitcms_token is defined and openqa_fedoramessaging"
+
+- name: Enable and start fedora-messaging ResultsDB reporter service
+  service: name=fm-consumer@fedora_openqa_resultsdb_reporter enabled=yes state=started
+  when: "openqa_resultsdb_url is defined and openqa_fedoramessaging"
+
 - name: Set up cron job to schedule live-respins jobs
   copy: src=schedule-live-respins.cron dest=/etc/cron.hourly/schedule-live-respins owner=root group=root mode=0755
   tags:

roles/openqa/dispatcher/templates/fedora_openqa_resultsdb_reporter.toml.j2

@@ -0,0 +1,101 @@
+# fedora-messaging consumer configuration file for the production
+# openQA ResultsDB reporter. Note you must change the UUID here to
+# something unique before using this.
+#
+# This file is in the TOML format.
+{% if deployment_type is defined and deployment_type == 'stg' %}
+amqp_url = "amqps://fedora.stg:@rabbitmq.stg.fedoraproject.org/%2Fpublic_pubsub"
+{% else %}
+amqp_url = "amqps://fedora:@rabbitmq.fedoraproject.org/%2Fpublic_pubsub"
+{% endif %}
+callback = "fedora_openqa.consumer:OpenQAResultsDBReporter"
+
+[tls]
+ca_cert = "/etc/fedora-messaging/cacert.pem"
+keyfile = "/etc/fedora-messaging/fedora-key.pem"
+certfile = "/etc/fedora-messaging/fedora-cert.pem"
+
+[client_properties]
+app = "Fedora openQA"
+
+[exchanges."amq.topic"]
+type = "topic"
+durable = true
+auto_delete = false
+arguments = {}
+
+# Queue names *must* be in the normal UUID format: run "uuidgen" and use the
+# output as your queue name. If your queue is not exclusive, anyone can connect
+# and consume from it, causing you to miss messages, so do not share your queue
+# name. Any queues that are not auto-deleted on disconnect are garbage-collected
+# after approximately one hour.
+#
+# If you require a stronger guarantee about delivery, please talk to Fedora's
+# Infrastructure team.
+{% if deployment_type is defined and deployment_type == 'prod' %}
+[queues.{{ openqa_resultsdb_reporter_prod_uuid }}]
+{% elif deployment_type is defined and deployment_type == 'stg' %}
+[queues.{{ openqa_resultsdb_reporter_stg_uuid }}]
+{% else %}
+[queues.{{ openqa_resultsdb_reporter_uuid }}]
+{% endif %}
+durable = false
+auto_delete = true
+exclusive = true
+arguments = {}
+
+[[bindings]]
+{% if deployment_type is defined and deployment_type == 'prod' %}
+queue = "{{ openqa_resultsdb_reporter_prod_uuid }}"
+{% elif deployment_type is defined and deployment_type == 'stg' %}
+queue = "{{ openqa_resultsdb_reporter_stg_uuid }}"
+{% else %}
+queue = "{{ openqa_resultsdb_reporter_uuid }}"
+{% endif %}
+exchange = "amq.topic"
+{% if deployment_type is defined and deployment_type == 'stg' %}
+routing_keys = ["org.fedoraproject.stg.openqa.job.done"]
+{% else %}
+routing_keys = ["org.fedoraproject.prod.openqa.job.done"]
+{% endif %}
+
+[consumer_config]
+openqa_hostname = "{{ openqa_hostname }}"
+openqa_baseurl = "https://{{ external_hostname|default(ansible_nodename) }}"
+resultsdb_url = {{ openqa_resultsdb_url }}
+do_report = true
+
+[qos]
+prefetch_size = 0
+prefetch_count = 25
+
+[log_config]
+version = 1
+disable_existing_loggers = true
+
+[log_config.formatters.simple]
+format = "[%(levelname)s %(name)s] %(message)s"
+
+[log_config.handlers.console]
+class = "logging.StreamHandler"
+formatter = "simple"
+stream = "ext://sys.stdout"
+
+[log_config.loggers.fedora_messaging]
+level = "INFO"
+propagate = false
+handlers = ["console"]
+
+[log_config.loggers.twisted]
+level = "INFO"
+propagate = false
+handlers = ["console"]
+
+[log_config.loggers.pika]
+level = "WARNING"
+propagate = false
+handlers = ["console"]
+
+[log_config.root]
+level = "ERROR"
+handlers = ["console"]

roles/openqa/dispatcher/templates/fedora_openqa_scheduler.toml.j2

@@ -0,0 +1,96 @@
+# fedora-messaging consumer configuration file for the production
+# openQA scheduler. Note you must change the UUID here to something
+# unique before using this.
+#
+# This file is in the TOML format.
+
+# we listen to the prod broker here even for staging as we need to use
+# prod messages to schedule jobs in openQA staging, composes and
+# updates just don't show up on the staging broker.
+amqp_url = "amqps://fedora:@rabbitmq.fedoraproject.org/%2Fpublic_pubsub"
+callback = "fedora_openqa.consumer:OpenQAScheduler"
+
+[tls]
+ca_cert = "/etc/fedora-messaging/cacert.pem"
+keyfile = "/etc/fedora-messaging/fedora-key.pem"
+certfile = "/etc/fedora-messaging/fedora-cert.pem"
+
+[client_properties]
+app = "Fedora openQA"
+
+[exchanges."amq.topic"]
+type = "topic"
+durable = true
+auto_delete = false
+arguments = {}
+
+# Queue names *must* be in the normal UUID format: run "uuidgen" and use the
+# output as your queue name. If your queue is not exclusive, anyone can connect
+# and consume from it, causing you to miss messages, so do not share your queue
+# name. Any queues that are not auto-deleted on disconnect are garbage-collected
+# after approximately one hour.
+#
+# If you require a stronger guarantee about delivery, please talk to Fedora's
+# Infrastructure team.
+{% if deployment_type is defined and deployment_type == 'prod' %}
+[queues.{{ openqa_scheduler_prod_uuid }}]
+{% elif deployment_type is defined and deployment_type == 'stg' %}
+[queues.{{ openqa_scheduler_stg_uuid }}]
+{% else %}
+[queues.{{ openqa_scheduler_uuid }}]
+{% endif %}
+durable = false
+auto_delete = true
+exclusive = true
+arguments = {}
+
+[[bindings]]
+{% if deployment_type is defined and deployment_type == 'prod' %}
+queue = "{{ openqa_scheduler_prod_uuid }}"
+{% elif deployment_type is defined and deployment_type == 'stg' %}
+queue = "{{ openqa_scheduler_stg_uuid }}"
+{% else %}
+queue = "{{ openqa_scheduler_uuid }}"
+{% endif %}
+exchange = "amq.topic"
+routing_keys = ["org.fedoraproject.prod.pungi.compose.status.change",
+                "org.fedoraproject.prod.bodhi.update.request.testing",
+                "org.fedoraproject.prod.bodhi.update.edit"]
+
+[consumer_config]
+openqa_hostname = "{{ openqa_hostname }}"
+
+[qos]
+prefetch_size = 0
+prefetch_count = 25
+
+[log_config]
+version = 1
+disable_existing_loggers = true
+
+[log_config.formatters.simple]
+format = "[%(levelname)s %(name)s] %(message)s"
+
+[log_config.handlers.console]
+class = "logging.StreamHandler"
+formatter = "simple"
+stream = "ext://sys.stdout"
+
+[log_config.loggers.fedora_messaging]
+level = "INFO"
+propagate = false
+handlers = ["console"]
+
+[log_config.loggers.twisted]
+level = "INFO"
+propagate = false
+handlers = ["console"]
+
+[log_config.loggers.pika]
+level = "WARNING"
+propagate = false
+handlers = ["console"]
+
+[log_config.root]
+level = "ERROR"
+handlers = ["console"]

roles/openqa/dispatcher/templates/fedora_openqa_wiki_reporter.toml.j2

@@ -0,0 +1,105 @@
+# fedora-messaging consumer configuration file for the production
+# openQA wiki reporter. Note you must change the UUID here to
+# something unique before using this.
+#
+# This file is in the TOML format.
+{% if deployment_type is defined and deployment_type == 'stg' %}
+amqp_url = "amqps://fedora.stg:@rabbitmq.stg.fedoraproject.org/%2Fpublic_pubsub"
+{% else %}
+amqp_url = "amqps://fedora:@rabbitmq.fedoraproject.org/%2Fpublic_pubsub"
+{% endif %}
+callback = "fedora_openqa.consumer:OpenQAWikiReporter"
+
+[tls]
+ca_cert = "/etc/fedora-messaging/cacert.pem"
+keyfile = "/etc/fedora-messaging/fedora-key.pem"
+certfile = "/etc/fedora-messaging/fedora-cert.pem"
+
+[client_properties]
+app = "Fedora openQA"
+
+[exchanges."amq.topic"]
+type = "topic"
+durable = true
+auto_delete = false
+arguments = {}
+
+# Queue names *must* be in the normal UUID format: run "uuidgen" and use the
+# output as your queue name. If your queue is not exclusive, anyone can connect
+# and consume from it, causing you to miss messages, so do not share your queue
+# name. Any queues that are not auto-deleted on disconnect are garbage-collected
+# after approximately one hour.
+#
+# If you require a stronger guarantee about delivery, please talk to Fedora's
+# Infrastructure team.
+{% if deployment_type is defined and deployment_type == 'prod' %}
+[queues.{{ openqa_wiki_reporter_prod_uuid }}]
+{% elif deployment_type is defined and deployment_type == 'stg' %}
+[queues.{{ openqa_wiki_reporter_stg_uuid }}]
+{% else %}
+[queues.{{ openqa_wiki_reporter_uuid }}]
+{% endif %}
+durable = false
+auto_delete = true
+exclusive = true
+arguments = {}
+
+[[bindings]]
+{% if deployment_type is defined and deployment_type == 'prod' %}
+queue = "{{ openqa_wiki_reporter_prod_uuid }}"
+{% elif deployment_type is defined and deployment_type == 'stg' %}
+queue = "{{ openqa_wiki_reporter_stg_uuid }}"
+{% else %}
+queue = "{{ openqa_wiki_reporter_uuid }}"
+{% endif %}
+exchange = "amq.topic"
+{% if deployment_type is defined and deployment_type == 'stg' %}
+routing_keys = ["org.fedoraproject.stg.openqa.job.done"]
+{% else %}
+routing_keys = ["org.fedoraproject.prod.openqa.job.done"]
+{% endif %}
+
+[consumer_config]
+openqa_hostname = "{{ openqa_hostname }}"
+openqa_baseurl = "https://{{ external_hostname|default(ansible_nodename) }}"
+{% if deployment_type is defined and deployment_type == 'stg' %}
+wiki_hostname = "stg.fedoraproject.org"
+{% else %}
+wiki_hostname = "fedoraproject.org"
+{% endif %}
+do_report = true
+
+[qos]
+prefetch_size = 0
+prefetch_count = 25
+
+[log_config]
+version = 1
+disable_existing_loggers = true
+
+[log_config.formatters.simple]
+format = "[%(levelname)s %(name)s] %(message)s"
+
+[log_config.handlers.console]
+class = "logging.StreamHandler"
+formatter = "simple"
+stream = "ext://sys.stdout"
+
+[log_config.loggers.fedora_messaging]
+level = "INFO"
+propagate = false
+handlers = ["console"]
+
+[log_config.loggers.twisted]
+level = "INFO"
+propagate = false
+handlers = ["console"]
+
+[log_config.loggers.pika]
+level = "WARNING"
+propagate = false
+handlers = ["console"]
+
+[log_config.root]
+level = "ERROR"
+handlers = ["console"]

roles/openqa/dispatcher/templates/openqa_consumer.py.j2

@@ -1,4 +1,5 @@
 config = {
+{% if not openqa_fedoramessaging %}
 {% if openqa_consumer %}
     'fedora_openqa.scheduler.prod.enabled': True,
 {% endif %}
@@ -6,11 +7,16 @@ config = {
 {% if wikitcms_token is defined %}
     'fedora_openqa.reporter.wiki.prod.enabled': True,
 {% endif %}
+{% if openqa_resultsdb_url is defined %}
     'fedora_openqa.reporter.resultsdb.prod.enabled': True,
+{% endif %}
 {% elif deployment_type is defined and deployment_type == 'stg' %}
 {% if wikitcms_token is defined %}
     'fedora_openqa.reporter.wiki.stg.enabled': True,
 {% endif %}
+{% if openqa_resultsdb_url is defined %}
     'fedora_openqa.reporter.resultsdb.stg.enabled': True,
 {% endif %}
+{% endif %}
+{% endif %}
 }

roles/openqa/dispatcher/templates/schedule.conf.j2

@@ -1,24 +1,23 @@
 [report]
+resultsdb_url: {{ openqa_resultsdb_url }}
 {% if deployment_type is defined and deployment_type == 'prod' %}
-resultsdb_url: http://resultsdb01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
 wiki_hostname: fedoraproject.org
 [consumers]
-prod_oqa_hostname: localhost
+prod_oqa_hostname: {{ openqa_hostname }}
 prod_oqa_baseurl: https://{{ external_hostname|default(ansible_nodename) }}
 prod_wiki_hostname: fedoraproject.org
 prod_wiki_report: true
-prod_rdb_url: http://resultsdb01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
+prod_rdb_url: {{ openqa_resultsdb_url }}
 prod_rdb_report: true
 {% elif deployment_type is defined and deployment_type == 'stg' %}
-resultsdb_url: http://resultsdb-stg01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
 wiki_hostname: stg.fedoraproject.org
 [consumers]
 # as we use the production scheduler
-prod_oqa_hostname: localhost
-stg_oqa_hostname: localhost
+prod_oqa_hostname: {{ openqa_hostname }}
+stg_oqa_hostname: {{ openqa_hostname }}
 stg_oqa_baseurl: https://{{ external_hostname|default(ansible_nodename) }}
 stg_wiki_hostname: stg.fedoraproject.org
 stg_wiki_report: true
-stg_rdb_url: http://resultsdb-stg01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
+stg_rdb_url: {{ openqa_resultsdb_url }}
 stg_rdb_report: true
 {% endif %}