Infrastructure/ansible
3
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

openqa/dispatcher: always use fedora-messaging, ditch fedmsg

Browse source 
This ditches the ability to choose between fedora-messaging and
fedmsg and always uses fedora-messaging; with this, we'll switch
the prod instance to fedora-messaging also (fingers crossed).

Signed-off-by: Adam Williamson <awilliam@redhat.com>
This commit is contained in:
Adam Williamson 2019-06-17 12:15:35 -07:00
parent 0a9630f0a4
commit 97dd8ae650
4 changed files with 42 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

1
inventory/group_vars/openqa_stg
View file

@ -37,7 +37,6 @@ openqa_extraarches: true
wikitcms_token: "{{ private }}/files/openidc/staging/wikitcms.json"
openqa_wikitcms_hostname: stg.fedoraproject.org
openqa_resultsdb_url: http://resultsdb-stg01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
openqa_fedoramessaging: true
# these should really be secrets, but it's not *that* big a deal that
# they aren't, just means someone could theoretically block us
openqa_scheduler_uuid: 47acfdb9-e323-4a13-a940-fdb903934e60

1
roles/openqa/dispatcher/defaults/main.yml
View file

@ -1,7 +1,6 @@
openqa_hostname: localhost
openqa_consumer: true
openqa_extraarches: false
openqa_fedoramessaging: false
openqa_amqp_passive: false
openqa_amqp_scheduler_url: "amqps://fedora:@rabbitmq.fedoraproject.org/%2Fpublic_pubsub"
openqa_amqp_scheduler_cacert: /etc/fedora-messaging/cacert.pem

121
roles/openqa/dispatcher/tasks/main.yml
View file

@ -17,12 +17,6 @@
## ppc64 and aarch64). Only set it if this instance will
## have worker hosts for those arches
## default - false
# - openqa_fedoramessaging
## bool - if true, configure and deploy the fedora-messaging
## consumers; if false, configure and deploy the fedmsg
## consumers. Note you must also set openqa_scheduler_uuid
## for non-Fedora-infra deployments, if this is set
## default - false
#
# Optional vars
# - wikitcms_token
@ -30,25 +24,22 @@
## a wiki using openidc auth. If this and
## openqa_wikitcms_hostname are set, wiki reporting will
## be enabled, if unset, it will not. Note you must also
## set openqa_amqp_wiki_reporter_queue if both this and
## openqa_fedoramessaging are set. You should probably
## NOT set this unless you are maintaining the Fedora
## infrastructure deployment.
## set openqa_amqp_wiki_reporter_queue if this is set.
## You should probably NOT set this unless you are
## maintaining the Fedora infrastructure deployment.
# - openqa_wikitcms_hostname
## string - Hostname of a wiki to report results to. If this and
## wikitcms_token are set, wiki reporting will be
## enabled. Note you must also set
## openqa_amqp_wiki_reporter_queue if both this and
## openqa_fedoramessaging are set. You should probably
## NOT set this unless you are maintaining the Fedora
## infrastructure deployment.
## openqa_amqp_wiki_reporter_queue if this is set. You
## should probably NOT set this unless you are
## maintaining the Fedora infrastructure deployment.
# - openqa_resultsdb_url
## string - a ResultsDB API URL to report results to. If set,
## ResultsDB reporting will be enabled. Note you must
## also set openqa_amqp_resultsdb_reporter_queue if both
## this and openqa_fedoramessaging are set. You should
## probably NOT set this unless you are maintaining the
## Fedora infrastructure deployment
## also set openqa_amqp_resultsdb_reporter_queue if this
## is set. You should probably NOT set this unless you
## are maintaining the Fedora infrastructure deployment.
# - deployment_type
## string - Fedora Infrastructure thing; for this role, the
## fedora_openqa config file and fedora-messaging config
@ -85,10 +76,9 @@
## enabled. To use the fedora-messaging scheduler with
## public authentication on the Fedora production AMQP
## broker (which is what you'd typically want), you only
## need to set this and set openqa_fedoramessaging to
## true. This should be a unique and private string; the
## official recommendation is to use a random UUID
## generated by uuidgen.
## need to set this. This should be a unique and private
## string; the official recommendation is to use a
## random UUID generated by uuidgen.
# - openqa_amqp_scheduler_routing_keys
## list - List of routing key names for the fedora-messaging
## scheduler to subscribe to. The role default for this
@ -123,11 +113,10 @@
## infra deployments should report to Fedora ResultsDB.
## To use the fedora-messaging scheduler with public
## authentication on the Fedora production AMQP broker,
## you only need to set this and set
## openqa_fedoramessaging to true; openqa_resultsdb_url
## must also be set correctly. This should be a unique
## and private string; the official recommendation is to
## use a random UUID generated by uuidgen.
## you only need to set this; openqa_resultsdb_url must
## also be set correctly. This should be a unique and
## private string; the official recommendation is to use
## a random UUID generated by uuidgen.
# - openqa_amqp_resultsdb_reporter_routing_keys
## list - List of routing key names for the fedora-messaging
## ResultsDB reporter to subscribe to. The role default
@ -142,12 +131,10 @@
## deployments should report to the Fedora wikis. To use
## the fedora-messaging scheduler with public
## authentication on the Fedora production AMQP broker,
## you only need to set this and set
## openqa_fedoramessaging to true;
## openqa_wikitcms_hostname must also be set correctly.
## This should be a unique and private string; the
## official recommendation is to use a random UUID
## generated by uuidgen.
## you only need to set this; openqa_wikitcms_hostname
## must also be set correctly. This should be a unique
## and private string; the official recommendation is to
## use a random UUID generated by uuidgen.
# - openqa_amqp_wiki_reporter_routing_keys
## list - List of routing key names for the fedora-messaging
## wiki reporter to subscribe to. The role default
@ -180,29 +167,13 @@
- name: Install required packages
package:
name: ['python3-fedfind', 'python3-wikitcms', 'python3-openqa_client', 'python3-requests',
'python3-resultsdb_api', 'python3-resultsdb_conventions-fedora', 'python3-setuptools',
'python3-six']
name: ['fedora-messaging', python3-fedfind', 'python3-wikitcms', 'python3-openqa_client',
'python3-requests', 'python3-resultsdb_api', 'python3-resultsdb_conventions-fedora',
'python3-setuptools', 'python3-six']
state: present
tags:
- packages
- name: Install required packages (fedmsg)
package:
name: python3-fedmsg
state: present
when: "not openqa_fedoramessaging|bool"
tags:
- packages
- name: Install required packages (fedora-messaging)
package:
name: fedora-messaging
state: present
when: "openqa_fedoramessaging|bool"
tags:
- packages
- name: Install required packages (wiki oidc auth)
package:
name: python3-openidc-client
@ -245,8 +216,8 @@
args:
chdir: /root/fedora_openqa
when: "gittools is changed or not insttools.stat.exists"
notify:
- restart fedmsg-hub
# notify:
# - restart fedmsg-hub
# For now, we only have enough ppc64/aarch64 workers for one instance,
# so we have a variant WANTED file that we deploy based on a config
@ -254,8 +225,8 @@
- name: Install extra arch WANTED images config if enabled
copy: src=images.json.extraarches dest=/etc/fedora-openqa/images.json
when: openqa_extraarches
notify:
- restart fedmsg-hub
# notify:
# - restart fedmsg-hub
- name: openQA client config
template: src=client.conf.j2 dest=/etc/openqa/client.conf owner=root group=fedmsg mode=0640
@ -267,8 +238,8 @@
- name: Write schedule.conf
template: src=schedule.conf.j2 dest=/etc/fedora-openqa/schedule.conf owner=root group=root mode=0644
notify:
- restart fedmsg-hub
# notify:
# - restart fedmsg-hub
tags:
- config
@ -289,16 +260,8 @@
tags:
- config
#- name: Write wikitcms token file for fedmsg
# copy: src={{ wikitcms_token }} dest=/usr/share/fedmsg/.openidc/oidc_wikitcms.json owner=root group=fedmsg mode=0660
# when: "wikitcms_token is defined"
# tags:
# - config
# We do this even if fedora-messaging is true, because in that case,
# we write this file empty to ensure no fedmsg consumers are enabled
- name: Configure fedmsg consumers
template: src=openqa_consumer.py.j2 dest=/etc/fedmsg.d/openqa_consumer.py owner=root group=root mode=0644
- name: Wipe the old fedmsg consumer config file
file: path=/etc/fedmsg.d/openqa_consumer.py state=absent
notify:
- restart fedmsg-hub
tags:
@ -311,7 +274,7 @@
owner: root
group: root
state: directory
when: "openqa_fedoramessaging|bool and deployment_type is defined"
when: "deployment_type is defined"
tags:
- config
@ -322,7 +285,7 @@
mode: 0644
owner: root
group: root
when: "openqa_fedoramessaging|bool and deployment_type is defined"
when: "deployment_type is defined"
tags:
- config
@ -333,7 +296,7 @@
mode: 0600
owner: root
group: root
when: "openqa_fedoramessaging|bool and deployment_type is defined"
when: "deployment_type is defined"
tags:
- config
@ -344,7 +307,7 @@
mode: 0644
owner: root
group: root
when: "openqa_fedoramessaging|bool and deployment_type is defined and deployment_type == 'stg'"
when: "deployment_type is defined and deployment_type == 'stg'"
tags:
- config
@ -355,42 +318,42 @@
mode: 0600
owner: root
group: root
when: "openqa_fedoramessaging|bool and deployment_type is defined and deployment_type == 'stg'"
when: "deployment_type is defined and deployment_type == 'stg'"
tags:
- config
- name: Configure fedora-messaging scheduler
template: src=fedora_openqa_scheduler.toml.j2 dest=/etc/fedora-messaging/fedora_openqa_scheduler.toml owner=root group=root mode=0640
register: fmscheduler
when: "openqa_amqp_scheduler_queue is defined and openqa_fedoramessaging|bool"
when: "openqa_amqp_scheduler_queue is defined"
tags:
- config
- name: Configure fedora-messaging wiki reporter
template: src=fedora_openqa_wiki_reporter.toml.j2 dest=/etc/fedora-messaging/fedora_openqa_wiki_reporter.toml owner=root group=root mode=0640
register: fmwiki
when: "openqa_amqp_wiki_reporter_queue is defined and openqa_fedoramessaging|bool"
when: "openqa_amqp_wiki_reporter_queue is defined"
tags:
- config
- name: Configure fedora-messaging ResultsDB reporter
template: src=fedora_openqa_resultsdb_reporter.toml.j2 dest=/etc/fedora-messaging/fedora_openqa_resultsdb_reporter.toml owner=root group=root mode=0640
register: fmresultsdb
when: "openqa_amqp_resultsdb_reporter_queue is defined and openqa_fedoramessaging|bool"
when: "openqa_amqp_resultsdb_reporter_queue is defined"
tags:
- config
- name: Enable and start fedora-messaging scheduler service
service: name=fm-consumer@fedora_openqa_scheduler enabled=yes state=started
when: "openqa_amqp_scheduler_queue is defined and openqa_fedoramessaging|bool"
when: "openqa_amqp_scheduler_queue is defined"
- name: Enable and start fedora-messaging wiki reporter service
service: name=fm-consumer@fedora_openqa_wiki_reporter enabled=yes state=started
when: "openqa_amqp_wiki_reporter_queue is defined and openqa_fedoramessaging|bool"
when: "openqa_amqp_wiki_reporter_queue is defined"
- name: Enable and start fedora-messaging ResultsDB reporter service
service: name=fm-consumer@fedora_openqa_resultsdb_reporter enabled=yes state=started
when: "openqa_amqp_resultsdb_reporter_queue is defined and openqa_fedoramessaging|bool"
when: "openqa_amqp_resultsdb_reporter_queue is defined"
- name: Restart fedora-messaging scheduler service
command: /usr/local/bin/conditional-restart.sh fm-consumer@fedora_openqa_scheduler

22
roles/openqa/dispatcher/templates/openqa_consumer.py.j2
View file

@ -1,22 +0,0 @@
config = {
{% if not openqa_fedoramessaging %}
{% if openqa_consumer %}
'fedora_openqa.scheduler.prod.enabled': True,
{% endif %}
{% if deployment_type is defined and deployment_type == 'prod' %}
{% if wikitcms_token is defined %}
'fedora_openqa.reporter.wiki.prod.enabled': True,
{% endif %}
{% if openqa_resultsdb_url is defined %}
'fedora_openqa.reporter.resultsdb.prod.enabled': True,
{% endif %}
{% elif deployment_type is defined and deployment_type == 'stg' %}
{% if wikitcms_token is defined %}
'fedora_openqa.reporter.wiki.stg.enabled': True,
{% endif %}
{% if openqa_resultsdb_url is defined %}
'fedora_openqa.reporter.resultsdb.stg.enabled': True,
{% endif %}
{% endif %}
{% endif %}
}