More cleanup on the openQA AMQP stuff

nirik and I went around and around a bit today and ended up back
where we started, but with a clearer understanding of where that
this. This explains it a bit better, and makes what's actually
going on in various places clearer with the use of appropriate
shared variables. This should not actually *change* anything at
all when deployed.

Signed-off-by: Adam Williamson <awilliam@redhat.com>

inventory/group_vars/checkcompose

@@ -11,9 +11,5 @@ checkcompose_subvariant_emails:
   AtomicHost:
     error: ["dusty@dustymabe.com", "walters@verbum.org", "atomic@lists.fedoraproject.org"]
 
-checkcompose_env: production
-checkcompose_env_suffix:
 checkcompose_greenwaveurl: https://greenwave.fedoraproject.org
 checkcompose_url: "https://{{ external_hostname }}"
-
-checkcompose_amqp_cacert: "/etc/fedora-messaging/cacert.pem"

inventory/group_vars/checkcompose_common

@@ -7,10 +7,18 @@ checkcompose_amqp_mailto: ["adamwill@fedoraproject.org", "lruzicka@fedoraproject
 checkcompose_amqp_smtp: bastion
 
 # fedora-messaging compose report sender settings
-#  We currently use the openQA user for convenience
+# we use the openQA rabbitmq account for this role as they run
-checkcompose_amqp_username: "openqa{{ checkcompose_env_suffix }}"
+# on the same server and it's convenient. this means it's also
-checkcompose_amqp_url: "amqps://{{ checkcompose_amqp_username }}:@rabbitmq{{ checkcompose_env_suffix }}.fedoraproject.org/%2Fpubsub"
+# safe and appropriate to use variables defined in the openqa
-checkcompose_amqp_cert: "/etc/pki/fedora-messaging/{{ checkcompose_amqp_username }}-cert.pem"
+# group_vars. if we ever split the roles up we will have to create
-checkcompose_amqp_key: "/etc/pki/fedora-messaging/{{ checkcompose_amqp_username }}-key.pem"
+# a separate account with separate creds
+checkcompose_amqp_username: "{{ openqa_amqp_this_username }}"
+checkcompose_amqp_url: "{{ openqa_amqp_this_url }}"
+checkcompose_amqp_cacert: "{{ openqa_amqp_this_cacertfile }}"
+checkcompose_amqp_cert: "{{ openqa_amqp_this_certfile }}"
+checkcompose_amqp_key: "{{ openqa_amqp_this_keyfile }}"
 checkcompose_amqp_queue: "{{ checkcompose_amqp_username }}_checkcomp"
 checkcompose_amqp_routing_keys: ["org.fedoraproject.{{ deployment_type }}.openqa.job.done"]
+
+checkcompose_env: "{{ openqa_env }}"
+checkcompose_env_suffix: "{{ openqa_env_suffix }}"

inventory/group_vars/checkcompose_stg

@@ -1,6 +1,3 @@
-checkcompose_amqp_cacert: "/etc/fedora-messaging/stg-cacert.pem"
-checkcompose_env: staging
-checkcompose_env_suffix: .stg
 checkcompose_greenwaveurl: https://greenwave-web-greenwave.apps.ocp.stg.fedoraproject.org
 checkcompose_prod: false
 checkcompose_url: "https://{{ external_hostname }}"

inventory/group_vars/fedora_nightlies

@@ -7,13 +7,16 @@ fedora_nightlies_amqp_mailto: ["adamwill@fedoraproject.org"]
 fedora_nightlies_amqp_smtp: bastion
 
 # fedora-messaging job scheduler settings
-# we currently use the openqa user, for convenience
+# we use the openQA rabbitmq account for this role as they run
-fedora_nightlies_amqp_username: "openqa"
+# on the same server and it's convenient. this means it's also
-fedora_nightlies_amqp_url: "amqps://{{ fedora_nightlies_amqp_username }}:@rabbitmq.fedoraproject.org/%2Fpubsub"
+# safe and appropriate to use variables defined in the openqa
-fedora_nightlies_amqp_cacert: "/etc/fedora-messaging/cacert.pem"
+# group_vars. if we ever split the roles up we will have to create
-fedora_nightlies_amqp_cert: "/etc/pki/fedora-messaging/{{ fedora_nightlies_amqp_username }}-cert.pem"
+# a separate account with separate creds
-fedora_nightlies_amqp_key: "/etc/pki/fedora-messaging/{{ fedora_nightlies_amqp_username }}-key.pem"
+fedora_nightlies_amqp_url: "{{ openqa_amqp_prod_url }}"
-fedora_nightlies_amqp_queue: "{{ fedora_nightlies_amqp_username }}_fedora_nightlies"
+fedora_nightlies_amqp_cacert: "{{ openqa_amqp_prod_cacertfile }}"
+fedora_nightlies_amqp_cert: "{{ openqa_amqp_prod_certfile }}"
+fedora_nightlies_amqp_key: "{{ openqa_amqp_prod_keyfile }}"
+fedora_nightlies_amqp_queue: "{{ openqa_amqp_prod_username }}_fedora_nightlies"
 fedora_nightlies_amqp_routing_keys: ["org.fedoraproject.prod.openqa.job.done", "org.fedoraproject.prod.pungi.compose.status.change"]
 fedora_nightlies_amqp_data_file: /usr/share/openqa/public/nightlies.json
 fedora_nightlies_amqp_html_file: /usr/share/openqa/public/nightlies.html

inventory/group_vars/openqa_servers_common

@@ -49,8 +49,8 @@ openqa_amqp_publisher_prefix: org.fedoraproject.{{ deployment_type }}
 openqa_amqp_publisher_url: "{{ openqa_amqp_this_url }}"
 
 # fedora-messaging job scheduler settings: most of these are the same
-# for prod and stg as they both must listen for prod messages. Only
+# for prod and stg as they use the same account to listen for prod
-# the queue names differs.
+# messages on prod rabbitmq. Only the queue names differs.
 openqa_amqp_scheduler_cert: "{{ openqa_amqp_prod_certfile }}"
 openqa_amqp_scheduler_key: "{{ openqa_amqp_prod_keyfile }}"
 openqa_amqp_scheduler_routing_keys: ["org.fedoraproject.prod.pungi.compose.status.change", "org.fedoraproject.prod.bodhi.update.request.testing", "org.fedoraproject.prod.bodhi.update.edit", "org.fedoraproject.prod.bodhi.update.status.testing.koji-build-group.build.complete", "org.fedoraproject.prod.coreos.build.state.change", "org.fedoraproject.prod.odcs.compose.state-changed"]

inventory/group_vars/relvalconsumer

@@ -1,5 +1,10 @@
-relvalamiconsumer_amqp_queue: "openqa_relvalamiconsumer"
+# we use the openQA rabbitmq account for this role as they run
-relvalconsumer_amqp_queue: "openqa_relvalconsumer"
+# on the same server and it's convenient. this means it's also
+# safe and appropriate to use variables defined in the openqa
+# group_vars. if we ever split the roles up we will have to create
+# a separate account with separate creds
+relvalamiconsumer_amqp_queue: "{{ openqa_amqp_this_username }}_relvalamiconsumer"
+relvalconsumer_amqp_queue: "{{ openqa_amqp_this_username }}_relvalconsumer"
 
 # This var should never be set for more than one machine
 relvalconsumer_prod: true

inventory/group_vars/relvalconsumer_common

@@ -8,12 +8,17 @@ relvalconsumer_amqp_smtp: bastion
 
 # fedora-messaging relvalconsumer settings: most of these are the same
 # for prod and stg as they both must listen for prod messages. Only
-# the queue names differs (so is set in the non-common files). We currently
+# the queue names differs (so is set in the non-common files).
-# use the openQA user for convenience
+#
-relvalconsumer_amqp_url: "amqps://openqa:@rabbitmq.fedoraproject.org/%2Fpubsub"
+# we use the openQA rabbitmq account for this role as they run
-relvalconsumer_amqp_cacert: /etc/fedora-messaging/cacert.pem
+# on the same server and it's convenient. this means it's also
-relvalconsumer_amqp_cert: /etc/pki/fedora-messaging/openqa-cert.pem
+# safe and appropriate to use variables defined in the openqa
-relvalconsumer_amqp_key: /etc/pki/fedora-messaging/openqa-key.pem
+# group_vars. if we ever split the roles up we will have to create
+# a separate account with separate creds
+relvalconsumer_amqp_url: "{{ openqa_amqp_prod_url }}"
+relvalconsumer_amqp_cacert: "{{ openqa_amqp_prod_cacertfile }}"
+relvalconsumer_amqp_cert: "{{ openqa_amqp_prod_certfile }}"
+relvalconsumer_amqp_key: "{{ openqa_amqp_prod_keyfile }}"
 relvalconsumer_amqp_routing_keys: ["org.fedoraproject.prod.pungi.compose.status.change"]
 
 relvalamiconsumer_amqp_routing_keys: ["org.fedoraproject.prod.fedimg.image.publish"]

inventory/group_vars/relvalconsumer_test

@@ -1,3 +1,8 @@
-relvalamiconsumer_amqp_queue: "openqa_relvalamiconsumer.test"
+# we use the openQA rabbitmq account for this role as they run
-relvalconsumer_amqp_queue: "openqa_relvalconsumer.test"
+# on the same server and it's convenient. this means it's also
+# safe and appropriate to use variables defined in the openqa
+# group_vars. if we ever split the roles up we will have to create
+# a separate account with separate creds
+relvalamiconsumer_amqp_queue: "{{ openqa_amqp_this_username }}_relvalamiconsumer.test"
+relvalconsumer_amqp_queue: "{{ openqa_amqp_this_username }}_relvalconsumer.test"
 relvalconsumer_prod: false

playbooks/groups/openqa.yml

@@ -43,8 +43,13 @@
    - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
 
   roles:
-  # we must always make sure the prod user exists, as stg
+  # we must always make sure the prod user exists, as stg uses the
-  # must listen on prod in several cases so uses the prod user
+  # prod account to listen to things on prod. **NOTE**: this is lazy
+  # and bad practice, but we've evaluated it and it doesn't really
+  # cause any significant security risk, it just means someone with
+  # root on openqa.stg could access the rabbitmq creds for the prod
+  # user and impersonate prod openqa on the message bus, which is
+  # not a huge deal. fixing it would be kinda tedious.
   - role: rabbit/user
     username: "{{ openqa_amqp_prod_username }}"
     publish_only: false
@@ -78,8 +83,8 @@
     queue_name: "{{ openqa_amqp_resultsdb_reporter_queue }}"
     routing_keys: "{{ openqa_amqp_resultsdb_reporter_routing_keys }}"
     vars:
-      # for this role to work correctly we do need to look like a
+      # this role does listen on stg, as it listens out for jobs
-      # staging server
+      # from openqa stg
       env: "{{ openqa_env }}"
       env_suffix: "{{ openqa_env_suffix }}"
     tags: ['rabbit']
@@ -88,8 +93,8 @@
     queue_name: "{{ openqa_amqp_wiki_reporter_queue }}"
     routing_keys: "{{ openqa_amqp_wiki_reporter_routing_keys }}"
     vars:
-      # for this role to work correctly we do need to look like a
+      # this role does listen on stg, as it listens out for jobs
-      # staging server
+      # from openqa stg
       env: "{{ openqa_env }}"
       env_suffix: "{{ openqa_env_suffix }}"
     tags: ['rabbit']
@@ -98,8 +103,8 @@
   - role: rabbit/queue
     queue_name: "{{ relvalconsumer_amqp_queue }}"
     vars:
-      # yes, even the staging scheduler listens to production, it
+      # yes, even the staging relvalconsumer listens to production,
-      # has to or else it wouldn't schedule any jobs
+      # it has to because composes don't happen in stg
       env: "production"
       env_suffix: ""
     tags: ['rabbit']
@@ -122,8 +127,8 @@
     queue_name: "{{ checkcompose_amqp_queue }}"
     routing_keys: "{{ checkcompose_amqp_routing_keys }}"
     vars:
-      # for this role to work correctly we do need to look like a
+      # this role does listen on stg, as it listens out for jobs
-      # staging server
+      # from openqa stg
       env: "{{ checkcompose_env }}"
       env_suffix: "{{ checkcompose_env_suffix }}"
     tags: ['rabbit']
@@ -134,6 +139,7 @@
     queue_name: "{{ fedora_nightlies_amqp_queue }}"
     routing_keys: "{{ fedora_nightlies_amqp_routing_keys }}"
     vars:
+      # this role listens on prod as it listens for composes
       env: "production"
       env_suffix: ""
     tags: ['rabbit']

roles/check-compose/tasks/main.yml

@@ -148,12 +148,13 @@
   tags:
   - config
 
-# We always use the openQA cert and key here for now; we don't really
+# as noted elsewhere, we are reusing the openqa user and creds for
-# need a separate identity for check-compose.
+# convenience while the roles are all run on the same system, and
+# stg uses the prod account and credentials to listen on prod
 - name: Deploy the Fedora infra fedora-messaging cert
   copy:
-    src: "{{ private }}/files/rabbitmq/{{ checkcompose_env }}/pki/issued/openqa{{ checkcompose_env_suffix }}.crt"
+    src: "{{ private }}/files/rabbitmq/{{ checkcompose_env }}/pki/issued/{{ openqa_amqp_this_username }}.crt"
-    dest: /etc/pki/fedora-messaging/openqa{{ checkcompose_env_suffix }}-cert.pem
+    dest: "/etc/pki/fedora-messaging/{{ openqa_amqp_this_username }}-cert.pem"
     mode: 0644
     owner: root
     group: root
@@ -167,8 +168,8 @@
 # line. This needs making cleaner somehow.
 - name: Deploy the Fedora infra fedora-messaging key
   copy:
-    src: "{{ private }}/files/rabbitmq/{{ checkcompose_env }}/pki/private/openqa{{ checkcompose_env_suffix }}.key"
+    src: "{{ private }}/files/rabbitmq/{{ checkcompose_env }}/pki/private/{{ openqa_amqp_this_username }}.key"
-    dest: /etc/pki/fedora-messaging/openqa{{ checkcompose_env_suffix }}-key.pem
+    dest: "/etc/pki/fedora-messaging/{{ openqa_amqp_this_username }}-key.pem"
     mode: 0640
     owner: root
     group: geekotest

roles/openqa/dispatcher/tasks/main.yml

@@ -277,8 +277,8 @@
 
 - name: Deploy the Fedora infra fedora-messaging cert
   copy:
-    src: "{{ private }}/files/rabbitmq/{{ openqa_env }}/pki/issued/openqa{{ openqa_env_suffix }}.crt"
+    src: "{{ private }}/files/rabbitmq/{{ openqa_env }}/pki/issued/{{ openqa_amqp_this_username }}.crt"
-    dest: /etc/pki/fedora-messaging/openqa{{ openqa_env_suffix }}-cert.pem
+    dest: "/etc/pki/fedora-messaging/{{ openqa_amqp_this_username }}-cert.pem"
     mode: 0644
     owner: root
     group: root
@@ -288,8 +288,8 @@
 
 - name: Deploy the Fedora infra fedora-messaging key
   copy:
-    src: "{{ private }}/files/rabbitmq/{{ openqa_env }}/pki/private/openqa{{ openqa_env_suffix }}.key"
+    src: "{{ private }}/files/rabbitmq/{{ openqa_env }}/pki/private/{{ openqa_amqp_this_username }}.key"
-    dest: /etc/pki/fedora-messaging/openqa{{ openqa_env_suffix }}-key.pem
+    dest: "/etc/pki/fedora-messaging/{{ openqa_amqp_this_username }}-key.pem"
     mode: 0640
     owner: root
     group: geekotest
@@ -297,10 +297,10 @@
   tags:
   - config
 
-- name: Deploy the Fedora infra fedora-messaging prod cert on stg (for scheduler)
+- name: Deploy the Fedora infra fedora-messaging prod cert on stg (for stg consumers that listen on prod)
   copy:
-    src: "{{ private }}/files/rabbitmq/production/pki/issued/openqa.crt"
+    src: "{{ private }}/files/rabbitmq/production/pki/issued/{{ openqa_amqp_prod_username }}.crt"
-    dest: /etc/pki/fedora-messaging/openqa-cert.pem
+    dest: "/etc/pki/fedora-messaging/{{ openqa_amqp_prod_username }}-cert.pem"
     mode: 0644
     owner: root
     group: root
@@ -308,10 +308,10 @@
   tags:
   - config
 
-- name: Deploy the Fedora infra fedora-messaging prod key on stg (for scheduler)
+- name: Deploy the Fedora infra fedora-messaging prod key on stg (for stg consumers that listen on prod)
   copy:
-    src: "{{ private }}/files/rabbitmq/production/pki/private/openqa.key"
+    src: "{{ private }}/files/rabbitmq/production/pki/private/{{ openqa_amqp_prod_username }}.key"
-    dest: /etc/pki/fedora-messaging/openqa-key.pem
+    dest: "/etc/pki/fedora-messaging/{{ openqa_amqp_prod_username }}-key.pem"
     mode: 0640
     owner: root
     group: geekotest

roles/relvalconsumer/tasks/main.yml

@@ -166,14 +166,13 @@
   tags:
   - config
 
-# We always use the openQA production cert and key here for now; we
+# as noted elsewhere, we are reusing the openqa user and creds for
-# don't really need a separate identity for relvalconsumer. We don't
+# convenience while the roles are all run on the same system, and
-# use the staging identity as even staging relvalconsumer listens on
+# stg uses the prod account and credentials to listen on prod
-# the prod bus.
 - name: Deploy the Fedora infra fedora-messaging cert (openQA production)
   copy:
-    src: "{{ private }}/files/rabbitmq/production/pki/issued/openqa.crt"
+    src: "{{ private }}/files/rabbitmq/production/pki/issued/{{ openqa_amqp_prod_username }}.crt"
-    dest: /etc/pki/fedora-messaging/openqa-cert.pem
+    dest: "/etc/pki/fedora-messaging/{{ openqa_amqp_prod_username }}-cert.pem"
     mode: 0644
     owner: root
     group: root
@@ -185,10 +184,10 @@
 # should exist so far as this role is concerned. But as we run this role
 # on the same box as openQA, in fact we need to keep the ownership in
 # line. This needs making cleaner somehow.
-- name: Deploy the Fedora infra fedora-messaging key
+- name: Deploy the Fedora infra fedora-messaging key (openQA production)
   copy:
-    src: "{{ private }}/files/rabbitmq/production/pki/private/openqa.key"
+    src: "{{ private }}/files/rabbitmq/production/pki/private/{{ openqa_amqp_prod_username }}.key"
-    dest: /etc/pki/fedora-messaging/openqa-key.pem
+    dest: "/etc/pki/fedora-messaging/{{ openqa_amqp_prod_username }}-key.pem"
     mode: 0640
     owner: root
     group: geekotest