diff --git a/playbooks/openshift-apps/cloud-image-uploader.yml b/playbooks/openshift-apps/cloud-image-uploader.yml index e48dae868f..93e3e377ab 100644 --- a/playbooks/openshift-apps/cloud-image-uploader.yml +++ b/playbooks/openshift-apps/cloud-image-uploader.yml @@ -16,11 +16,38 @@ - role: rabbit/queue queue_username: "cloud-image-uploader{{ env_suffix }}" - queue_name: "cloud-image-uploader{{ env_suffix }}" + queue_name: "cloud-image-uploader{{ env_suffix }}-aws" queue_routing_keys: - "org.fedoraproject.*.pungi.compose.status.change" queue_thresholds: - warning: 10 + warning: 25 + critical: 50 + + - role: rabbit/queue + queue_username: "cloud-image-uploader{{ env_suffix }}" + queue_name: "cloud-image-uploader{{ env_suffix }}-azure" + queue_routing_keys: + - "org.fedoraproject.*.pungi.compose.status.change" + queue_thresholds: + warning: 25 + critical: 50 + + - role: rabbit/queue + queue_username: "cloud-image-uploader{{ env_suffix }}" + queue_name: "cloud-image-uploader{{ env_suffix }}-containers" + queue_routing_keys: + - "org.fedoraproject.*.pungi.compose.status.change" + queue_thresholds: + warning: 25 + critical: 50 + + - role: rabbit/queue + queue_username: "cloud-image-uploader{{ env_suffix }}" + queue_name: "cloud-image-uploader{{ env_suffix }}-gcp" + queue_routing_keys: + - "org.fedoraproject.*.pungi.compose.status.change" + queue_thresholds: + warning: 25 critical: 50 - role: openshift/project diff --git a/roles/openshift-apps/cloud-image-uploader/templates/config.toml b/roles/openshift-apps/cloud-image-uploader/templates/config.toml index 0378c53096..63bfe6cb3e 100644 --- a/roles/openshift-apps/cloud-image-uploader/templates/config.toml +++ b/roles/openshift-apps/cloud-image-uploader/templates/config.toml @@ -18,21 +18,22 @@ keyfile = "/etc/pki/rabbitmq/key/cloud-image-uploader.key" certfile = "/etc/pki/rabbitmq/cert/cloud-image-uploader.crt" [client_properties] -app = "Fedora Cloud Image Uploader" +app = "Fedora Cloud Image Uploader ({{ queue_suffix }})" app_url = "https://pagure.io/cloud-image-uploader" app_contacts_email = "cloud@lists.fedoraproject.org" [[bindings]] -queue = "cloud-image-uploader{{ env_suffix }}" +queue = "cloud-image-uploader{{ env_suffix }}-{{ queue_suffix }}" exchange = "amq.topic" routing_keys = ["org.fedoraproject.*.pungi.compose.status.change"] -[queues."cloud-image-uploader{{ env_suffix }}"] +[queues."cloud-image-uploader{{ env_suffix }}-{{ queue_suffix }}"] durable = true auto_delete = false exclusive = false arguments = {} +{% if queue_suffix == "aws" %} [consumer_config.aws] base_region = "us-east-1" ami_volume_dev_name = "/dev/sda1" @@ -68,6 +69,10 @@ s3_bucket_name = "fedora-s3-bucket-fedimg-staging" s3_bucket_name = "fedora-s3-bucket-fedimg" {% endif %} +# End of AWS config +{% endif %} + +{% if queue_suffix == "azure" %} [consumer_config.azure] location = "eastus" {% if env == "staging" %} @@ -105,7 +110,11 @@ storage_account_type = "Standard_ZRS" {% endif %} +# End of Azure config +{% endif %} + +{% if queue_suffix == "containers" %} [consumer_config.container] publish_amqp_messages = true @@ -139,6 +148,10 @@ credential_prefix = "QUAY_IO_" {% endif %} +# End of Containers config +{% endif %} + +{% if queue_suffix == "gcp" %} # Google Cloud Engine [consumer_config.gcp] {% if env == "staging" %} @@ -151,6 +164,9 @@ bucket_name = "fedora-cloud-image-upload" storage_locations = ["us"] publish_amqp_messages = true +# End of Google Cloud config +{% endif %} + [qos] prefetch_size = 0 diff --git a/roles/openshift-apps/cloud-image-uploader/templates/configmap.yml.j2 b/roles/openshift-apps/cloud-image-uploader/templates/configmap.yml.j2 index 06c236f7d7..d95003944c 100644 --- a/roles/openshift-apps/cloud-image-uploader/templates/configmap.yml.j2 +++ b/roles/openshift-apps/cloud-image-uploader/templates/configmap.yml.j2 @@ -11,5 +11,11 @@ items: labels: app: cloud-image-uploader data: - config.toml: |- - {{ lookup('template', 'config.toml') | indent(6) }} + aws-config.toml: |- + {{ lookup('template', 'config.toml', template_vars={"queue_suffix": "aws"}) | indent(6) }} + azure-config.toml: |- + {{ lookup('template', 'config.toml', template_vars={"queue_suffix": "azure"}) | indent(6) }} + container-config.toml: |- + {{ lookup('template', 'config.toml', template_vars={"queue_suffix": "containers"}) | indent(6) }} + gcp-config.toml: |- + {{ lookup('template', 'config.toml', template_vars={"queue_suffix": "gcp"}) | indent(6) }} diff --git a/roles/openshift-apps/cloud-image-uploader/templates/deployment.yml.j2 b/roles/openshift-apps/cloud-image-uploader/templates/deployment.yml.j2 index 8111996487..464e5b0461 100644 --- a/roles/openshift-apps/cloud-image-uploader/templates/deployment.yml.j2 +++ b/roles/openshift-apps/cloud-image-uploader/templates/deployment.yml.j2 @@ -42,7 +42,7 @@ spec: - secret: name: registry-fedoraproject-key containers: - - name: cloud-image-uploader + - name: azure-image-uploader image: image-registry.openshift-image-registry.svc:5000/cloud-image-uploader/cloud-image-uploader:latest imagePullPolicy: Always workingDir: /srv/cloud-uploader/ @@ -67,6 +67,26 @@ spec: secretKeyRef: name: azure-credentials key: subscription_id + - name: FEDORA_MESSAGING_CONF + value: "/etc/fedora-messaging/azure-config.toml" + volumeMounts: + - name: config-volume + mountPath: /etc/fedora-messaging + readOnly: true + - name: fedora-messaging-ca-volume + mountPath: /etc/pki/rabbitmq/ca + readOnly: true + - name: fedora-messaging-key-volume + mountPath: /etc/pki/rabbitmq/key + readOnly: true + - name: fedora-messaging-cert-volume + mountPath: /etc/pki/rabbitmq/cert + readOnly: true + - name: aws-image-uploader + image: image-registry.openshift-image-registry.svc:5000/cloud-image-uploader/cloud-image-uploader:latest + imagePullPolicy: Always + workingDir: /srv/cloud-uploader/ + env: - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -77,6 +97,28 @@ spec: secretKeyRef: name: aws-credentials key: secret_access_key + - name: FEDORA_MESSAGING_CONF + value: "/etc/fedora-messaging/aws-config.toml" + volumeMounts: + - name: config-volume + mountPath: /etc/fedora-messaging + readOnly: true + - name: fedora-messaging-ca-volume + mountPath: /etc/pki/rabbitmq/ca + readOnly: true + - name: fedora-messaging-key-volume + mountPath: /etc/pki/rabbitmq/key + readOnly: true + - name: fedora-messaging-cert-volume + mountPath: /etc/pki/rabbitmq/cert + readOnly: true + - name: container-image-uploader + image: image-registry.openshift-image-registry.svc:5000/cloud-image-uploader/cloud-image-uploader:latest + imagePullPolicy: Always + workingDir: /srv/cloud-uploader/ + env: + - name: FEDORA_MESSAGING_CONF + value: "/etc/fedora-messaging/container-config.toml" - name: FEDORA_REGISTRY_USER valueFrom: secretKeyRef: @@ -89,8 +131,6 @@ spec: key: fedoraproject_registry_password - name: FEDORA_REGISTRY_CERT_DIR value: "/etc/pki/registry-fedoraproject-org/" - - name: GOOGLE_APPLICATION_CREDENTIALS - value: "/etc/pki/google-cloud/google_cloud_image_uploader.json" - name: QUAY_IO_USER valueFrom: secretKeyRef: @@ -117,6 +157,28 @@ spec: - name: registry-fedoraproject mountPath: /etc/pki/registry-fedoraproject-org/ readOnly: true + - name: google-cloud-image-uploader + image: image-registry.openshift-image-registry.svc:5000/cloud-image-uploader/cloud-image-uploader:latest + imagePullPolicy: Always + workingDir: /srv/cloud-uploader/ + env: + - name: FEDORA_MESSAGING_CONF + value: "/etc/fedora-messaging/gcp-config.toml" + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/etc/pki/google-cloud/google_cloud_image_uploader.json" + volumeMounts: + - name: config-volume + mountPath: /etc/fedora-messaging + readOnly: true + - name: fedora-messaging-ca-volume + mountPath: /etc/pki/rabbitmq/ca + readOnly: true + - name: fedora-messaging-key-volume + mountPath: /etc/pki/rabbitmq/key + readOnly: true + - name: fedora-messaging-cert-volume + mountPath: /etc/pki/rabbitmq/cert + readOnly: true - name: google-cloud-key-volume mountPath: /etc/pki/google-cloud/ readOnly: true