From 0571feb2ce16584038b938eacc143654cc1f4896 Mon Sep 17 00:00:00 2001 From: Jeremy Cline Date: Mon, 28 Apr 2025 15:09:05 -0400 Subject: [PATCH] fedora-image-uploader: deploy as multiple containers In the beginning, this just handled Azure images. Now it does Azure, AWS, GCP, and containers. Currently, it processes images serially, which is mostly okay. However, it does mean that whatever service is handled last has to wait for all the others to succeed before it starts, and it also means if any of the handlers for their respective platform fail, it retries *all* the images again. For most things this is a no-op (or a few inexpensive calls), but it does have to re-download the image from Koji to checksum it. This adds an AMQP message queue for each content type we handle, and produces a fedora-messaging config for each content type. The deployment is now made up of 4 containers: azure-image-uploader, aws-image-uploader, container-image-uploader, and google-cloud-image-uploader. They only differ in the secrets injected into them and the fedora-messaging config file they use. The end result is that images should be available faster and its more resilient to remote services being down. Finally, it's worth noting that this bumps the warning threshold for queue sizes. It can take some services (Azure and AWS) upwards of 30 minutes to replicate the images around the world, and since we subscribe to _any_ compose status changes, it's not unreasonable for 5-10 messages to stack up when we hit a compose change that is "FINISHED" with images. Signed-off-by: Jeremy Cline --- .../openshift-apps/cloud-image-uploader.yml | 31 ++++++++- .../templates/config.toml | 22 +++++- .../templates/configmap.yml.j2 | 10 ++- .../templates/deployment.yml.j2 | 68 ++++++++++++++++++- 4 files changed, 121 insertions(+), 10 deletions(-) diff --git a/playbooks/openshift-apps/cloud-image-uploader.yml b/playbooks/openshift-apps/cloud-image-uploader.yml index e48dae868f..93e3e377ab 100644 --- a/playbooks/openshift-apps/cloud-image-uploader.yml +++ b/playbooks/openshift-apps/cloud-image-uploader.yml @@ -16,11 +16,38 @@ - role: rabbit/queue queue_username: "cloud-image-uploader{{ env_suffix }}" - queue_name: "cloud-image-uploader{{ env_suffix }}" + queue_name: "cloud-image-uploader{{ env_suffix }}-aws" queue_routing_keys: - "org.fedoraproject.*.pungi.compose.status.change" queue_thresholds: - warning: 10 + warning: 25 + critical: 50 + + - role: rabbit/queue + queue_username: "cloud-image-uploader{{ env_suffix }}" + queue_name: "cloud-image-uploader{{ env_suffix }}-azure" + queue_routing_keys: + - "org.fedoraproject.*.pungi.compose.status.change" + queue_thresholds: + warning: 25 + critical: 50 + + - role: rabbit/queue + queue_username: "cloud-image-uploader{{ env_suffix }}" + queue_name: "cloud-image-uploader{{ env_suffix }}-containers" + queue_routing_keys: + - "org.fedoraproject.*.pungi.compose.status.change" + queue_thresholds: + warning: 25 + critical: 50 + + - role: rabbit/queue + queue_username: "cloud-image-uploader{{ env_suffix }}" + queue_name: "cloud-image-uploader{{ env_suffix }}-gcp" + queue_routing_keys: + - "org.fedoraproject.*.pungi.compose.status.change" + queue_thresholds: + warning: 25 critical: 50 - role: openshift/project diff --git a/roles/openshift-apps/cloud-image-uploader/templates/config.toml b/roles/openshift-apps/cloud-image-uploader/templates/config.toml index 0378c53096..63bfe6cb3e 100644 --- a/roles/openshift-apps/cloud-image-uploader/templates/config.toml +++ b/roles/openshift-apps/cloud-image-uploader/templates/config.toml @@ -18,21 +18,22 @@ keyfile = "/etc/pki/rabbitmq/key/cloud-image-uploader.key" certfile = "/etc/pki/rabbitmq/cert/cloud-image-uploader.crt" [client_properties] -app = "Fedora Cloud Image Uploader" +app = "Fedora Cloud Image Uploader ({{ queue_suffix }})" app_url = "https://pagure.io/cloud-image-uploader" app_contacts_email = "cloud@lists.fedoraproject.org" [[bindings]] -queue = "cloud-image-uploader{{ env_suffix }}" +queue = "cloud-image-uploader{{ env_suffix }}-{{ queue_suffix }}" exchange = "amq.topic" routing_keys = ["org.fedoraproject.*.pungi.compose.status.change"] -[queues."cloud-image-uploader{{ env_suffix }}"] +[queues."cloud-image-uploader{{ env_suffix }}-{{ queue_suffix }}"] durable = true auto_delete = false exclusive = false arguments = {} +{% if queue_suffix == "aws" %} [consumer_config.aws] base_region = "us-east-1" ami_volume_dev_name = "/dev/sda1" @@ -68,6 +69,10 @@ s3_bucket_name = "fedora-s3-bucket-fedimg-staging" s3_bucket_name = "fedora-s3-bucket-fedimg" {% endif %} +# End of AWS config +{% endif %} + +{% if queue_suffix == "azure" %} [consumer_config.azure] location = "eastus" {% if env == "staging" %} @@ -105,7 +110,11 @@ storage_account_type = "Standard_ZRS" {% endif %} +# End of Azure config +{% endif %} + +{% if queue_suffix == "containers" %} [consumer_config.container] publish_amqp_messages = true @@ -139,6 +148,10 @@ credential_prefix = "QUAY_IO_" {% endif %} +# End of Containers config +{% endif %} + +{% if queue_suffix == "gcp" %} # Google Cloud Engine [consumer_config.gcp] {% if env == "staging" %} @@ -151,6 +164,9 @@ bucket_name = "fedora-cloud-image-upload" storage_locations = ["us"] publish_amqp_messages = true +# End of Google Cloud config +{% endif %} + [qos] prefetch_size = 0 diff --git a/roles/openshift-apps/cloud-image-uploader/templates/configmap.yml.j2 b/roles/openshift-apps/cloud-image-uploader/templates/configmap.yml.j2 index 06c236f7d7..d95003944c 100644 --- a/roles/openshift-apps/cloud-image-uploader/templates/configmap.yml.j2 +++ b/roles/openshift-apps/cloud-image-uploader/templates/configmap.yml.j2 @@ -11,5 +11,11 @@ items: labels: app: cloud-image-uploader data: - config.toml: |- - {{ lookup('template', 'config.toml') | indent(6) }} + aws-config.toml: |- + {{ lookup('template', 'config.toml', template_vars={"queue_suffix": "aws"}) | indent(6) }} + azure-config.toml: |- + {{ lookup('template', 'config.toml', template_vars={"queue_suffix": "azure"}) | indent(6) }} + container-config.toml: |- + {{ lookup('template', 'config.toml', template_vars={"queue_suffix": "containers"}) | indent(6) }} + gcp-config.toml: |- + {{ lookup('template', 'config.toml', template_vars={"queue_suffix": "gcp"}) | indent(6) }} diff --git a/roles/openshift-apps/cloud-image-uploader/templates/deployment.yml.j2 b/roles/openshift-apps/cloud-image-uploader/templates/deployment.yml.j2 index 8111996487..464e5b0461 100644 --- a/roles/openshift-apps/cloud-image-uploader/templates/deployment.yml.j2 +++ b/roles/openshift-apps/cloud-image-uploader/templates/deployment.yml.j2 @@ -42,7 +42,7 @@ spec: - secret: name: registry-fedoraproject-key containers: - - name: cloud-image-uploader + - name: azure-image-uploader image: image-registry.openshift-image-registry.svc:5000/cloud-image-uploader/cloud-image-uploader:latest imagePullPolicy: Always workingDir: /srv/cloud-uploader/ @@ -67,6 +67,26 @@ spec: secretKeyRef: name: azure-credentials key: subscription_id + - name: FEDORA_MESSAGING_CONF + value: "/etc/fedora-messaging/azure-config.toml" + volumeMounts: + - name: config-volume + mountPath: /etc/fedora-messaging + readOnly: true + - name: fedora-messaging-ca-volume + mountPath: /etc/pki/rabbitmq/ca + readOnly: true + - name: fedora-messaging-key-volume + mountPath: /etc/pki/rabbitmq/key + readOnly: true + - name: fedora-messaging-cert-volume + mountPath: /etc/pki/rabbitmq/cert + readOnly: true + - name: aws-image-uploader + image: image-registry.openshift-image-registry.svc:5000/cloud-image-uploader/cloud-image-uploader:latest + imagePullPolicy: Always + workingDir: /srv/cloud-uploader/ + env: - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -77,6 +97,28 @@ spec: secretKeyRef: name: aws-credentials key: secret_access_key + - name: FEDORA_MESSAGING_CONF + value: "/etc/fedora-messaging/aws-config.toml" + volumeMounts: + - name: config-volume + mountPath: /etc/fedora-messaging + readOnly: true + - name: fedora-messaging-ca-volume + mountPath: /etc/pki/rabbitmq/ca + readOnly: true + - name: fedora-messaging-key-volume + mountPath: /etc/pki/rabbitmq/key + readOnly: true + - name: fedora-messaging-cert-volume + mountPath: /etc/pki/rabbitmq/cert + readOnly: true + - name: container-image-uploader + image: image-registry.openshift-image-registry.svc:5000/cloud-image-uploader/cloud-image-uploader:latest + imagePullPolicy: Always + workingDir: /srv/cloud-uploader/ + env: + - name: FEDORA_MESSAGING_CONF + value: "/etc/fedora-messaging/container-config.toml" - name: FEDORA_REGISTRY_USER valueFrom: secretKeyRef: @@ -89,8 +131,6 @@ spec: key: fedoraproject_registry_password - name: FEDORA_REGISTRY_CERT_DIR value: "/etc/pki/registry-fedoraproject-org/" - - name: GOOGLE_APPLICATION_CREDENTIALS - value: "/etc/pki/google-cloud/google_cloud_image_uploader.json" - name: QUAY_IO_USER valueFrom: secretKeyRef: @@ -117,6 +157,28 @@ spec: - name: registry-fedoraproject mountPath: /etc/pki/registry-fedoraproject-org/ readOnly: true + - name: google-cloud-image-uploader + image: image-registry.openshift-image-registry.svc:5000/cloud-image-uploader/cloud-image-uploader:latest + imagePullPolicy: Always + workingDir: /srv/cloud-uploader/ + env: + - name: FEDORA_MESSAGING_CONF + value: "/etc/fedora-messaging/gcp-config.toml" + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/etc/pki/google-cloud/google_cloud_image_uploader.json" + volumeMounts: + - name: config-volume + mountPath: /etc/fedora-messaging + readOnly: true + - name: fedora-messaging-ca-volume + mountPath: /etc/pki/rabbitmq/ca + readOnly: true + - name: fedora-messaging-key-volume + mountPath: /etc/pki/rabbitmq/key + readOnly: true + - name: fedora-messaging-cert-volume + mountPath: /etc/pki/rabbitmq/cert + readOnly: true - name: google-cloud-key-volume mountPath: /etc/pki/google-cloud/ readOnly: true