diff --git a/playbooks/openshift-apps/mirrormanager.yml b/playbooks/openshift-apps/mirrormanager.yml index 9783e93197..ea0f1d507a 100644 --- a/playbooks/openshift-apps/mirrormanager.yml +++ b/playbooks/openshift-apps/mirrormanager.yml @@ -127,6 +127,10 @@ app: mirrormanager template: cron-accesses.yml objectname: cron-accesses.yml + - role: openshift/object + app: mirrormanager + template: cron-mirrorlist-cache.yml + objectname: cron-mirrorlist-cache.yml - role: openshift/start-build app: mirrormanager diff --git a/roles/openshift-apps/mirrormanager/files/storage.yml b/roles/openshift-apps/mirrormanager/files/storage.yml index d19c939fc6..27a1848f59 100644 --- a/roles/openshift-apps/mirrormanager/files/storage.yml +++ b/roles/openshift-apps/mirrormanager/files/storage.yml @@ -2,13 +2,13 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: data + name: mirrorlist-cache spec: accessModes: - ReadWriteMany resources: requests: - storage: 10Gi + storage: 2Gi storageClassName: ocs-storagecluster-cephfs # --- # apiVersion: v1 diff --git a/roles/openshift-apps/mirrormanager/templates/configmap.yml b/roles/openshift-apps/mirrormanager/templates/configmap.yml index 01e2844799..3d3c648473 100644 --- a/roles/openshift-apps/mirrormanager/templates/configmap.yml +++ b/roles/openshift-apps/mirrormanager/templates/configmap.yml @@ -15,6 +15,8 @@ items: {{ load_file('mirrormanager2.cfg.py') | indent(6) }} gunicorn.conf.py: |- {{ load_file('gunicorn.conf.py') | indent(6) }} + generate-mirrorlist-cache.cfg: |- + {{ load_file('generate-mirrorlist-cache.cfg') | indent(6) }} - apiVersion: v1 kind: ConfigMap metadata: @@ -34,3 +36,5 @@ items: data: create_statistics.sh: |- {{ load_file('create_statistics.sh') | indent(6) }} + update-mirrorlist-cache.sh: |- + {{ load_file('update-mirrorlist-cache.sh') | indent(6) }} diff --git a/roles/openshift-apps/mirrormanager/templates/create_statistics.sh b/roles/openshift-apps/mirrormanager/templates/create_statistics.sh index 68238b45c3..3314e89645 100644 --- a/roles/openshift-apps/mirrormanager/templates/create_statistics.sh +++ b/roles/openshift-apps/mirrormanager/templates/create_statistics.sh @@ -13,6 +13,7 @@ OUTPUT=`mktemp -d` trap "rm -f ${OUTPUT}/*; rmdir ${OUTPUT}" QUIT TERM INT HUP EXIT +set -e set -x for proxy in ${MIRRORLIST_PROXIES}; do diff --git a/roles/openshift-apps/mirrormanager/templates/cron-mirrorlist-cache.yml b/roles/openshift-apps/mirrormanager/templates/cron-mirrorlist-cache.yml new file mode 100644 index 0000000000..bfd5b0f553 --- /dev/null +++ b/roles/openshift-apps/mirrormanager/templates/cron-mirrorlist-cache.yml @@ -0,0 +1,146 @@ +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: update-mirrorlist-cache +spec: + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + concurrencyPolicy: Forbid + schedule: "0,15,30,45 * * * *" + startingDeadlineSeconds: 500 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: mirrormanager + image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest + command: ["bash", "/opt/scripts/update-mirrorlist-cache.sh"] + volumeMounts: + - name: config + mountPath: "/etc/mirrormanager" + readOnly: true + - name: scripts + mountPath: "/opt/scripts" + readOnly: true + - name: ssh-key + mountPath: /etc/mirrormanager-ssh/ssh_mirrorlist_proxies.key + subPath: ssh_mirrorlist_proxies.key + readOnly: true + - name: data + mountPath: /data + volumes: + - name: config + configMap: + name: config + - name: scripts + configMap: + name: scripts + - name: ssh-key + secret: + secretName: ssh-mirrorlist-proxies-key + - name: data + persistentVolumeClaim: + claimName: mirrorlist-cache +--- +# Sync netblocks list once a day +apiVersion: batch/v1 +kind: CronJob +metadata: + name: netblocks-global +spec: + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + concurrencyPolicy: Forbid + schedule: "30 0 * * *" + startingDeadlineSeconds: 500 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: mirrormanager + image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest + command: ["/opt/app-root/bin/mm2_get-netblocks", "global", "-o", "/data/global_netblocks.txt"] + volumeMounts: + - name: config + mountPath: "/etc/mirrormanager" + readOnly: true + - name: data + mountPath: /data + volumes: + - name: config + configMap: + name: config + - name: data + persistentVolumeClaim: + claimName: mirrorlist-cache +--- +# Run the script for Internet2 netblocks very late to +# make sure the files are all there. The script is +# looking in date dependent directories and depending +# on the timezone of the remote server the files appear late. +apiVersion: batch/v1 +kind: CronJob +metadata: + name: netblocks-internet2 +spec: + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + concurrencyPolicy: Forbid + schedule: "0 23 * * *" + startingDeadlineSeconds: 500 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: mirrormanager + image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest + command: ["/opt/app-root/bin/mm2_get-netblocks", "internet2", "-o", "/data/i2_netblocks.txt"] + volumeMounts: + - name: config + mountPath: "/etc/mirrormanager" + readOnly: true + - name: data + mountPath: /data + volumes: + - name: config + configMap: + name: config + - name: data + persistentVolumeClaim: + claimName: mirrorlist-cache +--- +# Amazon EC2 netblocks +apiVersion: batch/v1 +kind: CronJob +metadata: + name: netblocks-internet2 +spec: + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + concurrencyPolicy: Forbid + schedule: "30 1 * * *" + startingDeadlineSeconds: 500 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: mirrormanager + image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest + command: ["/opt/app-root/bin/mm2_update-EC2-netblocks"] + volumeMounts: + - name: config + mountPath: "/etc/mirrormanager" + readOnly: true + volumes: + - name: config + configMap: + name: config diff --git a/roles/openshift-apps/mirrormanager/templates/generate-mirrorlist-cache.cfg b/roles/openshift-apps/mirrormanager/templates/generate-mirrorlist-cache.cfg new file mode 100644 index 0000000000..324dcb1d7e --- /dev/null +++ b/roles/openshift-apps/mirrormanager/templates/generate-mirrorlist-cache.cfg @@ -0,0 +1,5 @@ +{% if env == 'staging' %} +DB_URL="postgresql://{{ mirrormanager_stg_db_user }}:{{ mirrormanager_stg_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_stg_db_name }}" +{% else %} +DB_URL="postgresql://{{ mirrormanager_db_user }}:{{ mirrormanager_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_db_name }}" +{% endif %} diff --git a/roles/openshift-apps/mirrormanager/templates/update-mirrorlist-cache.sh b/roles/openshift-apps/mirrormanager/templates/update-mirrorlist-cache.sh new file mode 100644 index 0000000000..9760c192cf --- /dev/null +++ b/roles/openshift-apps/mirrormanager/templates/update-mirrorlist-cache.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +MIRRORLIST_PROXY="{% for host in groups['mirrorlist_proxies'] %} {{ host }} {% endfor %}" +MM_ROOT=/opt/app-root/src +CACHEDIR=/data + +set -e +set -x + +rm -rf ${CACHEDIR}/old +mkdir -p ${CACHEDIR}/old +cp -ar ${CACHEDIR}/* ${CACHEDIR}/old/ 2>/dev/null + +${MM_ROOT}/bin/generate-mirrorlist-cache -o ${CACHEDIR}/mirrorlist_cache.proto + +# Update the files on the proxies +{% if env == 'production' %} +for server in ${MIRRORLIST_PROXY}; do + rsync -az --delete-delay --delay-updates --delete \ + ${MM_ROOT}/mirrormanager2/crawler/country_continent.csv \ + ${CACHEDIR}/mirrorlist_cache.proto \ + ${CACHEDIR}/*.txt \ # Netblocks + ${server}:/srv/mirrorlist/data/mirrorlist1/ & +done +{% endif %}