MM: add cron jobs to generate the mirrorlist

Signed-off-by: Aurélien Bompard <aurelien@bompard.org>

playbooks/openshift-apps/mirrormanager.yml

@@ -127,6 +127,10 @@
       app: mirrormanager
       template: cron-accesses.yml
       objectname: cron-accesses.yml
+    - role: openshift/object
+      app: mirrormanager
+      template: cron-mirrorlist-cache.yml
+      objectname: cron-mirrorlist-cache.yml
 
     - role: openshift/start-build
       app: mirrormanager

roles/openshift-apps/mirrormanager/files/storage.yml

@@ -2,13 +2,13 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: data
+  name: mirrorlist-cache
 spec:
   accessModes:
     - ReadWriteMany
   resources:
     requests:
-      storage: 10Gi
+      storage: 2Gi
   storageClassName: ocs-storagecluster-cephfs
 # ---
 # apiVersion: v1

roles/openshift-apps/mirrormanager/templates/configmap.yml

@@ -15,6 +15,8 @@ items:
       {{ load_file('mirrormanager2.cfg.py') | indent(6) }}
     gunicorn.conf.py: |-
       {{ load_file('gunicorn.conf.py') | indent(6) }}
+    generate-mirrorlist-cache.cfg: |-
+      {{ load_file('generate-mirrorlist-cache.cfg') | indent(6) }}
 - apiVersion: v1
   kind: ConfigMap
   metadata:
@@ -34,3 +36,5 @@ items:
   data:
     create_statistics.sh: |-
       {{ load_file('create_statistics.sh') | indent(6) }}
+    update-mirrorlist-cache.sh: |-
+      {{ load_file('update-mirrorlist-cache.sh') | indent(6) }}

roles/openshift-apps/mirrormanager/templates/create_statistics.sh

@@ -13,6 +13,7 @@ OUTPUT=`mktemp -d`
 
 trap "rm -f ${OUTPUT}/*; rmdir ${OUTPUT}" QUIT TERM INT HUP EXIT
 
+set -e
 set -x
 
 for proxy in ${MIRRORLIST_PROXIES}; do

roles/openshift-apps/mirrormanager/templates/cron-mirrorlist-cache.yml

@@ -0,0 +1,146 @@
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: update-mirrorlist-cache
+spec:
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  concurrencyPolicy: Forbid
+  schedule: "0,15,30,45 * * * *"
+  startingDeadlineSeconds: 500
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: mirrormanager
+            image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest
+            command: ["bash", "/opt/scripts/update-mirrorlist-cache.sh"]
+            volumeMounts:
+              - name: config
+                mountPath: "/etc/mirrormanager"
+                readOnly: true
+              - name: scripts
+                mountPath: "/opt/scripts"
+                readOnly: true
+              - name: ssh-key
+                mountPath: /etc/mirrormanager-ssh/ssh_mirrorlist_proxies.key
+                subPath: ssh_mirrorlist_proxies.key
+                readOnly: true
+              - name: data
+                mountPath: /data
+          volumes:
+            - name: config
+              configMap:
+                name: config
+            - name: scripts
+              configMap:
+                name: scripts
+            - name: ssh-key
+              secret:
+                secretName: ssh-mirrorlist-proxies-key
+            - name: data
+              persistentVolumeClaim:
+                claimName: mirrorlist-cache
+---
+# Sync netblocks list once a day
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: netblocks-global
+spec:
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  concurrencyPolicy: Forbid
+  schedule: "30 0 * * *"
+  startingDeadlineSeconds: 500
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: mirrormanager
+            image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest
+            command: ["/opt/app-root/bin/mm2_get-netblocks", "global", "-o", "/data/global_netblocks.txt"]
+            volumeMounts:
+              - name: config
+                mountPath: "/etc/mirrormanager"
+                readOnly: true
+              - name: data
+                mountPath: /data
+          volumes:
+            - name: config
+              configMap:
+                name: config
+            - name: data
+              persistentVolumeClaim:
+                claimName: mirrorlist-cache
+---
+# Run the script for Internet2 netblocks very late to
+# make sure the files are all there. The script is
+# looking in date dependent directories and depending
+# on the timezone of the remote server the files appear late.
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: netblocks-internet2
+spec:
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  concurrencyPolicy: Forbid
+  schedule: "0 23 * * *"
+  startingDeadlineSeconds: 500
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: mirrormanager
+            image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest
+            command: ["/opt/app-root/bin/mm2_get-netblocks", "internet2", "-o", "/data/i2_netblocks.txt"]
+            volumeMounts:
+              - name: config
+                mountPath: "/etc/mirrormanager"
+                readOnly: true
+              - name: data
+                mountPath: /data
+          volumes:
+            - name: config
+              configMap:
+                name: config
+            - name: data
+              persistentVolumeClaim:
+                claimName: mirrorlist-cache
+---
+# Amazon EC2 netblocks
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: netblocks-internet2
+spec:
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  concurrencyPolicy: Forbid
+  schedule: "30 1 * * *"
+  startingDeadlineSeconds: 500
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: mirrormanager
+            image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest
+            command: ["/opt/app-root/bin/mm2_update-EC2-netblocks"]
+            volumeMounts:
+              - name: config
+                mountPath: "/etc/mirrormanager"
+                readOnly: true
+          volumes:
+            - name: config
+              configMap:
+                name: config

roles/openshift-apps/mirrormanager/templates/generate-mirrorlist-cache.cfg

@@ -0,0 +1,5 @@
+{% if env == 'staging' %}
+DB_URL="postgresql://{{ mirrormanager_stg_db_user }}:{{ mirrormanager_stg_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_stg_db_name }}"
+{% else %}
+DB_URL="postgresql://{{ mirrormanager_db_user }}:{{ mirrormanager_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_db_name }}"
+{% endif %}

roles/openshift-apps/mirrormanager/templates/update-mirrorlist-cache.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+MIRRORLIST_PROXY="{% for host in groups['mirrorlist_proxies'] %} {{ host }} {% endfor %}"
+MM_ROOT=/opt/app-root/src
+CACHEDIR=/data
+
+set -e
+set -x
+
+rm -rf ${CACHEDIR}/old
+mkdir -p ${CACHEDIR}/old
+cp -ar ${CACHEDIR}/*  ${CACHEDIR}/old/ 2>/dev/null
+
+${MM_ROOT}/bin/generate-mirrorlist-cache -o ${CACHEDIR}/mirrorlist_cache.proto
+
+# Update the files on the proxies
+{% if env == 'production' %}
+for server in ${MIRRORLIST_PROXY}; do
+	rsync -az --delete-delay --delay-updates --delete \
+		${MM_ROOT}/mirrormanager2/crawler/country_continent.csv \
+		${CACHEDIR}/mirrorlist_cache.proto \
+		${CACHEDIR}/*.txt \  # Netblocks
+		${server}:/srv/mirrorlist/data/mirrorlist1/ &
+done
+{% endif %}