MM2: setup some more cron jobs

Signed-off-by: Aurélien Bompard <aurelien@bompard.org>

playbooks/openshift-apps/mirrormanager.yml

@@ -121,10 +121,18 @@
       objectname: deploymentconfig.yml
 
     # Cron jobs
+    # - role: openshift/object
+    #   app: mirrormanager
+    #   file: cron.yml
+    #   objectname: cron.yml
     - role: openshift/object
       app: mirrormanager
-      file: cron.yml
+      file: cron-crawler.yml
-      objectname: cron.yml
+      objectname: cron-crawler.yml
+    - role: openshift/object
+      app: mirrormanager
+      file: cron-propagation.yml
+      objectname: cron-propagation.yml
 
     - role: openshift/start-build
       app: mirrormanager

roles/openshift-apps/mirrormanager/files/cron-crawler.yml

@@ -0,0 +1,41 @@
+#
+### Crawler cron jobs
+#
+{% for cronjob in mirrormanager_cron_crawler %}
+---
+# Category: {{ cronjob.category }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: crawler-{{ cronjob.category|lower|replace(" ", "-") }}
+spec:
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  concurrencyPolicy: Forbid
+  schedule: "{{ cronjob.schedule }}"
+  startingDeadlineSeconds: 500
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: mirrormanager
+            image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest
+            command:
+              - "/opt/app-root/bin/mm2_crawler"
+              - "--category"
+              - "{{ cronjob.category }}"
+              - "--threads"
+              - "{{ cronjob.threads }}"
+              - "--timeout-minutes"
+              - "240"
+            volumeMounts:
+              - name: config
+                mountPath: "/etc/mirrormanager"
+                readOnly: true
+          volumes:
+            - name: config
+              configMap:
+                name: config
+{% endfor %}

roles/openshift-apps/mirrormanager/files/cron-propagation.yml

@@ -0,0 +1,97 @@
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: propagation-check-current
+spec:
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  concurrencyPolicy: Forbid
+  schedule: "27 */2 * * *"
+  startingDeadlineSeconds: 500
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: mirrormanager
+            image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest
+            command:
+              - "/opt/app-root/bin/mm2_crawler"
+              - "--threads"
+              - "50"
+              - "propagation"
+            volumeMounts:
+              - name: config
+                mountPath: "/etc/mirrormanager"
+                readOnly: true
+          volumes:
+            - name: config
+              configMap:
+                name: config
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: propagation-check-centos-9
+spec:
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  concurrencyPolicy: Forbid
+  schedule: "44 */2 * * *"
+  startingDeadlineSeconds: 500
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: mirrormanager
+            image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest
+            command:
+              - "/opt/app-root/bin/mm2_crawler"
+              - "--threads"
+              - "50"
+              - "propagation"
+              - "--product"
+              - "CentOS"
+              - "--version"
+              - "9-stream"
+            volumeMounts:
+              - name: config
+                mountPath: "/etc/mirrormanager"
+                readOnly: true
+          volumes:
+            - name: config
+              configMap:
+                name: config
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: propagation-expire-stats
+spec:
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  concurrencyPolicy: Forbid
+  schedule: "10 2,18 * * *"
+  startingDeadlineSeconds: 500
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: mirrormanager
+            image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest
+            command:
+              - "/opt/app-root/bin/mm2_expire-stats"
+            volumeMounts:
+              - name: config
+                mountPath: "/etc/mirrormanager"
+                readOnly: true
+          volumes:
+            - name: config
+              configMap:
+                name: config

roles/openshift-apps/mirrormanager/files/cron.yml

@@ -1,13 +1,16 @@
+#
+# Propagation
+#
 ---
 apiVersion: batch/v1
 kind: CronJob
 metadata:
-  name: generate-worldmap
+  name: propagation-check
 spec:
   successfulJobsHistoryLimit: 3
   failedJobsHistoryLimit: 3
   concurrencyPolicy: Forbid
-  schedule: "50 20 * * *"
+  schedule: "27 */2 * * *"
   startingDeadlineSeconds: 500
   jobTemplate:
     spec:
@@ -18,45 +21,7 @@ spec:
           - name: mirrormanager
             image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest
             command:
-              - "/opt/app-root/bin/mm2_generate-worldmap"
+              - "mm2_crawler"
-              - "--output"
-              - "/var/lib/mirrormanager/statistics/map/"
-            volumeMounts:
-              - name: config
-                mountPath: "/etc/mirrormanager"
-                readOnly: true
-              - name: data
-                mountPath: "/var/lib/mirrormanager"
-          volumes:
-            - name: config
-              configMap:
-                name: config
-            - name: data
-              persistentVolumeClaim:
-                claimName: data
-          securityContext:
-            supplementalGroups: [1001280000]
----
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: get-geoip
-spec:
-  successfulJobsHistoryLimit: 3
-  failedJobsHistoryLimit: 3
-  concurrencyPolicy: Forbid
-  schedule: "48 * * * *"
-  startingDeadlineSeconds: 500
-  jobTemplate:
-    spec:
-      template:
-        spec:
-          restartPolicy: Never
-          containers:
-          - name: mirrormanager
-            image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest
-            command:
-              - "curl"
               - "--create-dirs"
               - "--output-dir"
               - "/var/lib/mirrormanager/geoip"

roles/openshift-apps/mirrormanager/templates/mirrormanager2.cfg.py

@@ -79,6 +79,9 @@ SMTP_SERVER = "bastion{{ env_suffix }}.fedoraproject.org"
 # to locate the statistics files and display them.
 # STATISTICS_BASE = "/var/lib/mirrormanager/statistics/data"
 
+# How long to keep the access stats, in days
+#ACCESS_STATS_KEEP_DAYS = 30
+
 # Countries which have to be excluded.
 EMBARGOED_COUNTRIES = ["CU", "IR", "KP", "SD", "SY"]
 
@@ -103,6 +106,10 @@ EMBARGOED_COUNTRIES = ["CU", "IR", "KP", "SD", "SY"]
 # Where to look for the above mentioned propagation images.
 # PROPAGATION_BASE = "/var/lib/mirrormanager/statistics/data/propagation"
 
+# How long to keep the propagation stats, in days
+#PROPAGATION_KEEP_DAYS = 30
+PROPAGATION_KEEP_DAYS = 14
+
 # Where the GeoIP database lives
 GEOIP_BASE = "/var/lib/mirrormanager/geoip"
 
@@ -180,6 +187,9 @@ CRAWLER_SEND_EMAIL =  False
 # the host will be disable automatically (user_active)
 # CRAWLER_AUTO_DISABLE = 4
 
+# To get the current versions
+BODHI_URL = "https://bodhi{{ env_suffix }}.fedoraproject.org"
+
 UMDL_PREFIX = "/srv/"
 
 UMDL_MASTER_DIRECTORIES = [

vars/apps/mirrormanager.yml

@@ -4,3 +4,22 @@ mirrormanager_db_host: "db01{{ env_suffix }}"
 mirrormanager_oidc_client_id: "mirrormanager"
 # Move that to an env var to target a specific deployment
 mm2_checkin: false
+mirrormanager_cron_crawler:
+  - category: "Fedora Linux"
+    schedule: "0 */12 * * *"
+    threads: 20
+  - category: "Fedora Secondary Arches"
+    schedule: "0 3,9 * * *"
+    threads: 10
+  - category: "Fedora EPEL"
+    schedule: "45 */6 * * *"
+    threads: 20
+  - category: "Fedora Archive"
+    schedule: "0 2 * * *"
+    threads: 10
+  - category: "Fedora Other"
+    schedule: "0 14 * * *"
+    threads: 10
+  - category: "CentOS"
+    schedule: "0 5,11 * * *"
+    threads: 20