MM: add a cron job to get the access statistics

Signed-off-by: Aurélien Bompard <aurelien@bompard.org>

playbooks/openshift-apps/mirrormanager.yml

@@ -133,6 +133,10 @@
       app: mirrormanager
       template: cron-propagation.yml
       objectname: cron-propagation.yml
+    - role: openshift/object
+      app: mirrormanager
+      template: cron-accesses.yml
+      objectname: cron-accesses.yml
 
     - role: openshift/start-build
       app: mirrormanager

roles/openshift-apps/mirrormanager/templates/configmap.yml

@@ -32,8 +32,8 @@ items:
     labels:
       app: mirrormanager
   data:
-    sync-crawler-logs.sh: |-
+    create_statistics.sh: |-
-      {{ load_file('sync-crawler-logs.sh') | indent(6) }}
+      {{ load_file('create_statistics.sh') | indent(6) }}
 - apiVersion: v1
   kind: ConfigMap
   metadata:

roles/openshift-apps/mirrormanager/templates/create_statistics.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+MIRRORLIST_PROXIES="{% for host in groups['mirrorlist_proxies'] %} {{ host }} {% endfor %}"
+
+MIRRORLIST_LOGDIR="/var/log/mirrormanager"
+MIRRORLIST_LOGFILES="mirrorlist1.service.log mirrorlist2.service.log"
+SSH_KEY="/etc/mirrormanager-secrets/ssh_mirrorlist_proxies.key"
+REMOTE_USER="mirrormanager"
+
+SSH="ssh -i ${SSH_KEY}"
+DATE=`date +%Y%m%d`
+OUTPUT=`mktemp -d`
+
+trap "rm -f ${OUTPUT}/*; rmdir ${OUTPUT}" QUIT TERM INT HUP EXIT
+
+for proxy in ${MIRRORLIST_PROXIES}; do
+	if [ "$1" == "yesterday" ]; then
+		for logfile in ${MIRRORLIST_LOGFILES}; do
+			${SSH} ${REMOTE_USER}@${proxy} "( xzcat ${MIRRORLIST_LOGDIR}/${logfile}-${DATE}.xz | grep -v 127.0.0.1 | gzip -4 )" >> ${OUTPUT}/mirrorlist.log.gz
+		done
+	fi
+	for logfile in ${MIRRORLIST_LOGFILES}; do
+		${SSH} ${REMOTE_USER}@${proxy} "( cat ${MIRRORLIST_LOGDIR}/${logfile} | grep -v 127.0.0.1 | gzip -4 )" >> ${OUTPUT}/mirrorlist.log.gz
+	done
+done
+
+if [ "$1" == "yesterday" ]; then
+	OPTIONS="-o 1"
+else
+	OPTIONS=""
+fi
+
+mm2_mirrorlist-statistics ${OPTIONS} -l ${OUTPUT}/mirrorlist.log.gz

roles/openshift-apps/mirrormanager/templates/cron-accesses.yml

@@ -0,0 +1,69 @@
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: mirrorlist-statistics
+spec:
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  concurrencyPolicy: Forbid
+  schedule: "4 */2 * * *"
+  startingDeadlineSeconds: 500
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: mirrormanager
+            image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest
+            command: ["bash", "/opt/scripts/create_statistics.sh"]
+            volumeMounts:
+              - name: config
+                mountPath: "/etc/mirrormanager"
+                readOnly: true
+              - name: scripts
+                mountPath: "/opt/scripts"
+                readOnly: true
+          volumes:
+            - name: config
+              configMap:
+                name: config
+            - name: scripts
+              configMap:
+                name: scripts
+---
+## Also include stats from the previous day
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: mirrorlist-statistics-yesterday
+spec:
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  concurrencyPolicy: Forbid
+  schedule: "55 0 * * *"
+  startingDeadlineSeconds: 500
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: mirrormanager
+            image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest
+            command: ["bash", "/opt/scripts/create_statistics.sh", "yesterday"]
+            volumeMounts:
+              - name: config
+                mountPath: "/etc/mirrormanager"
+                readOnly: true
+              - name: scripts
+                mountPath: "/opt/scripts"
+                readOnly: true
+          volumes:
+            - name: config
+              configMap:
+                name: config
+            - name: scripts
+              configMap:
+                name: scripts

roles/openshift-apps/mirrormanager/templates/secrets.yml

@@ -9,3 +9,5 @@ metadata:
 stringData:
   client_secrets.json: |-
     {{ load_file('client_secrets.json') | indent }}
+  ssh_mirrorlist_proxies.key: |-
+    {{ load_file(private + '/files/mirrormanager/id_rsa.pub') | indent }}