From 5a972ede1f980999f9e69609305aa341a0489980 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Thu, 22 Feb 2024 18:55:54 +0100 Subject: [PATCH] MM: add a cron job to get the access statistics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Aurélien Bompard --- playbooks/openshift-apps/mirrormanager.yml | 4 ++ .../mirrormanager/templates/configmap.yml | 4 +- .../templates/create_statistics.sh | 33 +++++++++ .../mirrormanager/templates/cron-accesses.yml | 69 +++++++++++++++++++ .../mirrormanager/templates/secrets.yml | 2 + 5 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 roles/openshift-apps/mirrormanager/templates/create_statistics.sh create mode 100644 roles/openshift-apps/mirrormanager/templates/cron-accesses.yml diff --git a/playbooks/openshift-apps/mirrormanager.yml b/playbooks/openshift-apps/mirrormanager.yml index b3820b621e..e9eea59f85 100644 --- a/playbooks/openshift-apps/mirrormanager.yml +++ b/playbooks/openshift-apps/mirrormanager.yml @@ -133,6 +133,10 @@ app: mirrormanager template: cron-propagation.yml objectname: cron-propagation.yml + - role: openshift/object + app: mirrormanager + template: cron-accesses.yml + objectname: cron-accesses.yml - role: openshift/start-build app: mirrormanager diff --git a/roles/openshift-apps/mirrormanager/templates/configmap.yml b/roles/openshift-apps/mirrormanager/templates/configmap.yml index 8b624c3c16..71b92fbbb9 100644 --- a/roles/openshift-apps/mirrormanager/templates/configmap.yml +++ b/roles/openshift-apps/mirrormanager/templates/configmap.yml @@ -32,8 +32,8 @@ items: labels: app: mirrormanager data: - sync-crawler-logs.sh: |- - {{ load_file('sync-crawler-logs.sh') | indent(6) }} + create_statistics.sh: |- + {{ load_file('create_statistics.sh') | indent(6) }} - apiVersion: v1 kind: ConfigMap metadata: diff --git a/roles/openshift-apps/mirrormanager/templates/create_statistics.sh b/roles/openshift-apps/mirrormanager/templates/create_statistics.sh new file mode 100644 index 0000000000..4850fef44b --- /dev/null +++ b/roles/openshift-apps/mirrormanager/templates/create_statistics.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +MIRRORLIST_PROXIES="{% for host in groups['mirrorlist_proxies'] %} {{ host }} {% endfor %}" + +MIRRORLIST_LOGDIR="/var/log/mirrormanager" +MIRRORLIST_LOGFILES="mirrorlist1.service.log mirrorlist2.service.log" +SSH_KEY="/etc/mirrormanager-secrets/ssh_mirrorlist_proxies.key" +REMOTE_USER="mirrormanager" + +SSH="ssh -i ${SSH_KEY}" +DATE=`date +%Y%m%d` +OUTPUT=`mktemp -d` + +trap "rm -f ${OUTPUT}/*; rmdir ${OUTPUT}" QUIT TERM INT HUP EXIT + +for proxy in ${MIRRORLIST_PROXIES}; do + if [ "$1" == "yesterday" ]; then + for logfile in ${MIRRORLIST_LOGFILES}; do + ${SSH} ${REMOTE_USER}@${proxy} "( xzcat ${MIRRORLIST_LOGDIR}/${logfile}-${DATE}.xz | grep -v 127.0.0.1 | gzip -4 )" >> ${OUTPUT}/mirrorlist.log.gz + done + fi + for logfile in ${MIRRORLIST_LOGFILES}; do + ${SSH} ${REMOTE_USER}@${proxy} "( cat ${MIRRORLIST_LOGDIR}/${logfile} | grep -v 127.0.0.1 | gzip -4 )" >> ${OUTPUT}/mirrorlist.log.gz + done +done + +if [ "$1" == "yesterday" ]; then + OPTIONS="-o 1" +else + OPTIONS="" +fi + +mm2_mirrorlist-statistics ${OPTIONS} -l ${OUTPUT}/mirrorlist.log.gz diff --git a/roles/openshift-apps/mirrormanager/templates/cron-accesses.yml b/roles/openshift-apps/mirrormanager/templates/cron-accesses.yml new file mode 100644 index 0000000000..1d07a648d5 --- /dev/null +++ b/roles/openshift-apps/mirrormanager/templates/cron-accesses.yml @@ -0,0 +1,69 @@ +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: mirrorlist-statistics +spec: + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + concurrencyPolicy: Forbid + schedule: "4 */2 * * *" + startingDeadlineSeconds: 500 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: mirrormanager + image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest + command: ["bash", "/opt/scripts/create_statistics.sh"] + volumeMounts: + - name: config + mountPath: "/etc/mirrormanager" + readOnly: true + - name: scripts + mountPath: "/opt/scripts" + readOnly: true + volumes: + - name: config + configMap: + name: config + - name: scripts + configMap: + name: scripts +--- +## Also include stats from the previous day +apiVersion: batch/v1 +kind: CronJob +metadata: + name: mirrorlist-statistics-yesterday +spec: + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + concurrencyPolicy: Forbid + schedule: "55 0 * * *" + startingDeadlineSeconds: 500 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: mirrormanager + image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest + command: ["bash", "/opt/scripts/create_statistics.sh", "yesterday"] + volumeMounts: + - name: config + mountPath: "/etc/mirrormanager" + readOnly: true + - name: scripts + mountPath: "/opt/scripts" + readOnly: true + volumes: + - name: config + configMap: + name: config + - name: scripts + configMap: + name: scripts diff --git a/roles/openshift-apps/mirrormanager/templates/secrets.yml b/roles/openshift-apps/mirrormanager/templates/secrets.yml index 593510aa7b..f9d93f99db 100644 --- a/roles/openshift-apps/mirrormanager/templates/secrets.yml +++ b/roles/openshift-apps/mirrormanager/templates/secrets.yml @@ -9,3 +9,5 @@ metadata: stringData: client_secrets.json: |- {{ load_file('client_secrets.json') | indent }} + ssh_mirrorlist_proxies.key: |- + {{ load_file(private + '/files/mirrormanager/id_rsa.pub') | indent }}