From 543672d4540e3d5cdbaa8649294870055fbca03e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Thu, 22 Feb 2024 13:46:27 +0100 Subject: [PATCH] MM2: setup some more cron jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Aurélien Bompard --- playbooks/openshift-apps/mirrormanager.yml | 12 ++- .../mirrormanager/files/cron-crawler.yml | 41 ++++++++ .../mirrormanager/files/cron-propagation.yml | 97 +++++++++++++++++++ .../mirrormanager/files/cron.yml | 47 ++------- .../templates/mirrormanager2.cfg.py | 10 ++ vars/apps/mirrormanager.yml | 19 ++++ 6 files changed, 183 insertions(+), 43 deletions(-) create mode 100644 roles/openshift-apps/mirrormanager/files/cron-crawler.yml create mode 100644 roles/openshift-apps/mirrormanager/files/cron-propagation.yml diff --git a/playbooks/openshift-apps/mirrormanager.yml b/playbooks/openshift-apps/mirrormanager.yml index 831727beb7..9d5ca53cb4 100644 --- a/playbooks/openshift-apps/mirrormanager.yml +++ b/playbooks/openshift-apps/mirrormanager.yml @@ -121,10 +121,18 @@ objectname: deploymentconfig.yml # Cron jobs + # - role: openshift/object + # app: mirrormanager + # file: cron.yml + # objectname: cron.yml - role: openshift/object app: mirrormanager - file: cron.yml - objectname: cron.yml + file: cron-crawler.yml + objectname: cron-crawler.yml + - role: openshift/object + app: mirrormanager + file: cron-propagation.yml + objectname: cron-propagation.yml - role: openshift/start-build app: mirrormanager diff --git a/roles/openshift-apps/mirrormanager/files/cron-crawler.yml b/roles/openshift-apps/mirrormanager/files/cron-crawler.yml new file mode 100644 index 0000000000..61b7f29df9 --- /dev/null +++ b/roles/openshift-apps/mirrormanager/files/cron-crawler.yml @@ -0,0 +1,41 @@ +# +### Crawler cron jobs +# +{% for cronjob in mirrormanager_cron_crawler %} +--- +# Category: {{ cronjob.category }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: crawler-{{ cronjob.category|lower|replace(" ", "-") }} +spec: + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + concurrencyPolicy: Forbid + schedule: "{{ cronjob.schedule }}" + startingDeadlineSeconds: 500 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: mirrormanager + image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest + command: + - "/opt/app-root/bin/mm2_crawler" + - "--category" + - "{{ cronjob.category }}" + - "--threads" + - "{{ cronjob.threads }}" + - "--timeout-minutes" + - "240" + volumeMounts: + - name: config + mountPath: "/etc/mirrormanager" + readOnly: true + volumes: + - name: config + configMap: + name: config +{% endfor %} diff --git a/roles/openshift-apps/mirrormanager/files/cron-propagation.yml b/roles/openshift-apps/mirrormanager/files/cron-propagation.yml new file mode 100644 index 0000000000..dcc568c569 --- /dev/null +++ b/roles/openshift-apps/mirrormanager/files/cron-propagation.yml @@ -0,0 +1,97 @@ +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: propagation-check-current +spec: + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + concurrencyPolicy: Forbid + schedule: "27 */2 * * *" + startingDeadlineSeconds: 500 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: mirrormanager + image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest + command: + - "/opt/app-root/bin/mm2_crawler" + - "--threads" + - "50" + - "propagation" + volumeMounts: + - name: config + mountPath: "/etc/mirrormanager" + readOnly: true + volumes: + - name: config + configMap: + name: config +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: propagation-check-centos-9 +spec: + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + concurrencyPolicy: Forbid + schedule: "44 */2 * * *" + startingDeadlineSeconds: 500 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: mirrormanager + image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest + command: + - "/opt/app-root/bin/mm2_crawler" + - "--threads" + - "50" + - "propagation" + - "--product" + - "CentOS" + - "--version" + - "9-stream" + volumeMounts: + - name: config + mountPath: "/etc/mirrormanager" + readOnly: true + volumes: + - name: config + configMap: + name: config +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: propagation-expire-stats +spec: + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + concurrencyPolicy: Forbid + schedule: "10 2,18 * * *" + startingDeadlineSeconds: 500 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: mirrormanager + image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest + command: + - "/opt/app-root/bin/mm2_expire-stats" + volumeMounts: + - name: config + mountPath: "/etc/mirrormanager" + readOnly: true + volumes: + - name: config + configMap: + name: config diff --git a/roles/openshift-apps/mirrormanager/files/cron.yml b/roles/openshift-apps/mirrormanager/files/cron.yml index 4081f36c71..3d72f0232b 100644 --- a/roles/openshift-apps/mirrormanager/files/cron.yml +++ b/roles/openshift-apps/mirrormanager/files/cron.yml @@ -1,13 +1,16 @@ +# +# Propagation +# --- apiVersion: batch/v1 kind: CronJob metadata: - name: generate-worldmap + name: propagation-check spec: successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 concurrencyPolicy: Forbid - schedule: "50 20 * * *" + schedule: "27 */2 * * *" startingDeadlineSeconds: 500 jobTemplate: spec: @@ -18,45 +21,7 @@ spec: - name: mirrormanager image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest command: - - "/opt/app-root/bin/mm2_generate-worldmap" - - "--output" - - "/var/lib/mirrormanager/statistics/map/" - volumeMounts: - - name: config - mountPath: "/etc/mirrormanager" - readOnly: true - - name: data - mountPath: "/var/lib/mirrormanager" - volumes: - - name: config - configMap: - name: config - - name: data - persistentVolumeClaim: - claimName: data - securityContext: - supplementalGroups: [1001280000] ---- -apiVersion: batch/v1 -kind: CronJob -metadata: - name: get-geoip -spec: - successfulJobsHistoryLimit: 3 - failedJobsHistoryLimit: 3 - concurrencyPolicy: Forbid - schedule: "48 * * * *" - startingDeadlineSeconds: 500 - jobTemplate: - spec: - template: - spec: - restartPolicy: Never - containers: - - name: mirrormanager - image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest - command: - - "curl" + - "mm2_crawler" - "--create-dirs" - "--output-dir" - "/var/lib/mirrormanager/geoip" diff --git a/roles/openshift-apps/mirrormanager/templates/mirrormanager2.cfg.py b/roles/openshift-apps/mirrormanager/templates/mirrormanager2.cfg.py index 75d83b53fc..56de026e57 100644 --- a/roles/openshift-apps/mirrormanager/templates/mirrormanager2.cfg.py +++ b/roles/openshift-apps/mirrormanager/templates/mirrormanager2.cfg.py @@ -79,6 +79,9 @@ SMTP_SERVER = "bastion{{ env_suffix }}.fedoraproject.org" # to locate the statistics files and display them. # STATISTICS_BASE = "/var/lib/mirrormanager/statistics/data" +# How long to keep the access stats, in days +#ACCESS_STATS_KEEP_DAYS = 30 + # Countries which have to be excluded. EMBARGOED_COUNTRIES = ["CU", "IR", "KP", "SD", "SY"] @@ -103,6 +106,10 @@ EMBARGOED_COUNTRIES = ["CU", "IR", "KP", "SD", "SY"] # Where to look for the above mentioned propagation images. # PROPAGATION_BASE = "/var/lib/mirrormanager/statistics/data/propagation" +# How long to keep the propagation stats, in days +#PROPAGATION_KEEP_DAYS = 30 +PROPAGATION_KEEP_DAYS = 14 + # Where the GeoIP database lives GEOIP_BASE = "/var/lib/mirrormanager/geoip" @@ -180,6 +187,9 @@ CRAWLER_SEND_EMAIL = False # the host will be disable automatically (user_active) # CRAWLER_AUTO_DISABLE = 4 +# To get the current versions +BODHI_URL = "https://bodhi{{ env_suffix }}.fedoraproject.org" + UMDL_PREFIX = "/srv/" UMDL_MASTER_DIRECTORIES = [ diff --git a/vars/apps/mirrormanager.yml b/vars/apps/mirrormanager.yml index 3bb97637a3..e43ca118be 100644 --- a/vars/apps/mirrormanager.yml +++ b/vars/apps/mirrormanager.yml @@ -4,3 +4,22 @@ mirrormanager_db_host: "db01{{ env_suffix }}" mirrormanager_oidc_client_id: "mirrormanager" # Move that to an env var to target a specific deployment mm2_checkin: false +mirrormanager_cron_crawler: + - category: "Fedora Linux" + schedule: "0 */12 * * *" + threads: 20 + - category: "Fedora Secondary Arches" + schedule: "0 3,9 * * *" + threads: 10 + - category: "Fedora EPEL" + schedule: "45 */6 * * *" + threads: 20 + - category: "Fedora Archive" + schedule: "0 2 * * *" + threads: 10 + - category: "Fedora Other" + schedule: "0 14 * * *" + threads: 10 + - category: "CentOS" + schedule: "0 5,11 * * *" + threads: 20