languages: use templating for similar cronjobs & allow initial import with playbook tags

playbooks/openshift-apps/languages.yml

@@ -68,3 +68,33 @@
     tags:
     - never
     - delete
+
+  post_tasks:
+  - name: run initial f.10 import
+    command: "oc create job stats-10-{{ lookup('pipe','date +%s') }}-init --from=cronjob/stats-10"
+    tags:
+    - never
+    - init
+    - f10
+ 
+  - name: run initial f.20 import
+    command: "oc create job stats-20-{{ lookup('pipe','date +%s') }}-init --from=cronjob/stats-20"
+    tags:
+    - never
+    - init
+    - f20
+
+  - name: run initial f.30 import
+    command: "oc create job stats-30-{{ lookup('pipe','date +%s') }}-init --from=cronjob/stats-30"
+    tags:
+    - never
+    - init
+    - f30
+
+  - name: run initial f.latest import
+    command: "oc create job stats-latest-{{ lookup('pipe','date +%s') }}-init --from=cronjob/stats-latest"
+    tags:
+    - never
+    - init
+    - latest
+

roles/openshift-apps/languages/templates/cron.yml

@@ -1,13 +1,14 @@
 {% if env == "production" %}
+{%   for release in ("10", "20", "30") %}
 apiVersion: batch/v1beta1
 kind: CronJob
 metadata:
-  name: stats-33
+  name: "stats-{{ release }}"
 spec:
   successfulJobsHistoryLimit: 3
   failedJobsHistoryLimit: 2
   concurrencyPolicy: Forbid
-  schedule: "0 2 * * *"
+  schedule: "0 2 */16 * *"
   startingDeadlineSeconds: 500
   jobTemplate:
     spec:
@@ -17,79 +18,17 @@ spec:
           labels:
             parent: "cron_stats"
         spec:
-          volumes:
+{% filter indent(width=10, first=True) %}
-          - name: srpm-data
+{%     include "cron_template.yml" %}
-            persistentVolumeClaim:
+{% endfilter %}
-              claimName: "languages-storage{{ '-stg' if env == 'staging' else '' }}"
-          restartPolicy: Never
-          initContainers:
-          - name: clean-tmp
-            image: "docker-registry.default.svc:5000/languages/builder:latest"
-            imagePullPolicy: Always
-            env:
-            - name: RELEASEVER
-              value: "33"
-            - name: TMPDIR
-              value: /src/results/f$(RELEASEVER)/tmp
-            command: ["/bin/sh", "-c"]
-            args:
-            - >
-              [ -d ${TMPDIR} ] && rm -Rf ${TMPDIR} ;
-              mkdir -p ${TMPDIR}
-            volumeMounts:
-            - name: srpm-data
-              readOnly: false
-              mountPath: /src/results
-          - name: extract
-            image: "docker-registry.default.svc:5000/languages/builder:latest"
-            imagePullPolicy: Always
-            command:
-            - ./build.py
-            - "--results"
-            - f$(RELEASEVER)
-            env:
-            - name: RELEASEVER
-              value: "33"
-            - name: DNF_CONF
-              value: "dnf{{ '_stg' if env == 'staging' else '' }}_$(RELEASEVER).conf"
-            - name: TMPDIR
-              value: /src/results/f$(RELEASEVER)/tmp
-            - name: THREAD_MAX_CONN
-              value: "3"
-            - name: THREAD_MAX_EXTR
-              value: "3"
-            - name: THREAD_MAX
-              value: "8"
-            volumeMounts:
-            - name: srpm-data
-              readOnly: false
-              mountPath: /src/results
-          containers:
-          - name: stats
-            image: "docker-registry.default.svc:5000/languages/builder:latest"
-            imagePullPolicy: Always
-            command: ["/bin/sh", "-c"]
-            args:
-            - >
-              ./build_language_list.py --results f$(RELEASEVER) --refresh &&
-              ./build_language_list.py --results f$(RELEASEVER) --analyzealllang &&
-              ./build_tm.py --results f$(RELEASEVER) --compress &&
-              ./build_stats.py --results f$(RELEASEVER)
-            env:
-            - name: RELEASEVER
-              value: "33"
-            - name: TMPDIR
-              value: /src/results/f$(RELEASEVER)/tmp
-            volumeMounts:
-            - name: srpm-data
-              readOnly: false
-              mountPath: /src/results
 ---
+{%   endfor %}
 {% endif %}
+{% set release="34" %}
 apiVersion: batch/v1beta1
 kind: CronJob
 metadata:
-  name: stats-34
+  name: "stats-latest"
 spec:
   successfulJobsHistoryLimit: 3
   failedJobsHistoryLimit: 2
@@ -104,73 +43,9 @@ spec:
           labels:
             parent: "cron_stats"
         spec:
-          volumes:
+{% filter indent(width=10, first=True) %}
-          - name: srpm-data
+{%     include "cron_template.yml" %}
-            persistentVolumeClaim:
+{% endfilter %}
-              claimName: "languages-storage{{ '-stg' if env == 'staging' else '' }}"
-          restartPolicy: Never
-          initContainers:
-          - name: clean-tmp
-            image: "docker-registry.default.svc:5000/languages/builder:latest"
-            imagePullPolicy: Always
-            env:
-            - name: RELEASEVER
-              value: "34"
-            - name: TMPDIR
-              value: /src/results/f$(RELEASEVER)/tmp
-            command: ["/bin/sh", "-c"]
-            args:
-            - >
-              [ -d ${TMPDIR} ] && rm -Rf ${TMPDIR} ;
-              mkdir -p ${TMPDIR}
-            volumeMounts:
-            - name: srpm-data
-              readOnly: false
-              mountPath: /src/results
-          - name: extract
-            image: "docker-registry.default.svc:5000/languages/builder:latest"
-            imagePullPolicy: Always
-            command:
-            - ./build.py
-            - "--results"
-            - f$(RELEASEVER)
-            env:
-            - name: RELEASEVER
-              value: "34"
-            - name: DNF_CONF
-              value: "dnf{{ '_stg' if env == 'staging' else '' }}_$(RELEASEVER).conf"
-            - name: TMPDIR
-              value: /src/results/f$(RELEASEVER)/tmp
-            - name: THREAD_MAX_CONN
-              value: "3"
-            - name: THREAD_MAX_EXTR
-              value: "3"
-            - name: THREAD_MAX
-              value: "8"
-            volumeMounts:
-            - name: srpm-data
-              readOnly: false
-              mountPath: /src/results
-          containers:
-          - name: stats
-            image: "docker-registry.default.svc:5000/languages/builder:latest"
-            imagePullPolicy: Always
-            command: ["/bin/sh", "-c"]
-            args:
-            - >
-              ./build_language_list.py --results f$(RELEASEVER) --refresh &&
-              ./build_language_list.py --results f$(RELEASEVER) --analyzealllang &&
-              ./build_tm.py --results f$(RELEASEVER) --compress &&
-              ./build_stats.py --results f$(RELEASEVER)
-            env:
-            - name: RELEASEVER
-              value: "34"
-            - name: TMPDIR
-              value: /src/results/f$(RELEASEVER)/tmp
-            volumeMounts:
-            - name: srpm-data
-              readOnly: false
-              mountPath: /src/results
 ---
 apiVersion: batch/v1beta1
 kind: CronJob
@@ -202,8 +77,9 @@ spec:
             command: ["/bin/sh", "-c"]
             args:
             - >
-              ./build_website.py -v --results f33 ;
+              for rel in $(ls -1d results/f*); do 
-              ./build_website.py -v --results f34 ;
+                ./build_website.py -v --results $(basename ${rel}) ;
+              done ;
               cd website && hugo -D --baseURL https://languages{{ env_suffix }}.fedoraproject.org &&
               cp -R public /src/results/
             volumeMounts:

roles/openshift-apps/languages/templates/cron_template.yml

@@ -0,0 +1,68 @@
+volumes:
+- name: srpm-data
+  persistentVolumeClaim:
+    claimName: "languages-storage{{ '-stg' if env == 'staging' else '' }}"
+restartPolicy: Never
+initContainers:
+- name: clean-tmp
+  image: "docker-registry.default.svc:5000/languages/builder:latest"
+  imagePullPolicy: Always
+  env:
+  - name: RELEASEVER
+    value: "{{ release }}"
+  - name: TMPDIR
+    value: /src/results/f$(RELEASEVER)/tmp
+  command: ["/bin/sh", "-c"]
+  args:
+  - >
+    [ -d ${TMPDIR} ] && rm -Rf ${TMPDIR} ;
+    mkdir -p ${TMPDIR}
+  volumeMounts:
+  - name: srpm-data
+    readOnly: false
+    mountPath: /src/results
+- name: extract
+  image: "docker-registry.default.svc:5000/languages/builder:latest"
+  imagePullPolicy: Always
+  command:
+  - ./build.py
+  - "--results"
+  - f$(RELEASEVER)
+  env:
+  - name: RELEASEVER
+    value: "{{ release }}"
+  - name: DNF_CONF
+    value: "dnf{{ '_stg' if env == 'staging' else '' }}_$(RELEASEVER).conf"
+  - name: TMPDIR
+    value: /src/results/f$(RELEASEVER)/tmp
+  - name: THREAD_MAX_CONN
+    value: "3"
+  - name: THREAD_MAX_EXTR
+    value: "3"
+  - name: THREAD_MAX
+    value: "8"
+  volumeMounts:
+  - name: srpm-data
+    readOnly: false
+    mountPath: /src/results
+containers:
+- name: stats
+  image: "docker-registry.default.svc:5000/languages/builder:latest"
+  imagePullPolicy: Always
+  command: ["/bin/sh", "-c"]
+  args:
+  - >
+    ./build_language_list.py --results f$(RELEASEVER) --refresh &&
+    ./build_language_list.py --results f$(RELEASEVER) --analyzealllang &&
+    ./build_tm.py --results f$(RELEASEVER) --compress &&
+    ./build_stats.py --results f$(RELEASEVER)
+  env:
+  - name: RELEASEVER
+    value: "{{ release }}"
+  - name: TMPDIR
+    value: /src/results/f$(RELEASEVER)/tmp
+  volumeMounts:
+  - name: srpm-data
+    readOnly: false
+    mountPath: /src/results
+