diff --git a/playbooks/openshift-apps/languages.yml b/playbooks/openshift-apps/languages.yml index 4b0bb1f6ac..3e143ff25f 100644 --- a/playbooks/openshift-apps/languages.yml +++ b/playbooks/openshift-apps/languages.yml @@ -68,3 +68,33 @@ tags: - never - delete + + post_tasks: + - name: run initial f.10 import + command: "oc create job stats-10-{{ lookup('pipe','date +%s') }}-init --from=cronjob/stats-10" + tags: + - never + - init + - f10 + + - name: run initial f.20 import + command: "oc create job stats-20-{{ lookup('pipe','date +%s') }}-init --from=cronjob/stats-20" + tags: + - never + - init + - f20 + + - name: run initial f.30 import + command: "oc create job stats-30-{{ lookup('pipe','date +%s') }}-init --from=cronjob/stats-30" + tags: + - never + - init + - f30 + + - name: run initial f.latest import + command: "oc create job stats-latest-{{ lookup('pipe','date +%s') }}-init --from=cronjob/stats-latest" + tags: + - never + - init + - latest + diff --git a/roles/openshift-apps/languages/templates/cron.yml b/roles/openshift-apps/languages/templates/cron.yml index 8d934251d6..54b0aa0765 100644 --- a/roles/openshift-apps/languages/templates/cron.yml +++ b/roles/openshift-apps/languages/templates/cron.yml @@ -1,13 +1,14 @@ {% if env == "production" %} +{% for release in ("10", "20", "30") %} apiVersion: batch/v1beta1 kind: CronJob metadata: - name: stats-33 + name: "stats-{{ release }}" spec: successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 2 concurrencyPolicy: Forbid - schedule: "0 2 * * *" + schedule: "0 2 */16 * *" startingDeadlineSeconds: 500 jobTemplate: spec: @@ -17,79 +18,17 @@ spec: labels: parent: "cron_stats" spec: - volumes: - - name: srpm-data - persistentVolumeClaim: - claimName: "languages-storage{{ '-stg' if env == 'staging' else '' }}" - restartPolicy: Never - initContainers: - - name: clean-tmp - image: "docker-registry.default.svc:5000/languages/builder:latest" - imagePullPolicy: Always - env: - - name: RELEASEVER - value: "33" - - name: TMPDIR - value: /src/results/f$(RELEASEVER)/tmp - command: ["/bin/sh", "-c"] - args: - - > - [ -d ${TMPDIR} ] && rm -Rf ${TMPDIR} ; - mkdir -p ${TMPDIR} - volumeMounts: - - name: srpm-data - readOnly: false - mountPath: /src/results - - name: extract - image: "docker-registry.default.svc:5000/languages/builder:latest" - imagePullPolicy: Always - command: - - ./build.py - - "--results" - - f$(RELEASEVER) - env: - - name: RELEASEVER - value: "33" - - name: DNF_CONF - value: "dnf{{ '_stg' if env == 'staging' else '' }}_$(RELEASEVER).conf" - - name: TMPDIR - value: /src/results/f$(RELEASEVER)/tmp - - name: THREAD_MAX_CONN - value: "3" - - name: THREAD_MAX_EXTR - value: "3" - - name: THREAD_MAX - value: "8" - volumeMounts: - - name: srpm-data - readOnly: false - mountPath: /src/results - containers: - - name: stats - image: "docker-registry.default.svc:5000/languages/builder:latest" - imagePullPolicy: Always - command: ["/bin/sh", "-c"] - args: - - > - ./build_language_list.py --results f$(RELEASEVER) --refresh && - ./build_language_list.py --results f$(RELEASEVER) --analyzealllang && - ./build_tm.py --results f$(RELEASEVER) --compress && - ./build_stats.py --results f$(RELEASEVER) - env: - - name: RELEASEVER - value: "33" - - name: TMPDIR - value: /src/results/f$(RELEASEVER)/tmp - volumeMounts: - - name: srpm-data - readOnly: false - mountPath: /src/results +{% filter indent(width=10, first=True) %} +{% include "cron_template.yml" %} +{% endfilter %} --- +{% endfor %} {% endif %} +{% set release="34" %} apiVersion: batch/v1beta1 kind: CronJob metadata: - name: stats-34 + name: "stats-latest" spec: successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 2 @@ -104,73 +43,9 @@ spec: labels: parent: "cron_stats" spec: - volumes: - - name: srpm-data - persistentVolumeClaim: - claimName: "languages-storage{{ '-stg' if env == 'staging' else '' }}" - restartPolicy: Never - initContainers: - - name: clean-tmp - image: "docker-registry.default.svc:5000/languages/builder:latest" - imagePullPolicy: Always - env: - - name: RELEASEVER - value: "34" - - name: TMPDIR - value: /src/results/f$(RELEASEVER)/tmp - command: ["/bin/sh", "-c"] - args: - - > - [ -d ${TMPDIR} ] && rm -Rf ${TMPDIR} ; - mkdir -p ${TMPDIR} - volumeMounts: - - name: srpm-data - readOnly: false - mountPath: /src/results - - name: extract - image: "docker-registry.default.svc:5000/languages/builder:latest" - imagePullPolicy: Always - command: - - ./build.py - - "--results" - - f$(RELEASEVER) - env: - - name: RELEASEVER - value: "34" - - name: DNF_CONF - value: "dnf{{ '_stg' if env == 'staging' else '' }}_$(RELEASEVER).conf" - - name: TMPDIR - value: /src/results/f$(RELEASEVER)/tmp - - name: THREAD_MAX_CONN - value: "3" - - name: THREAD_MAX_EXTR - value: "3" - - name: THREAD_MAX - value: "8" - volumeMounts: - - name: srpm-data - readOnly: false - mountPath: /src/results - containers: - - name: stats - image: "docker-registry.default.svc:5000/languages/builder:latest" - imagePullPolicy: Always - command: ["/bin/sh", "-c"] - args: - - > - ./build_language_list.py --results f$(RELEASEVER) --refresh && - ./build_language_list.py --results f$(RELEASEVER) --analyzealllang && - ./build_tm.py --results f$(RELEASEVER) --compress && - ./build_stats.py --results f$(RELEASEVER) - env: - - name: RELEASEVER - value: "34" - - name: TMPDIR - value: /src/results/f$(RELEASEVER)/tmp - volumeMounts: - - name: srpm-data - readOnly: false - mountPath: /src/results +{% filter indent(width=10, first=True) %} +{% include "cron_template.yml" %} +{% endfilter %} --- apiVersion: batch/v1beta1 kind: CronJob @@ -202,8 +77,9 @@ spec: command: ["/bin/sh", "-c"] args: - > - ./build_website.py -v --results f33 ; - ./build_website.py -v --results f34 ; + for rel in $(ls -1d results/f*); do + ./build_website.py -v --results $(basename ${rel}) ; + done ; cd website && hugo -D --baseURL https://languages{{ env_suffix }}.fedoraproject.org && cp -R public /src/results/ volumeMounts: diff --git a/roles/openshift-apps/languages/templates/cron_template.yml b/roles/openshift-apps/languages/templates/cron_template.yml new file mode 100644 index 0000000000..a087dbaf4b --- /dev/null +++ b/roles/openshift-apps/languages/templates/cron_template.yml @@ -0,0 +1,68 @@ +volumes: +- name: srpm-data + persistentVolumeClaim: + claimName: "languages-storage{{ '-stg' if env == 'staging' else '' }}" +restartPolicy: Never +initContainers: +- name: clean-tmp + image: "docker-registry.default.svc:5000/languages/builder:latest" + imagePullPolicy: Always + env: + - name: RELEASEVER + value: "{{ release }}" + - name: TMPDIR + value: /src/results/f$(RELEASEVER)/tmp + command: ["/bin/sh", "-c"] + args: + - > + [ -d ${TMPDIR} ] && rm -Rf ${TMPDIR} ; + mkdir -p ${TMPDIR} + volumeMounts: + - name: srpm-data + readOnly: false + mountPath: /src/results +- name: extract + image: "docker-registry.default.svc:5000/languages/builder:latest" + imagePullPolicy: Always + command: + - ./build.py + - "--results" + - f$(RELEASEVER) + env: + - name: RELEASEVER + value: "{{ release }}" + - name: DNF_CONF + value: "dnf{{ '_stg' if env == 'staging' else '' }}_$(RELEASEVER).conf" + - name: TMPDIR + value: /src/results/f$(RELEASEVER)/tmp + - name: THREAD_MAX_CONN + value: "3" + - name: THREAD_MAX_EXTR + value: "3" + - name: THREAD_MAX + value: "8" + volumeMounts: + - name: srpm-data + readOnly: false + mountPath: /src/results +containers: +- name: stats + image: "docker-registry.default.svc:5000/languages/builder:latest" + imagePullPolicy: Always + command: ["/bin/sh", "-c"] + args: + - > + ./build_language_list.py --results f$(RELEASEVER) --refresh && + ./build_language_list.py --results f$(RELEASEVER) --analyzealllang && + ./build_tm.py --results f$(RELEASEVER) --compress && + ./build_stats.py --results f$(RELEASEVER) + env: + - name: RELEASEVER + value: "{{ release }}" + - name: TMPDIR + value: /src/results/f$(RELEASEVER)/tmp + volumeMounts: + - name: srpm-data + readOnly: false + mountPath: /src/results +