languages: initial deployment on staging

playbooks/include/proxies-reverseproxy.yml

@@ -782,3 +782,13 @@
     keephost: true
     tags: packager-dashboard
 
+  - role: httpd/reverseproxy
+    website: "languages{{ env_suffix }}.fedoraproject.org"
+    destname: os
+    balancer_name: os
+    targettype: openshift
+    balancer_members: "{{ openshift_masters }}"
+    keephost: true
+    when: env == "staging"
+    tags:
+    - languages

playbooks/include/proxies-websites.yml

@@ -925,6 +925,14 @@
     - lists.pagure.io
     when: env == "production"
 
+  - role: httpd/website
+    site_name: languages.fedoraproject.org
+    server_aliases: [languages.stg.fedoraproject.org]
+    sslonly: true
+    cert_name: "{{wildcard_cert_name}}"
+    when: env == "staging"
+    tags: languages
+
   - role: httpd/website
     site_name: fpdc.fedoraproject.org
     sslonly: true

playbooks/openshift-apps/languages.yml

@@ -0,0 +1,73 @@
+- name: make the app be real
+  hosts: os_masters_stg[0]
+  user: root
+  gather_facts: False
+  
+  vars_files:
+    - /srv/web/infra/ansible/vars/global.yml
+    - "/srv/private/ansible/vars.yml"
+    - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+
+  roles:
+  - role: openshift/project
+    app: languages
+    description: Localization measurement and tooling
+    appowners:
+    - darknao
+    - jibecfed
+
+  - role: openshift/imagestream
+    app: languages
+    imagename: builder
+
+  - role: openshift/object
+    app: languages
+    objectname: buildconfig.yml
+    template: buildconfig.yml
+
+  - role: openshift/start-build
+    app: languages
+    buildname: build-latest
+
+  - role: openshift/start-build
+    app: languages
+    buildname: build-f33
+
+  - role: openshift/object
+    app: languages
+    template: service.yml
+    objectname: service.yml
+
+  - role: openshift/route
+    app: languages
+    routename: languages
+    host: "languages{{ env_suffix }}.fedoraproject.org"
+    serviceport: web
+    servicename: web
+
+  - role: openshift/object
+    app: languages
+    template: pvc.yml
+    objectname: pvc.yml
+
+  - role: openshift/object
+    app: languages
+    template: deploymentconfig.yml
+    objectname: deploymentconfig.yml
+
+  - role: openshift/object
+    app: languages
+    template: cron.yml
+    objectname: cron.yml
+
+###############################################
+# actions to delete the project from OpenShift
+###############################################
+# to run: sudo rbac-playbook -l os_masters_stg[0] -t delete openshift-apps/languages.yml
+  - role: openshift/object-delete
+    app: languages
+    objecttype: project
+    objectname: languages
+    tags:
+    - never
+    - delete

roles/openshift-apps/languages/defaults/main.yml

@@ -0,0 +1 @@
+registry: "docker-registry.default.svc:5000"

roles/openshift-apps/languages/templates/buildconfig.yml

@@ -0,0 +1,45 @@
+apiVersion: v1
+kind: BuildConfig
+metadata:
+  labels:
+    build: build-f33
+  name: build-f33
+spec:
+  successfulBuildsHistoryLimit: 2
+  failedBuildsHistoryLimit: 1
+  runPolicy: Serial
+  source:
+    git:
+      uri: "https://pagure.io/fedora-l10n/fedora-localization-statistics.git"
+      ref: main
+  strategy:
+    type: Docker
+    dockerStrategy:
+      dockerfilePath: docker/Dockerfile.33
+  output:
+    to:
+      kind: ImageStreamTag
+      name: builder:f33
+---
+apiVersion: v1
+kind: BuildConfig
+metadata:
+  labels:
+    build: build-latest
+  name: build-latest
+spec:
+  successfulBuildsHistoryLimit: 2
+  failedBuildsHistoryLimit: 1
+  runPolicy: Serial
+  source:
+    git:
+      uri: "https://pagure.io/fedora-l10n/fedora-localization-statistics.git"
+      ref: main
+  strategy:
+    type: Docker
+    dockerStrategy:
+      dockerfilePath: docker/Dockerfile.34
+  output:
+    to:
+      kind: ImageStreamTag
+      name: builder:latest

roles/openshift-apps/languages/templates/cron.yml

@@ -0,0 +1,229 @@
+{% if env == "production" %}
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: extract-33
+spec:
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 2
+  concurrencyPolicy: Forbid
+  schedule: "0 4 * * *"
+  startingDeadlineSeconds: 500
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      template:
+        metadata:
+          labels:
+            parent: "cron_extract"
+        spec:
+          containers:
+          - name: extract
+            image: "{{ registry }}/{{ app }}/builder:f33"
+            imagePullPolicy: Always
+            command:
+            - ./build.py
+            - "--results"
+            - f33
+            volumeMounts:
+            - name: srpm-data
+              readOnly: false
+              mountPath: /src/results
+          volumes:
+          - name: srpm-data
+            persistentVolumeClaim:
+              claimName: "{{ app }}-storage"
+          restartPolicy: Never
+---
+{% endif %}
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: extract-34
+spec:
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 2
+  concurrencyPolicy: Forbid
+  schedule: "0 4 * * *"
+  startingDeadlineSeconds: 500
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      template:
+        metadata:
+          labels:
+            parent: "cron_extract"
+        spec:
+          containers:
+          - name: extract
+            image: "{{ registry }}/{{ app }}/builder:latest"
+            imagePullPolicy: Always
+            command:
+            - ./build.py
+            - "--results"
+            - f34
+            volumeMounts:
+            - name: srpm-data
+              readOnly: false
+              mountPath: /src/results
+          volumes:
+          - name: srpm-data
+            persistentVolumeClaim:
+              claimName: "{{ app }}-storage"
+          restartPolicy: Never
+---
+{% if env == "production" %}
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: stats-33
+spec:
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 2
+  concurrencyPolicy: Forbid
+  schedule: "0 20 * * *"
+  startingDeadlineSeconds: 500
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      template:
+        metadata:
+          labels:
+            parent: "cron_stats"
+        spec:
+          volumes:
+          - name: srpm-data
+            persistentVolumeClaim:
+              claimName: "{{ app }}-storage"
+          restartPolicy: Never
+          containers:
+          - name: stats
+            image: "{{ registry }}/{{ app }}/builder:latest"
+            imagePullPolicy: Always
+            command: ["/bin/sh", "-c"]
+            args:
+            - >
+              ./build_language_list.py --results f33 --refresh &&
+              ./build_language_list.py --results f33 --analyzealllang &&
+              ./build_tm.py --results f33 --compress &&
+              ./build_stats.py --results f33
+            volumeMounts:
+            - name: srpm-data
+              readOnly: false
+              mountPath: /src/results
+---
+{% endif %}
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: stats-34
+spec:
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 2
+  concurrencyPolicy: Forbid
+  schedule: "0 10 * * *"
+  startingDeadlineSeconds: 500
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      template:
+        metadata:
+          labels:
+            parent: "cron_stats"
+        spec:
+          volumes:
+          - name: srpm-data
+            persistentVolumeClaim:
+              claimName: "{{ app }}-storage"
+          restartPolicy: Never
+          containers:
+          - name: stats
+            image: "{{ registry }}/{{ app }}/builder:latest"
+            imagePullPolicy: Always
+            command: ["/bin/sh", "-c"]
+            args:
+            - >
+              ./build_language_list.py --results f34 --refresh &&
+              ./build_language_list.py --results f34 --analyzealllang &&
+              ./build_tm.py --results f34 --compress &&
+              ./build_stats.py --results f34
+            volumeMounts:
+            - name: srpm-data
+              readOnly: false
+              mountPath: /src/results
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: build-website
+spec:
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 2
+  concurrencyPolicy: Forbid
+  schedule: "0 */2 * * *"
+  startingDeadlineSeconds: 500
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      template:
+        metadata:
+          labels:
+            parent: "cron_stats"
+        spec:
+          volumes:
+          - name: srpm-data
+            persistentVolumeClaim:
+              claimName: "{{ app }}-storage"
+          restartPolicy: Never
+          containers:
+          - name: stats
+            image: "{{ registry }}/{{ app }}/builder:latest"
+            imagePullPolicy: Always
+            command: ["/bin/sh", "-c"]
+            args:
+            - >
+              ./build_website.py -v --results f33 ;
+              ./build_website.py -v --results f34 ;
+              cd website && hugo -D --baseURL https://languages{{ env_suffix }}.fedoraproject.org &&
+              cp -R public /src/results/
+            volumeMounts:
+            - name: srpm-data
+              readOnly: false
+              mountPath: /src/results
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: stats-website
+spec:
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 2
+  concurrencyPolicy: Forbid
+  schedule: "0 * * * *"
+  startingDeadlineSeconds: 500
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      template:
+        metadata:
+          labels:
+            parent: "cron_stats"
+        spec:
+          restartPolicy: Never
+          containers:
+          - image: allinurl/goaccess
+            name: goaccess
+            imagePullPolicy: IfNotPresent
+            args:
+            - "-a"
+            - "--log-format=COMBINED"
+            - "--log-file=/html/logs/access.log"
+            - "--output=/html/public/stats.html"
+            ports:
+            volumeMounts:
+            - name: nginx-data
+              mountPath: /html
+          volumes:
+          - name: nginx-data
+            persistentVolumeClaim:
+              claimName: "{{ app }}-storage"

roles/openshift-apps/languages/templates/deploymentconfig.yml

@@ -0,0 +1,39 @@
+apiVersion: v1
+kind: DeploymentConfig
+metadata:
+  name: "web"
+  labels:
+    app: "web"
+spec:
+  replicas: 1
+  selector:
+    app: "web"
+  template:
+    metadata:
+      labels:
+        app: "web"
+    spec:
+      containers:
+      - image: registry.redhat.io/rhel8/nginx-116
+        name: nginx
+        imagePullPolicy: IfNotPresent
+        args:
+        - nginx
+        - "-g"
+        - "daemon off;"
+        ports:
+        - containerPort: 8080
+          protocol: TCP
+        volumeMounts:
+        - name: nginx-data
+          mountPath: /opt/app-root/src
+          subPath: public
+        - name: nginx-data
+          subPath: logs
+          mountPath: /var/log/nginx
+      volumes:
+      - name: nginx-data
+        persistentVolumeClaim:
+          claimName: "{{ app }}-storage"
+      restartPolicy: Always
+

roles/openshift-apps/languages/templates/pvc.yml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: "{{ app }}-storage"
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+{%    if env == 'staging' %}
+      storage: 40Gi
+{%    else %}
+      storage: 70Gi
+{%    endif %}
+  storageClassName: ""

roles/openshift-apps/languages/templates/service.yml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: web
+  labels:
+    app: web
+spec:
+  selector:
+    app: web
+  ports:
+  - name: web
+    port: 8080
+    targetPort: 8080