Add the staging openshift config for datagrepper

Signed-off-by: Aurélien Bompard <aurelien@bompard.org>

playbooks/openshift-apps/datagrepper.yml

@@ -0,0 +1,82 @@
+- name: make the app be real
+  hosts: os_masters_stg[0]
+  user: root
+  gather_facts: False
+
+  vars_files:
+    - /srv/web/infra/ansible/vars/global.yml
+    - "/srv/private/ansible/vars.yml"
+    - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+
+  vars:
+
+  roles:
+
+  - role: openshift/project
+    app: datagrepper
+    description: "Query the message history"
+    appowners:
+    - abompard
+    - ryanlerch
+    tags:
+      - apply-appowners
+    when: env == "production"
+  - role: openshift/project
+    app: datagrepper
+    description: "Query the message history"
+    appowners:
+    - abompard
+    - ryanlerch
+    - james02135
+    - lenkaseg
+    - scoady
+    tags:
+      - apply-appowners
+    when: env == "staging"
+
+  - role: openshift/imagestream
+    app: datagrepper
+    imagename: datagrepper
+
+  - role: openshift/object
+    app: datagrepper
+    template: buildconfig.yml
+    objectname: buildconfig.yml
+
+  - role: openshift/object
+    app: datagrepper
+    template: configmap.yml
+    objectname: configmap.yml
+
+  - role: openshift/object
+    app: datagrepper
+    template: service.yml
+    objectname: service.yml
+
+  - role: openshift/route
+    app: datagrepper
+    routename: datagrepper
+    host: "apps{{ env_suffix }}.fedoraproject.org"
+    path: "/datagrepper"
+    serviceport: web
+    servicename: datagrepper-web
+    annotations:
+      haproxy.router.openshift.io/timeout: 5m
+
+  - role: openshift/object
+    app: datagrepper
+    template: secret-webhook.yml
+    objectname: secret-webhook.yml
+
+  - role: openshift/object
+    app: datagrepper
+    template: deploymentconfig.yml
+    objectname: deploymentconfig.yml
+
+  # - role: openshift/start-build
+  #   app: datagrepper
+  #   buildname: datagrepper
+
+  # - role: openshift/rollout
+  #   app: datagrepper
+  #   dcname: datagrepper

roles/openshift-apps/datagrepper/files/imagestream.yml

@@ -0,0 +1,23 @@
+---
+apiVersion: v1
+kind: ImageStream
+metadata:
+  name: datagrepper
+---
+apiVersion: v1
+kind: ImageStream
+metadata:
+  name: ubi8-python-39
+  spec:
+    lookupPolicy:
+      local: false
+    tags:
+    - name: latest
+      from:
+        kind: DockerImage
+        name: registry.access.redhat.com/ubi8/python-39:latest
+      importPolicy:
+        scheduled: true
+      referencePolicy:
+        type: Source
+

roles/openshift-apps/datagrepper/templates/buildconfig.yml

@@ -0,0 +1,37 @@
+apiVersion: build.openshift.io/v1
+kind: BuildConfig
+metadata:
+  name: datagrepper
+  labels:
+    app: Datagrepper
+    build: datagrepper
+spec:
+  runPolicy: Serial
+  source:
+    type: Git
+    git:
+      uri: https://github.com/fedora-infra/datagrepper.git
+{% if env == "staging" %}
+      ref: staging
+{% else %}
+      ref: stable
+{% endif %}
+    contextDir: /
+  strategy:
+    type: Source
+    sourceStrategy:
+      from:
+        kind: ImageStreamTag
+        name: ubi8-python-39:latest
+  output:
+    to:
+      kind: ImageStreamTag
+      name: datagrepper:latest
+  triggers:
+  - type: ConfigChange
+  - type: ImageChange
+{% if datagrepper_stg_github_secret is defined and env == 'staging' %}
+  - type: GitHub
+    github:
+      secret: "{{ datagrepper_stg_github_secret }}"
+{% endif %}

roles/openshift-apps/datagrepper/templates/configmap.yml

@@ -0,0 +1,27 @@
+{% macro load_file(filename) %}{% include filename %}{%- endmacro -%}
+---
+apiVersion: v1
+kind: List
+metadata: {}
+items:
+- apiVersion: v1
+  kind: ConfigMap
+  metadata:
+    name: datagrepper-config
+    labels:
+      app: Datagrepper
+  data:
+    datagrepper.cfg: |-
+      {{ load_file('datagrepper.cfg.py') | indent(6) }}
+    gunicorn.conf.py: |-
+      {{ load_file('gunicorn.conf.py') | indent(6) }}
+- apiVersion: v1
+  kind: ConfigMap
+  metadata:
+    name: wsgi-script
+    labels:
+      app: {{ app }}
+  data:
+    wsgi.py: |-
+      {{ load_file('wsgi.py') | indent(6) }}
+  __init__.py: ""

roles/openshift-apps/datagrepper/templates/datagrepper.cfg.py

@@ -0,0 +1,7 @@
+#
+# This is the config file for Datagrepper as intended to be used in OpenShift
+#
+
+APP_PATH = "https://apps{{ env_suffix }}.fedoraproject.org/datagrepper"
+DEFAULT_QUERY_DELTA = 3600
+DATANOMMER_SQLALCHEMY_URL = "postgresql://{{ datanommerDBUser }}:{{ datanommerDBPassword }}@db-datanommer01{{ env_suffix }}.iad2.fedoraproject.org/datanommer2"

roles/openshift-apps/datagrepper/templates/deploymentconfig.yml

@@ -0,0 +1,83 @@
+apiVersion: apps.openshift.io/v1
+kind: DeploymentConfig
+metadata:
+  name: datagrepper
+  labels:
+    app: Datagrepper
+spec:
+  replicas: 1
+  selector:
+    app: Datagrepper
+    deploymentconfig: datagrepper
+  strategy:
+    type: Rolling
+    activeDeadlineSeconds: 21600
+    rollingParams:
+      intervalSeconds: 1
+      maxSurge: 25%
+      maxUnavailable: 25%
+      timeoutSeconds: 600
+      updatePeriodSeconds: 1
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: Datagrepper
+        deploymentconfig: datagrepper
+    spec:
+      containers:
+      - name: datagrepper
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 8080
+          #protocol: TCP
+        #resources: {}
+        #terminationMessagePath: /dev/termination-log
+        #terminationMessagePolicy: File
+        volumeMounts:
+        - name: datagrepper-config-volume
+          mountPath: "/etc/datagrepper"
+          readOnly: true
+        - name: wsgi-script-volume
+          mountPath: "/opt/app-root/src/deploy"
+          readOnly: true
+        env:
+        - name: DATAGREPPER_CONFIG
+          value: "/etc/datagrepper/datagrepper.cfg"
+        - name: APP_MODULE
+          value: "deploy.wsgi"
+        - name: APP_CONFIG
+          value: "/etc/datagrepper/gunicorn.conf.py"
+        - name: SCRIPT_NAME
+          value: "/datagrepper"
+        # readinessProbe:
+        #   timeoutSeconds: 10
+        #   initialDelaySeconds: 5
+        #   periodSeconds: 60
+        #   httpGet:
+        #     path: "/datagrepper/healthz/ready"
+        #     port: 8080
+        # livenessProbe:
+        #   timeoutSeconds: 10
+        #   initialDelaySeconds: 10
+        #   periodSeconds: 60
+        #   httpGet:
+        #     path: "/datagrepper/healthz/live"
+        #     port: 8080
+      volumes:
+      - name: datagrepper-config-volume
+        configMap:
+          name: datagrepper-config
+      - name: wsgi-script-volume
+        configMap:
+          name: wsgi-script
+  triggers:
+  - imageChangeParams:
+      automatic: true
+      containerNames:
+      - datagrepper
+      from:
+        kind: ImageStreamTag
+        name: datagrepper:latest
+    type: ImageChange
+  - type: ConfigChange

roles/openshift-apps/datagrepper/templates/fedora-messaging.toml

@@ -0,0 +1,37 @@
+amqp_url = "amqps://noggin:@rabbitmq{{ env_suffix }}.fedoraproject.org/%2Fpubsub"
+
+# Just check if the queue exist, don't try to create it (the server does not allow it).
+passive_declares = true
+
+# The topic_prefix configuration value will add a prefix to the topics of every sent message.
+# This is used for migrating from fedmsg, and should not be used afterwards.
+{% if env == "staging" %}
+topic_prefix = "org.fedoraproject.stg"
+{% else %}
+topic_prefix = "org.fedoraproject.prod"
+{% endif %}
+
+[tls]
+ca_cert = "/etc/pki/fedora-messaging/cacert.pem"
+keyfile = "/etc/pki/fedora-messaging/noggin-key.pem"
+certfile = "/etc/pki/fedora-messaging/noggin-cert.pem"
+
+[client_properties]
+app = "Noggin"
+
+[log_config]
+version = 1
+disable_existing_loggers = true
+[log_config.formatters.simple]
+format = "[%(levelname)s %(name)s] %(message)s"
+[log_config.handlers.console]
+class = "logging.StreamHandler"
+formatter = "simple"
+stream = "ext://sys.stdout"
+[log_config.loggers.fedora_messaging]
+level = "INFO"
+propagate = false
+handlers = ["console"]
+[log_config.root]
+level = "WARNING"
+handlers = ["console"]

roles/openshift-apps/datagrepper/templates/gunicorn.conf.py

@@ -0,0 +1 @@
+timeout = 60

roles/openshift-apps/datagrepper/templates/secret-webhook.yml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: datagrepper-github-webhook-secret
+data:
+  WebHookSecretKey: "{{ datagrepper_stg_github_secret }}"
+type: Opaque

roles/openshift-apps/datagrepper/templates/service.yml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: datagrepper-web
+  labels:
+    app: Datagrepper
+spec:
+  ports:
+    - name: web
+      port: 8080
+      targetPort: 8080
+  selector:
+    app: Datagrepper
+    deploymentconfig: datagrepper

roles/openshift-apps/datagrepper/templates/wsgi.py

@@ -0,0 +1,3 @@
+from werkzeug.middleware.proxy_fix import ProxyFix
+from datagrepper.app import app as application
+application.wsgi_app = ProxyFix(application.wsgi_app, x_proto=1, x_host=1)

roles/openshift-apps/datanommer/templates/configmap.yml

@@ -5,7 +5,7 @@ kind: ConfigMap
 metadata:
   name: fedora-messaging-config
   labels:
-    app: datanommer
+    app: Datanommer
 data:
   config.toml: |-
     {{ load_file('fedora-messaging-config.toml') | indent(6) }}