poc vor datagrepper in openshift

playbooks/openshift-apps/monitor_dashboard.yml

@@ -35,11 +35,21 @@
     template: dashboard_config.yml
     objectname: dashboard_config.yml
 
+  - role: openshift/object
+    app: monitor-dashboard
+    template: datagrepper_config.yml
+    objectname: datagrepper_config.yml
+
   - role: openshift/object
     app: monitor-dashboard
     file: service.yml
     objectname: service.yml
 
+  - role: openshift/object
+    app: monitor-dashboard
+    file: datagrepper_service.yml
+    objectname: datagrepper_service.yml
+
   - role: openshift/object
     app: monitor-dashboard
     file: route_serviceaccount.yml
@@ -51,6 +61,16 @@
     file: route.yml
     objectname: route.yml
 
+  - role: openshift/object
+    app: monitor-dashboard
+    file: datagrepper_route.yml
+    objectname: datagrepper_route.yml
+
+  - role: openshift/object
+    app: monitor-dashboard
+    template: datagrepper_deploymentconfig.yml
+    objectname: datagrepper_deploymentconfig.yml
+
   - role: openshift/object
     app: monitor-dashboard
     template: deploymentconfig.yml

roles/openshift-apps/monitor-dashboard/files/datagrepper_route.yml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Route
+metadata:
+  name: daragrepper
+  labels:
+    app: daragrepper
+spec:
+  #host: waiverdb.stg.fedoraproject.org
+  port:
+    targetPort: daragrepper
+  to:
+    kind: Service
+    name: daragrepper
+  tls:
+    termination: Edge
+    insecureEdgeTerminationPolicy: Redirect

roles/openshift-apps/monitor-dashboard/files/datagrepper_service.yml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: datagrepper
+  labels:
+    app: datagrepper
+spec:
+  selector:
+    app: datagrepper
+    service: datagrepper
+  ports:
+  - name: datagrepper
+    port: 8080
+    targetPort: 8080

roles/openshift-apps/monitor-dashboard/files/dockerfile-datagrepper

@@ -0,0 +1,15 @@
+FROM fedora:32
+LABEL \
+  name="datagrepper" \
+  vendor="Fedora Infrastructure" \
+  license="MIT"
+RUN dnf -y install fedora-messaging python3 python3-fedmsg python3-gunicorn python3-pip python3-psycopg2 git
+RUN git clone https://github.com/fedora-infra/datagrepper.git /srv/datanommer && \
+    cd /srv/datanommer && \
+    python3 -m pip install -r requirements.txt && \
+    python3 -m pip install . --no-use-pep517 && \
+    mkdir -p /usr/share/datagrepper && \
+    cp /srv/datanommer/apache/datagrepper.wsgi /usr/share/datagrepper/datagrepper.wsgi && \
+    cp /srv/datanommer/fedmsg.d/example-datagrepper.py /etc/fedmsg.d/datagrepper.py
+env DATAGREPPER_CONFIG=/srv/datanommer/apache/datagrepper.cfg
+CMD ["gunicorn", "-b", "0.0.0.0:8080", "-w", "4", "--log-level", "DEBUG", "-t", "180", "datagrepper.app:app"]

roles/openshift-apps/monitor-dashboard/templates/buildconfig.yml

@@ -0,0 +1,25 @@
+{% macro load_file(filename) %}{% include filename %}{%- endmacro -%}
+apiVersion: v1
+items:
+- apiVersion: v1
+  kind: BuildConfig
+  metadata:
+    labels:
+      build: datagrepper
+    name: datagrepper
+  spec:
+    runPolicy: Serial
+    source:
+      dockerfile: |-
+        {{ load_file('dockerfile-base') | indent(8) }}
+      type: Dockerfile
+    strategy:
+      type: Docker
+      dockerStrategy:
+        noCache: false
+    output:
+      to:
+        kind: ImageStreamTag
+        name: datagrepper:latest
+kind: List
+metadata: {}

roles/openshift-apps/monitor-dashboard/templates/datagrepper_configmap.yml

@@ -0,0 +1,49 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: datagrepper
+  labels:
+    app: datagrepper
+data:
+  datagrepper.py: |-
+    # Configuration for the datagrepper webapp.
+    config = {
+        # We don't actually want to run the datanommer consumer on this machine.
+        'datanommer.enabled': False,
+
+        # Note that this is connecting to db02.  That's fine for now, but we want to
+        # move the db for datanommer to a whole other db host in the future.  We
+        # expect the amount of data it generates to grow pretty steadily over time
+        # and we don't want *read* operations on that database to slow down all our
+        # other apps.
+        'datanommer.sqlalchemy.url': 'postgresql://datanommer_ro:{{ datanommer_ro_password }}@db-datanommer01.iad2.fedoraproject.org/datanommer',
+        'fedmenu_url': 'https://apps.fedoraproject.org/fedmenu',
+        'fedmenu_data_url': 'https://apps.fedoraproject.org/js/data.js',
+
+        # Only allow ajax/websockets connections back to our domains.
+        # https://github.com/fedora-infra/datagrepper/pull/192
+        'content_security_policy': 'connect-src https://*.fedoraproject.org wss://*.fedoraproject.org'
+    }
+  daragrepper.cfg: |-
+    from datetime import timedelta
+
+    ### Secret key for the Flask application
+    SECRET_KEY = '{{ datagrepperCookieSecret }}'
+
+    ### Unhappy mako
+    MAKO_OUTPUT_ENCODING='utf-8'
+
+    DATAGREPPER_BASE_URL='https://apps.fedoraproject.org/datagrepper/'
+
+    DATAGREPPER_CACHE_BACKEND='dogpile.cache.memcached'
+
+    DATAGREPPER_CACHE_KWARGS={'arguments': {'url': ['memcached01.phx2.fedoraproject.org:11211']}}
+
+    SQLALCHEMY_DATABASE_URI='postgresql+psycopg2://{{ datagrepper_app_user }}:{{ datagrepper_app_password }}@db01.iad2.fedoraproject.org:5432/datagrepper'
+
+    DATAGREPPER_OPENID_ENDPOINT='id.fedoraproject.org'
+
+    RUNNER_LOCKFILE='/var/run/fedmsg/datagrepper.lock'
+    JOB_OUTPUT_DIR='/var/cache/datagrepper'
+    JOB_EXPIRY=timedelta(days=7)

roles/openshift-apps/monitor-dashboard/templates/datagrepper_deploymentconfig.yml

@@ -0,0 +1,48 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dashboard
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: dashboard
+  template:
+    metadata:
+      labels:
+        name: dashboard
+        app: dashboard
+        service: web
+      name: dashboard
+    spec:
+      containers:
+        - command: ["gunicorn"]
+        - args:
+            - "-b"
+            - "0.0.0.0:8080"
+            - "-w"
+            - "4"
+            - "--log-level"
+            - "DEBUG"
+            - "-t"
+            - "180"
+            - "datagrepper.app:app"
+          image: registry.hub.docker.com/openshift/oauth-proxy:latest
+          name: oauth-proxy
+          ports:
+            - containerPort: 8080
+          env:
+            - name: "DATAGREPPER_CONFIG"
+              value: "/srv/datanommer/apache/datagrepper.cfg"
+          volumeMounts:
+            - mountPath: /etc/fedmsg.d/datagrepper.py
+              subpath: datagrepper.py  
+              name: datagrepper
+            - mountPath: /srv/datanommer/fedmsg.d/daragrepper.cfg
+              subpath: daragrepper.cfg  
+              name: datagrepper  
+      volumes:
+        - configMap:
+            name: datagrepper
+          name: datagrepper