diff --git a/playbooks/openshift-apps/monitor_dashboard.yml b/playbooks/openshift-apps/monitor_dashboard.yml index b6f6dee208..ae3546aa87 100644 --- a/playbooks/openshift-apps/monitor_dashboard.yml +++ b/playbooks/openshift-apps/monitor_dashboard.yml @@ -35,11 +35,21 @@ template: dashboard_config.yml objectname: dashboard_config.yml + - role: openshift/object + app: monitor-dashboard + template: datagrepper_config.yml + objectname: datagrepper_config.yml + - role: openshift/object app: monitor-dashboard file: service.yml objectname: service.yml + - role: openshift/object + app: monitor-dashboard + file: datagrepper_service.yml + objectname: datagrepper_service.yml + - role: openshift/object app: monitor-dashboard file: route_serviceaccount.yml @@ -51,6 +61,16 @@ file: route.yml objectname: route.yml + - role: openshift/object + app: monitor-dashboard + file: datagrepper_route.yml + objectname: datagrepper_route.yml + + - role: openshift/object + app: monitor-dashboard + template: datagrepper_deploymentconfig.yml + objectname: datagrepper_deploymentconfig.yml + - role: openshift/object app: monitor-dashboard template: deploymentconfig.yml diff --git a/roles/openshift-apps/monitor-dashboard/files/datagrepper_route.yml b/roles/openshift-apps/monitor-dashboard/files/datagrepper_route.yml new file mode 100644 index 0000000000..06d89b7b4c --- /dev/null +++ b/roles/openshift-apps/monitor-dashboard/files/datagrepper_route.yml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Route +metadata: + name: daragrepper + labels: + app: daragrepper +spec: + #host: waiverdb.stg.fedoraproject.org + port: + targetPort: daragrepper + to: + kind: Service + name: daragrepper + tls: + termination: Edge + insecureEdgeTerminationPolicy: Redirect diff --git a/roles/openshift-apps/monitor-dashboard/files/datagrepper_service.yml b/roles/openshift-apps/monitor-dashboard/files/datagrepper_service.yml new file mode 100644 index 0000000000..2232926b5f --- /dev/null +++ b/roles/openshift-apps/monitor-dashboard/files/datagrepper_service.yml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: datagrepper + labels: + app: datagrepper +spec: + selector: + app: datagrepper + service: datagrepper + ports: + - name: datagrepper + port: 8080 + targetPort: 8080 diff --git a/roles/openshift-apps/monitor-dashboard/files/dockerfile-datagrepper b/roles/openshift-apps/monitor-dashboard/files/dockerfile-datagrepper new file mode 100644 index 0000000000..7598b61357 --- /dev/null +++ b/roles/openshift-apps/monitor-dashboard/files/dockerfile-datagrepper @@ -0,0 +1,15 @@ +FROM fedora:32 +LABEL \ + name="datagrepper" \ + vendor="Fedora Infrastructure" \ + license="MIT" +RUN dnf -y install fedora-messaging python3 python3-fedmsg python3-gunicorn python3-pip python3-psycopg2 git +RUN git clone https://github.com/fedora-infra/datagrepper.git /srv/datanommer && \ + cd /srv/datanommer && \ + python3 -m pip install -r requirements.txt && \ + python3 -m pip install . --no-use-pep517 && \ + mkdir -p /usr/share/datagrepper && \ + cp /srv/datanommer/apache/datagrepper.wsgi /usr/share/datagrepper/datagrepper.wsgi && \ + cp /srv/datanommer/fedmsg.d/example-datagrepper.py /etc/fedmsg.d/datagrepper.py +env DATAGREPPER_CONFIG=/srv/datanommer/apache/datagrepper.cfg +CMD ["gunicorn", "-b", "0.0.0.0:8080", "-w", "4", "--log-level", "DEBUG", "-t", "180", "datagrepper.app:app"] diff --git a/roles/openshift-apps/monitor-dashboard/templates/buildconfig.yml b/roles/openshift-apps/monitor-dashboard/templates/buildconfig.yml new file mode 100644 index 0000000000..9ce4b9498d --- /dev/null +++ b/roles/openshift-apps/monitor-dashboard/templates/buildconfig.yml @@ -0,0 +1,25 @@ +{% macro load_file(filename) %}{% include filename %}{%- endmacro -%} +apiVersion: v1 +items: +- apiVersion: v1 + kind: BuildConfig + metadata: + labels: + build: datagrepper + name: datagrepper + spec: + runPolicy: Serial + source: + dockerfile: |- + {{ load_file('dockerfile-base') | indent(8) }} + type: Dockerfile + strategy: + type: Docker + dockerStrategy: + noCache: false + output: + to: + kind: ImageStreamTag + name: datagrepper:latest +kind: List +metadata: {} diff --git a/roles/openshift-apps/monitor-dashboard/templates/datagrepper_configmap.yml b/roles/openshift-apps/monitor-dashboard/templates/datagrepper_configmap.yml new file mode 100644 index 0000000000..a5cfac49f6 --- /dev/null +++ b/roles/openshift-apps/monitor-dashboard/templates/datagrepper_configmap.yml @@ -0,0 +1,49 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: datagrepper + labels: + app: datagrepper +data: + datagrepper.py: |- + # Configuration for the datagrepper webapp. + config = { + # We don't actually want to run the datanommer consumer on this machine. + 'datanommer.enabled': False, + + # Note that this is connecting to db02. That's fine for now, but we want to + # move the db for datanommer to a whole other db host in the future. We + # expect the amount of data it generates to grow pretty steadily over time + # and we don't want *read* operations on that database to slow down all our + # other apps. + 'datanommer.sqlalchemy.url': 'postgresql://datanommer_ro:{{ datanommer_ro_password }}@db-datanommer01.iad2.fedoraproject.org/datanommer', + 'fedmenu_url': 'https://apps.fedoraproject.org/fedmenu', + 'fedmenu_data_url': 'https://apps.fedoraproject.org/js/data.js', + + # Only allow ajax/websockets connections back to our domains. + # https://github.com/fedora-infra/datagrepper/pull/192 + 'content_security_policy': 'connect-src https://*.fedoraproject.org wss://*.fedoraproject.org' + } + daragrepper.cfg: |- + from datetime import timedelta + + ### Secret key for the Flask application + SECRET_KEY = '{{ datagrepperCookieSecret }}' + + ### Unhappy mako + MAKO_OUTPUT_ENCODING='utf-8' + + DATAGREPPER_BASE_URL='https://apps.fedoraproject.org/datagrepper/' + + DATAGREPPER_CACHE_BACKEND='dogpile.cache.memcached' + + DATAGREPPER_CACHE_KWARGS={'arguments': {'url': ['memcached01.phx2.fedoraproject.org:11211']}} + + SQLALCHEMY_DATABASE_URI='postgresql+psycopg2://{{ datagrepper_app_user }}:{{ datagrepper_app_password }}@db01.iad2.fedoraproject.org:5432/datagrepper' + + DATAGREPPER_OPENID_ENDPOINT='id.fedoraproject.org' + + RUNNER_LOCKFILE='/var/run/fedmsg/datagrepper.lock' + JOB_OUTPUT_DIR='/var/cache/datagrepper' + JOB_EXPIRY=timedelta(days=7) diff --git a/roles/openshift-apps/monitor-dashboard/templates/datagrepper_deploymentconfig.yml b/roles/openshift-apps/monitor-dashboard/templates/datagrepper_deploymentconfig.yml new file mode 100644 index 0000000000..258f296ae8 --- /dev/null +++ b/roles/openshift-apps/monitor-dashboard/templates/datagrepper_deploymentconfig.yml @@ -0,0 +1,48 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dashboard +spec: + replicas: 1 + selector: + matchLabels: + name: dashboard + template: + metadata: + labels: + name: dashboard + app: dashboard + service: web + name: dashboard + spec: + containers: + - command: ["gunicorn"] + - args: + - "-b" + - "0.0.0.0:8080" + - "-w" + - "4" + - "--log-level" + - "DEBUG" + - "-t" + - "180" + - "datagrepper.app:app" + image: registry.hub.docker.com/openshift/oauth-proxy:latest + name: oauth-proxy + ports: + - containerPort: 8080 + env: + - name: "DATAGREPPER_CONFIG" + value: "/srv/datanommer/apache/datagrepper.cfg" + volumeMounts: + - mountPath: /etc/fedmsg.d/datagrepper.py + subpath: datagrepper.py + name: datagrepper + - mountPath: /srv/datanommer/fedmsg.d/daragrepper.cfg + subpath: daragrepper.cfg + name: datagrepper + volumes: + - configMap: + name: datagrepper + name: datagrepper