From 13ebc4e684e964ff90400b2ddf4efe708ef4cdb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Thu, 12 Aug 2021 16:28:17 +0200 Subject: [PATCH] Add the staging openshift config for datagrepper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Aurélien Bompard --- playbooks/openshift-apps/datagrepper.yml | 82 ++++++++++++++++++ .../datagrepper/files/imagestream.yml | 23 +++++ .../datagrepper/templates/buildconfig.yml | 37 +++++++++ .../datagrepper/templates/configmap.yml | 27 ++++++ .../datagrepper/templates/datagrepper.cfg.py | 7 ++ .../templates/deploymentconfig.yml | 83 +++++++++++++++++++ .../templates/fedora-messaging.toml | 37 +++++++++ .../datagrepper/templates/gunicorn.conf.py | 1 + .../datagrepper/templates/secret-webhook.yml | 7 ++ .../datagrepper/templates/service.yml | 14 ++++ .../datagrepper/templates/wsgi.py | 3 + .../datanommer/templates/configmap.yml | 2 +- 12 files changed, 322 insertions(+), 1 deletion(-) create mode 100644 playbooks/openshift-apps/datagrepper.yml create mode 100644 roles/openshift-apps/datagrepper/files/imagestream.yml create mode 100644 roles/openshift-apps/datagrepper/templates/buildconfig.yml create mode 100644 roles/openshift-apps/datagrepper/templates/configmap.yml create mode 100644 roles/openshift-apps/datagrepper/templates/datagrepper.cfg.py create mode 100644 roles/openshift-apps/datagrepper/templates/deploymentconfig.yml create mode 100644 roles/openshift-apps/datagrepper/templates/fedora-messaging.toml create mode 100644 roles/openshift-apps/datagrepper/templates/gunicorn.conf.py create mode 100644 roles/openshift-apps/datagrepper/templates/secret-webhook.yml create mode 100644 roles/openshift-apps/datagrepper/templates/service.yml create mode 100644 roles/openshift-apps/datagrepper/templates/wsgi.py diff --git a/playbooks/openshift-apps/datagrepper.yml b/playbooks/openshift-apps/datagrepper.yml new file mode 100644 index 0000000000..34c769e455 --- /dev/null +++ b/playbooks/openshift-apps/datagrepper.yml @@ -0,0 +1,82 @@ +- name: make the app be real + hosts: os_masters_stg[0] + user: root + gather_facts: False + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + vars: + + roles: + + - role: openshift/project + app: datagrepper + description: "Query the message history" + appowners: + - abompard + - ryanlerch + tags: + - apply-appowners + when: env == "production" + - role: openshift/project + app: datagrepper + description: "Query the message history" + appowners: + - abompard + - ryanlerch + - james02135 + - lenkaseg + - scoady + tags: + - apply-appowners + when: env == "staging" + + - role: openshift/imagestream + app: datagrepper + imagename: datagrepper + + - role: openshift/object + app: datagrepper + template: buildconfig.yml + objectname: buildconfig.yml + + - role: openshift/object + app: datagrepper + template: configmap.yml + objectname: configmap.yml + + - role: openshift/object + app: datagrepper + template: service.yml + objectname: service.yml + + - role: openshift/route + app: datagrepper + routename: datagrepper + host: "apps{{ env_suffix }}.fedoraproject.org" + path: "/datagrepper" + serviceport: web + servicename: datagrepper-web + annotations: + haproxy.router.openshift.io/timeout: 5m + + - role: openshift/object + app: datagrepper + template: secret-webhook.yml + objectname: secret-webhook.yml + + - role: openshift/object + app: datagrepper + template: deploymentconfig.yml + objectname: deploymentconfig.yml + + # - role: openshift/start-build + # app: datagrepper + # buildname: datagrepper + + # - role: openshift/rollout + # app: datagrepper + # dcname: datagrepper diff --git a/roles/openshift-apps/datagrepper/files/imagestream.yml b/roles/openshift-apps/datagrepper/files/imagestream.yml new file mode 100644 index 0000000000..9134d2e1bf --- /dev/null +++ b/roles/openshift-apps/datagrepper/files/imagestream.yml @@ -0,0 +1,23 @@ +--- +apiVersion: v1 +kind: ImageStream +metadata: + name: datagrepper +--- +apiVersion: v1 +kind: ImageStream +metadata: + name: ubi8-python-39 + spec: + lookupPolicy: + local: false + tags: + - name: latest + from: + kind: DockerImage + name: registry.access.redhat.com/ubi8/python-39:latest + importPolicy: + scheduled: true + referencePolicy: + type: Source + diff --git a/roles/openshift-apps/datagrepper/templates/buildconfig.yml b/roles/openshift-apps/datagrepper/templates/buildconfig.yml new file mode 100644 index 0000000000..07256ffa70 --- /dev/null +++ b/roles/openshift-apps/datagrepper/templates/buildconfig.yml @@ -0,0 +1,37 @@ +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: datagrepper + labels: + app: Datagrepper + build: datagrepper +spec: + runPolicy: Serial + source: + type: Git + git: + uri: https://github.com/fedora-infra/datagrepper.git +{% if env == "staging" %} + ref: staging +{% else %} + ref: stable +{% endif %} + contextDir: / + strategy: + type: Source + sourceStrategy: + from: + kind: ImageStreamTag + name: ubi8-python-39:latest + output: + to: + kind: ImageStreamTag + name: datagrepper:latest + triggers: + - type: ConfigChange + - type: ImageChange +{% if datagrepper_stg_github_secret is defined and env == 'staging' %} + - type: GitHub + github: + secret: "{{ datagrepper_stg_github_secret }}" +{% endif %} diff --git a/roles/openshift-apps/datagrepper/templates/configmap.yml b/roles/openshift-apps/datagrepper/templates/configmap.yml new file mode 100644 index 0000000000..1e378d444f --- /dev/null +++ b/roles/openshift-apps/datagrepper/templates/configmap.yml @@ -0,0 +1,27 @@ +{% macro load_file(filename) %}{% include filename %}{%- endmacro -%} +--- +apiVersion: v1 +kind: List +metadata: {} +items: +- apiVersion: v1 + kind: ConfigMap + metadata: + name: datagrepper-config + labels: + app: Datagrepper + data: + datagrepper.cfg: |- + {{ load_file('datagrepper.cfg.py') | indent(6) }} + gunicorn.conf.py: |- + {{ load_file('gunicorn.conf.py') | indent(6) }} +- apiVersion: v1 + kind: ConfigMap + metadata: + name: wsgi-script + labels: + app: {{ app }} + data: + wsgi.py: |- + {{ load_file('wsgi.py') | indent(6) }} + __init__.py: "" diff --git a/roles/openshift-apps/datagrepper/templates/datagrepper.cfg.py b/roles/openshift-apps/datagrepper/templates/datagrepper.cfg.py new file mode 100644 index 0000000000..9f287b43a8 --- /dev/null +++ b/roles/openshift-apps/datagrepper/templates/datagrepper.cfg.py @@ -0,0 +1,7 @@ +# +# This is the config file for Datagrepper as intended to be used in OpenShift +# + +APP_PATH = "https://apps{{ env_suffix }}.fedoraproject.org/datagrepper" +DEFAULT_QUERY_DELTA = 3600 +DATANOMMER_SQLALCHEMY_URL = "postgresql://{{ datanommerDBUser }}:{{ datanommerDBPassword }}@db-datanommer01{{ env_suffix }}.iad2.fedoraproject.org/datanommer2" diff --git a/roles/openshift-apps/datagrepper/templates/deploymentconfig.yml b/roles/openshift-apps/datagrepper/templates/deploymentconfig.yml new file mode 100644 index 0000000000..6ae06b8b8a --- /dev/null +++ b/roles/openshift-apps/datagrepper/templates/deploymentconfig.yml @@ -0,0 +1,83 @@ +apiVersion: apps.openshift.io/v1 +kind: DeploymentConfig +metadata: + name: datagrepper + labels: + app: Datagrepper +spec: + replicas: 1 + selector: + app: Datagrepper + deploymentconfig: datagrepper + strategy: + type: Rolling + activeDeadlineSeconds: 21600 + rollingParams: + intervalSeconds: 1 + maxSurge: 25% + maxUnavailable: 25% + timeoutSeconds: 600 + updatePeriodSeconds: 1 + template: + metadata: + creationTimestamp: null + labels: + app: Datagrepper + deploymentconfig: datagrepper + spec: + containers: + - name: datagrepper + imagePullPolicy: Always + ports: + - containerPort: 8080 + #protocol: TCP + #resources: {} + #terminationMessagePath: /dev/termination-log + #terminationMessagePolicy: File + volumeMounts: + - name: datagrepper-config-volume + mountPath: "/etc/datagrepper" + readOnly: true + - name: wsgi-script-volume + mountPath: "/opt/app-root/src/deploy" + readOnly: true + env: + - name: DATAGREPPER_CONFIG + value: "/etc/datagrepper/datagrepper.cfg" + - name: APP_MODULE + value: "deploy.wsgi" + - name: APP_CONFIG + value: "/etc/datagrepper/gunicorn.conf.py" + - name: SCRIPT_NAME + value: "/datagrepper" + # readinessProbe: + # timeoutSeconds: 10 + # initialDelaySeconds: 5 + # periodSeconds: 60 + # httpGet: + # path: "/datagrepper/healthz/ready" + # port: 8080 + # livenessProbe: + # timeoutSeconds: 10 + # initialDelaySeconds: 10 + # periodSeconds: 60 + # httpGet: + # path: "/datagrepper/healthz/live" + # port: 8080 + volumes: + - name: datagrepper-config-volume + configMap: + name: datagrepper-config + - name: wsgi-script-volume + configMap: + name: wsgi-script + triggers: + - imageChangeParams: + automatic: true + containerNames: + - datagrepper + from: + kind: ImageStreamTag + name: datagrepper:latest + type: ImageChange + - type: ConfigChange diff --git a/roles/openshift-apps/datagrepper/templates/fedora-messaging.toml b/roles/openshift-apps/datagrepper/templates/fedora-messaging.toml new file mode 100644 index 0000000000..e1c9313743 --- /dev/null +++ b/roles/openshift-apps/datagrepper/templates/fedora-messaging.toml @@ -0,0 +1,37 @@ +amqp_url = "amqps://noggin:@rabbitmq{{ env_suffix }}.fedoraproject.org/%2Fpubsub" + +# Just check if the queue exist, don't try to create it (the server does not allow it). +passive_declares = true + +# The topic_prefix configuration value will add a prefix to the topics of every sent message. +# This is used for migrating from fedmsg, and should not be used afterwards. +{% if env == "staging" %} +topic_prefix = "org.fedoraproject.stg" +{% else %} +topic_prefix = "org.fedoraproject.prod" +{% endif %} + +[tls] +ca_cert = "/etc/pki/fedora-messaging/cacert.pem" +keyfile = "/etc/pki/fedora-messaging/noggin-key.pem" +certfile = "/etc/pki/fedora-messaging/noggin-cert.pem" + +[client_properties] +app = "Noggin" + +[log_config] +version = 1 +disable_existing_loggers = true +[log_config.formatters.simple] +format = "[%(levelname)s %(name)s] %(message)s" +[log_config.handlers.console] +class = "logging.StreamHandler" +formatter = "simple" +stream = "ext://sys.stdout" +[log_config.loggers.fedora_messaging] +level = "INFO" +propagate = false +handlers = ["console"] +[log_config.root] +level = "WARNING" +handlers = ["console"] diff --git a/roles/openshift-apps/datagrepper/templates/gunicorn.conf.py b/roles/openshift-apps/datagrepper/templates/gunicorn.conf.py new file mode 100644 index 0000000000..368dcad2ea --- /dev/null +++ b/roles/openshift-apps/datagrepper/templates/gunicorn.conf.py @@ -0,0 +1 @@ +timeout = 60 diff --git a/roles/openshift-apps/datagrepper/templates/secret-webhook.yml b/roles/openshift-apps/datagrepper/templates/secret-webhook.yml new file mode 100644 index 0000000000..c7a1a9a179 --- /dev/null +++ b/roles/openshift-apps/datagrepper/templates/secret-webhook.yml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: datagrepper-github-webhook-secret +data: + WebHookSecretKey: "{{ datagrepper_stg_github_secret }}" +type: Opaque diff --git a/roles/openshift-apps/datagrepper/templates/service.yml b/roles/openshift-apps/datagrepper/templates/service.yml new file mode 100644 index 0000000000..ec550447cd --- /dev/null +++ b/roles/openshift-apps/datagrepper/templates/service.yml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: datagrepper-web + labels: + app: Datagrepper +spec: + ports: + - name: web + port: 8080 + targetPort: 8080 + selector: + app: Datagrepper + deploymentconfig: datagrepper diff --git a/roles/openshift-apps/datagrepper/templates/wsgi.py b/roles/openshift-apps/datagrepper/templates/wsgi.py new file mode 100644 index 0000000000..ffd87e1534 --- /dev/null +++ b/roles/openshift-apps/datagrepper/templates/wsgi.py @@ -0,0 +1,3 @@ +from werkzeug.middleware.proxy_fix import ProxyFix +from datagrepper.app import app as application +application.wsgi_app = ProxyFix(application.wsgi_app, x_proto=1, x_host=1) diff --git a/roles/openshift-apps/datanommer/templates/configmap.yml b/roles/openshift-apps/datanommer/templates/configmap.yml index 375477903a..2fc7492c0c 100644 --- a/roles/openshift-apps/datanommer/templates/configmap.yml +++ b/roles/openshift-apps/datanommer/templates/configmap.yml @@ -5,7 +5,7 @@ kind: ConfigMap metadata: name: fedora-messaging-config labels: - app: datanommer + app: Datanommer data: config.toml: |- {{ load_file('fedora-messaging-config.toml') | indent(6) }}