FMN: add the collectd stats generator

Signed-off-by: Aurélien Bompard <aurelien@bompard.org>

roles/collectd/server/files/fmn-types.db

@@ -0,0 +1 @@
+fmn_cache        value:GAUGE:0:U

roles/collectd/server/files/fmn.conf

@@ -0,0 +1 @@
+TypesDB "/usr/share/collectd/fmn-types.db"

roles/collectd/server/tasks/main.yml

@@ -29,9 +29,27 @@
   - collectd/server
   when: ansible_distribution_major_version|int == 8
 
-# install collectd rrd config
+# install collectd types
-- name: install collectd rrd config
+- name: install collectd types
-  copy: src=rrdtool.conf dest=/etc/collectd.d/rrdtool.conf
+  copy:
+    src: "{{ item }}"
+    dest: /usr/share/collectd/{{ item }}
+  loop:
+  - fmn-types.db
+  notify:
+  - restart collectd
+  tags:
+  - config
+  - collectd/server
+
+# install collectd configs
+- name: install collectd config
+  copy:
+    src: "{{ item }}"
+    dest: /etc/collectd.d/{{ item }}
+  loop:
+  - rrdtool.conf
+  - fmn-data.conf
   notify:
   - restart collectd
   tags:

roles/openshift-apps/fmn/files/imagestream.yml

@@ -23,6 +23,11 @@ items:
             scheduled: true
           referencePolicy:
             type: Source
+  # The Python 3.10 builder image plus collectd
+  - apiVersion: image.openshift.io/v1
+    kind: ImageStream
+    metadata:
+      name: python-310-collectd
   # Sendria (to view email notifications in staging)
   - apiVersion: image.openshift.io/v1
     kind: ImageStream

roles/openshift-apps/fmn/templates/Dockerfile-builder-image

@@ -0,0 +1,8 @@
+FROM quay.io/fedora/python-310:latest
+LABEL \
+    name="bodhi-base" \
+    vendor="Fedora Infrastructure" \
+    license="MIT"
+USER root
+RUN dnf install -y collectd collectd-python patchutils
+USER 1001

roles/openshift-apps/fmn/templates/buildconfig.yml

@@ -2,6 +2,32 @@
 ---
 apiVersion: build.openshift.io/v1
 kind: BuildConfig
+metadata:
+  name: python-310-collectd
+  labels:
+    app: fmn
+    build: python-310-collectd
+spec:
+  runPolicy: Serial
+  source:
+    type: Dockerfile
+    dockerfile: |-
+      {{ load_file('Dockerfile-builder-image') | indent(6) }}
+  strategy:
+    type: Docker
+    dockerStrategy:
+      from:
+        kind: ImageStreamTag
+        name: quay.io/fedora/python-310:latest
+  output:
+    to:
+      kind: ImageStreamTag
+      name: python-310-collectd:latest
+  triggers:
+  - type: ImageChange
+---
+apiVersion: build.openshift.io/v1
+kind: BuildConfig
 metadata:
   name: fmn
   labels:
@@ -20,10 +46,8 @@ spec:
     sourceStrategy:
       from:
         kind: ImageStreamTag
-        # name: python:3.10
+        # Our own ImageStream with collectd added to it
-        # namespace: openshift
+        name: python-310-collectd:latest
-        # OpenShift does not have Python 3.10 (yet), use our own ImageStream
-        name: python-310:latest
       env:
       # The .s2i/bin/assemble script uses this to build the frontend with the
       # correct API and Ipsilon addresses.

roles/openshift-apps/fmn/templates/collectd.conf

@@ -0,0 +1,51 @@
+#
+# Collectd configuration (fmn.core.collectd)
+#
+
+Hostname    "notifications{{ env_suffix }}.fedoraproject.org"
+FQDNLookup   true
+#BaseDir     "/usr/var/lib/collectd"
+#PIDFile     "/usr/var/run/collectd.pid"
+#PluginDir   "/usr/lib/collectd"
+#Interval     10
+#ReadThreads  5
+
+# This is the default but it needs to be defined so we can add more DB files later.
+TypesDB     "/usr/share/collectd/types.db"
+
+LoadPlugin logfile
+
+<Plugin LogFile>
+    LogLevel "info"
+    File STDOUT
+    Timestamp true
+</Plugin>
+
+# Send data to log01
+LoadPlugin network
+<Plugin "network">
+    Server "log01"
+</Plugin>
+
+# FMN
+
+TypesDB "/opt/app-root/src/config/collectd-types.db"
+
+<LoadPlugin python>
+  Globals true
+</LoadPlugin>
+
+<Plugin python>
+  LogTraces true
+  Interactive false
+  ModulePath "/opt/app-root/src"
+  Import "fmn.core.collectd"
+
+  <Module "fmn.core.collectd">
+    ## Interval between two collections. The collectd default of 10 seconds is
+    ## way too short, this plugin sets the default to 1h (3600s). Adjust
+    ## depending on how frequently the cache is rebuilt. Remember that if you
+    ## change the interval, you'll have to recreate your RRD files.
+    # Interval 3600
+  </Module>
+</Plugin>

roles/openshift-apps/fmn/templates/configmap.yml

@@ -23,3 +23,5 @@ items:
       {{ load_file('sender-irc.toml') | indent(6) }}
     sender-matrix.toml: |-
       {{ load_file('sender-matrix.toml') | indent(6) }}
+    collectd.conf: |-
+      {{ load_file('collectd.conf') | indent(6) }}

roles/openshift-apps/fmn/templates/deploymentconfig.yml

@@ -520,6 +520,63 @@ spec:
           kind: ImageStreamTag
           name: redis:latest
 ---
+# Collectd stats collector
+apiVersion: apps.openshift.io/v1
+kind: DeploymentConfig
+metadata:
+  name: collectd
+  labels:
+    app: fmn
+spec:
+  replicas: 1
+  selector:
+    app: fmn
+    deploymentconfig: collectd
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: fmn
+        deploymentconfig: collectd
+    spec:
+      containers:
+        - name: collectd
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: etc-fmn
+              mountPath: "/etc/fmn"
+              readOnly: true
+          env:
+            - name: APP_SCRIPT
+              value: ".s2i/run-collectd.sh"
+            - name: CACHE__SETUP_ARGS__PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: fmn
+                  key: redis-password
+            - name: CACHE__SETUP_ARGS__HASH_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: fmn
+                  key: cache-hash-key
+
+      volumes:
+        - name: etc-fmn
+          configMap:
+            name: fmn
+  triggers:
+    - type: ConfigChange
+    - type: ImageChange
+      imageChangeParams:
+        automatic: true
+        containerNames:
+          - collectd
+        from:
+          kind: ImageStreamTag
+          name: fmn:latest
+---
 # Sendria, to display email notifications in staging
 {% if env == 'staging' %}
 apiVersion: apps.openshift.io/v1