From 565758665c750515b6bfde346ed44e25ec0ce98e Mon Sep 17 00:00:00 2001
From: Mikolaj Izdebski <mizdebsk@redhat.com>
Date: Thu, 21 Feb 2019 17:11:05 +0100
Subject: [PATCH] Refactor fedmsg-hub memory limit

---
 .../files/systemd/systemd-memory-limit.conf   |  2 -
 roles/badges/backend/tasks/main.yml           | 55 -------------------
 .../templates/systemd-restart-limit.conf.j2   |  6 --
 roles/fedmsg/hub/defaults/main.yml            |  6 ++
 roles/fedmsg/hub/tasks/main.yml               | 23 ++++++++
 .../hub/templates/fedmsg-hub-systemd.conf.j2  | 23 ++++++++
 6 files changed, 52 insertions(+), 63 deletions(-)
 delete mode 100644 roles/badges/backend/files/systemd/systemd-memory-limit.conf
 delete mode 100644 roles/badges/backend/templates/systemd-restart-limit.conf.j2
 create mode 100644 roles/fedmsg/hub/templates/fedmsg-hub-systemd.conf.j2

diff --git a/roles/badges/backend/files/systemd/systemd-memory-limit.conf b/roles/badges/backend/files/systemd/systemd-memory-limit.conf
deleted file mode 100644
index 6a91cce212..0000000000
--- a/roles/badges/backend/files/systemd/systemd-memory-limit.conf
+++ /dev/null
@@ -1,2 +0,0 @@
-[Service]
-MemoryLimit=8G
diff --git a/roles/badges/backend/tasks/main.yml b/roles/badges/backend/tasks/main.yml
index 87d8952608..7befdbbfbb 100644
--- a/roles/badges/backend/tasks/main.yml
+++ b/roles/badges/backend/tasks/main.yml
@@ -14,61 +14,6 @@
   - badges
   - badges/backend
 
-- name: create systemd config directories
-  file: >
-    path="/etc/systemd/system/{{ item }}.service.d"
-    state=directory
-  with_items: 
-  - fedmsg-hub
-  notify:
-  - reload systemd
-  tags:
-  - badges/backend
-  - config
-  - service
-
-- name: install systemd config files (memory)
-  copy: >
-    src=systemd/systemd-memory-limit.conf
-    dest=/etc/systemd/system/{{ item }}.service.d/memory-limit.conf
-  with_items:
-  - fedmsg-hub
-  notify:
-  - reload systemd
-  - restart fedmsg-hub
-  tags:
-  - badges/backend
-  - config
-  - service
-
-- name: install systemd config files (restart)
-  template: >
-    src=systemd-restart-limit.conf.j2
-    dest=/etc/systemd/system/{{ item }}.service.d/restart.conf
-  with_items: 
-  - fedmsg-hub
-  notify:
-  - reload systemd
-  tags:
-  - badges/backend
-  - config
-  - service
-
-- name: install systemd config files (environment)
-  template: >
-    src=systemd-environment.conf.j2
-    dest=/etc/systemd/system/{{ item }}.service.d/environment.conf
-  with_items: 
-  - fedmsg-hub
-  notify:
-  - reload systemd
-  - restart fedmsg-hub
-  tags:
-  - badges/backend
-  - config
-  - service
-
-
 - name: copy database configuration
   template: >
     src={{ item }} dest="/etc/fedmsg.d/{{ item }}"
diff --git a/roles/badges/backend/templates/systemd-restart-limit.conf.j2 b/roles/badges/backend/templates/systemd-restart-limit.conf.j2
deleted file mode 100644
index 9cf6fd950d..0000000000
--- a/roles/badges/backend/templates/systemd-restart-limit.conf.j2
+++ /dev/null
@@ -1,6 +0,0 @@
-[Service]
-{% if env == 'staging' %}
-RestartSec=600
-{% else %}
-RestartSec=10
-{% endif %}
diff --git a/roles/fedmsg/hub/defaults/main.yml b/roles/fedmsg/hub/defaults/main.yml
index 0224714ea9..4f0428de67 100644
--- a/roles/fedmsg/hub/defaults/main.yml
+++ b/roles/fedmsg/hub/defaults/main.yml
@@ -1,2 +1,8 @@
 # This is false by default.  You have to override it when including the role.
 enable_websocket_server: False
+
+# Absolute memory limit in MB for fedmsg-hub.service. Zero means infinity.
+fedmsg_hub_memory_limit: 0
+
+# Restart fedmsg-hub.service on failure.
+fedmsg_hub_auto_restart: False
diff --git a/roles/fedmsg/hub/tasks/main.yml b/roles/fedmsg/hub/tasks/main.yml
index 0f7b73b4e7..f669f66363 100644
--- a/roles/fedmsg/hub/tasks/main.yml
+++ b/roles/fedmsg/hub/tasks/main.yml
@@ -63,3 +63,26 @@
   - restart nrpe
   tags:
   - fedmsgmonitor
+
+- name: create systemd config directoryies
+  file: path="/etc/systemd/system/{{ item }}.service.d" state=directory
+  with_items:
+  - fedmsg-hub
+  - fedmsg-hub-3
+  notify:
+  - reload systemd
+  tags:
+  - fedmsg/hub
+
+- name: install systemd config file
+  template:
+    src: fedmsg-hub-systemd.conf.j2
+    dest: "/etc/systemd/system/{{ item }}.service.d/systemd.conf"
+  with_items:
+  - fedmsg-hub
+  - fedmsg-hub-3
+  notify:
+  - reload systemd
+  - restart fedmsg-hub
+  tags:
+  - fedmsg/hub
diff --git a/roles/fedmsg/hub/templates/fedmsg-hub-systemd.conf.j2 b/roles/fedmsg/hub/templates/fedmsg-hub-systemd.conf.j2
new file mode 100644
index 0000000000..bdbf5d4361
--- /dev/null
+++ b/roles/fedmsg/hub/templates/fedmsg-hub-systemd.conf.j2
@@ -0,0 +1,23 @@
+[Service]
+{% if fedmsg_hub_memory_limit %}
+{% if ansible_distribution_major_version|int > 7 %}
+# Don't let fedmsg-hub use more that half of swap available in the
+# system, so that we don't get Nagios alerts about low swap.  We would
+# rather have fedmsg-hub OOM and be auto-restarted.
+MemorySwapMax={{ ansible_memory_mb.swap.total / 2 }}M
+# Limit RAM usage too
+MemoryMax={{fedmsg_hub_memory_limit}}
+{% else %}
+# On RHEL 7 we can only control RAM usage.
+MemoryLimit={{fedmsg_hub_memory_limit}}
+{% endif %}
+{% endif %}
+
+{% if fedmsg_hub_auto_restart %}
+Restart=on-failure
+{% if env == 'staging' %}
+RestartSec=600
+{% else %}
+RestartSec=10
+{% endif %}
+{% endif %}