Refactor fedmsg-hub memory limit

roles/badges/backend/files/systemd/systemd-memory-limit.conf

@@ -1,2 +0,0 @@
-[Service]
-MemoryLimit=8G

roles/badges/backend/tasks/main.yml

@@ -14,61 +14,6 @@
   - badges
   - badges/backend
 
-- name: create systemd config directories
-  file: >
-    path="/etc/systemd/system/{{ item }}.service.d"
-    state=directory
-  with_items: 
-  - fedmsg-hub
-  notify:
-  - reload systemd
-  tags:
-  - badges/backend
-  - config
-  - service
-
-- name: install systemd config files (memory)
-  copy: >
-    src=systemd/systemd-memory-limit.conf
-    dest=/etc/systemd/system/{{ item }}.service.d/memory-limit.conf
-  with_items:
-  - fedmsg-hub
-  notify:
-  - reload systemd
-  - restart fedmsg-hub
-  tags:
-  - badges/backend
-  - config
-  - service
-
-- name: install systemd config files (restart)
-  template: >
-    src=systemd-restart-limit.conf.j2
-    dest=/etc/systemd/system/{{ item }}.service.d/restart.conf
-  with_items: 
-  - fedmsg-hub
-  notify:
-  - reload systemd
-  tags:
-  - badges/backend
-  - config
-  - service
-
-- name: install systemd config files (environment)
-  template: >
-    src=systemd-environment.conf.j2
-    dest=/etc/systemd/system/{{ item }}.service.d/environment.conf
-  with_items: 
-  - fedmsg-hub
-  notify:
-  - reload systemd
-  - restart fedmsg-hub
-  tags:
-  - badges/backend
-  - config
-  - service
-
-
 - name: copy database configuration
   template: >
     src={{ item }} dest="/etc/fedmsg.d/{{ item }}"

roles/badges/backend/templates/systemd-restart-limit.conf.j2

@@ -1,6 +0,0 @@
-[Service]
-{% if env == 'staging' %}
-RestartSec=600
-{% else %}
-RestartSec=10
-{% endif %}

roles/fedmsg/hub/defaults/main.yml

@@ -1,2 +1,8 @@
 # This is false by default.  You have to override it when including the role.
 enable_websocket_server: False
+
+# Absolute memory limit in MB for fedmsg-hub.service. Zero means infinity.
+fedmsg_hub_memory_limit: 0
+
+# Restart fedmsg-hub.service on failure.
+fedmsg_hub_auto_restart: False

roles/fedmsg/hub/tasks/main.yml

@@ -63,3 +63,26 @@
   - restart nrpe
   tags:
   - fedmsgmonitor
+
+- name: create systemd config directoryies
+  file: path="/etc/systemd/system/{{ item }}.service.d" state=directory
+  with_items:
+  - fedmsg-hub
+  - fedmsg-hub-3
+  notify:
+  - reload systemd
+  tags:
+  - fedmsg/hub
+
+- name: install systemd config file
+  template:
+    src: fedmsg-hub-systemd.conf.j2
+    dest: "/etc/systemd/system/{{ item }}.service.d/systemd.conf"
+  with_items:
+  - fedmsg-hub
+  - fedmsg-hub-3
+  notify:
+  - reload systemd
+  - restart fedmsg-hub
+  tags:
+  - fedmsg/hub

roles/fedmsg/hub/templates/fedmsg-hub-systemd.conf.j2

@@ -0,0 +1,23 @@
+[Service]
+{% if fedmsg_hub_memory_limit %}
+{% if ansible_distribution_major_version|int > 7 %}
+# Don't let fedmsg-hub use more that half of swap available in the
+# system, so that we don't get Nagios alerts about low swap.  We would
+# rather have fedmsg-hub OOM and be auto-restarted.
+MemorySwapMax={{ ansible_memory_mb.swap.total / 2 }}M
+# Limit RAM usage too
+MemoryMax={{fedmsg_hub_memory_limit}}
+{% else %}
+# On RHEL 7 we can only control RAM usage.
+MemoryLimit={{fedmsg_hub_memory_limit}}
+{% endif %}
+{% endif %}
+
+{% if fedmsg_hub_auto_restart %}
+Restart=on-failure
+{% if env == 'staging' %}
+RestartSec=600
+{% else %}
+RestartSec=10
+{% endif %}
+{% endif %}