Add thresholds on Bodhi's RabbitMQ queues

Signed-off-by: Aurélien Bompard <aurelien@bompard.org>

playbooks/groups/bodhi-backend.yml

@@ -71,6 +71,9 @@
     username: "bodhi{{ env_suffix }}"
     queue_name: "{{ bodhi_message_queue_name }}"
     routing_keys: "{{ bodhi_message_routing_keys }}"
+    thresholds:
+      warning: 10
+      critical: 100
 
 
   tasks:

playbooks/openshift-apps/bodhi.yml

@@ -26,6 +26,9 @@
     username: "bodhi{{ env_suffix }}"
     queue_name: "{{ bodhi_message_queue_name }}"
     routing_keys: "{{ bodhi_message_routing_keys }}"
+    thresholds:
+      warning: 10
+      critical: 100
   - role: openshift/project
     app: bodhi
     description: bodhi

roles/rabbit/queue/defaults/main.yml

@@ -1,5 +1,7 @@
-rabbitmq_server: "rabbitmq01{{ env_suffix }}.phx2.fedoraproject.org"
+rabbitmq_server: "rabbitmq03{{ env_suffix }}.phx2.fedoraproject.org"
 vhost: /pubsub
 default_exchange: amq.topic
 routing_keys:
   - "#"
+thresholds: false
+nagios_server: noc01.phx2.fedoraproject.org

roles/rabbit/queue/tasks/main.yml

@@ -17,6 +17,9 @@
 #
 #   - write_queues (list): A list of queue name prefixes to which the user will
 #                          be allowed to publish.
+#   - thresholds (dict): A dictionary with two keys: "warning" and "critical".
+#                        The values are numbers. Generate an alert in Nagios if
+#                        the number of messages go above these values.
 
 - assert:
     that:
@@ -61,3 +64,25 @@
     login_user: admin
     login_password: "{{ (env == 'production')|ternary(rabbitmq_admin_password_production, rabbitmq_admin_password_staging) }}"
   loop: "{{ routing_keys }}"
+
+- name: Monitor the {{ queue_name }} queue in Nagios (NRPE)
+  when: thresholds and env == "production"
+  delegate_to: "{{ rabbitmq_server }}"
+  template:
+    src: nrpe.cfg.j2
+    dest: /etc/nrpe.d/check_rabbitmq_queue_{{ queue_name }}.cfg
+    owner: root
+    group: root
+    mode: 0644
+  notify:
+  - restart nrpe
+  tags:
+  - nagios_client
+
+- name: Monitor the {{ queue_name }} queue in Nagios
+  when: thresholds and env == "production"
+  delegate_to: "{{ nagios_server }}"
+  template:
+    src: nagios.cfg.j2
+    dest: /etc/nagios/services/rabbitmq-queue-{{ queue_name }}.cfg
+  notify: restart nagios

roles/rabbit/queue/templates/nagios.cfg.j2

@@ -0,0 +1,6 @@
+define service {
+  host_name             {{ rabbitmq_server }}
+  service_description   Check queue {{ queue_name }}
+  check_command         check_by_nrpe!check_rabbitmq_queue_{{ vhost.replace("/", "") }}_{{ queue_name }}
+  use                   defaulttemplate
+}

roles/rabbit/queue/templates/nrpe.cfg.j2

@@ -0,0 +1 @@
+command[check_rabbitmq_queue_{{ vhost.replace("/", "") }}_{{ queue_name }}]=/usr/lib64/nagios/plugins-rabbitmq/check_rabbitmq_queue --extra-opts=common@/etc/nrpe.d/rabbitmq_args.ini --vhost {{ vhost }} --queue {{ queue_name }} {% if thresholds.warning %}-w {{ thresholds.warning }}{% endif %}{% if thresholds.critical %}-c {{ thresholds.critical }}{% endif %}