Nagios monitoring for bodhi2.

roles/nagios_client/tasks/main.yml

@@ -90,6 +90,7 @@
   - check_fedmsg_irc_proc.cfg
   - check_fedmsg_relay_proc.cfg
   - check_fedmsg_gateway_proc.cfg
+  - check_fedmsg_masher_proc.cfg
   - check_redis_proc.cfg
   - check_fcomm_cache_worker_proc.cfg
   - check_fcomm_queue.cfg

roles/nagios_client/templates/check_fedmsg_consumers.cfg.j2

@@ -12,6 +12,7 @@ command[check_fedmsg_cp_notifs_backend]={{libdir}}/nagios/plugins/check_fedmsg_p
 command[check_fedmsg_cp_bugzilla2fedmsg]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py moksha-hub BugzillaConsumer MonitoringProducer
 command[check_fedmsg_cp_fedimg_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub KojiConsumer MonitoringProducer
 command[check_fedmsg_cp_hotness_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub BugzillaTicketFiler MonitoringProducer
+command[check_fedmsg_cp_bodhi_backend_hub]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub Masher MonitoringProducer
 
 command[check_fedmsg_cexceptions_busgateway_hub]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub Nommer 1 10
 command[check_fedmsg_cexceptions_busgateway_relay]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-relay RelayConsumer 1 10
@@ -26,6 +27,7 @@ command[check_fedmsg_cexceptions_notifs_backend]={{libdir}}/nagios/plugins/check
 command[check_fedmsg_cexceptions_bugzilla2fedmsg]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py moksha-hub BugzillaConsumer 1 10
 command[check_fedmsg_cexceptions_fedimg_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub KojiConsumer 1 10
 command[check_fedmsg_cexceptions_hotness_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub BugzillaTicketFiler 1 10
+command[check_fedmsg_cexceptions_bodhi_backend_hub]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub Masher 1 10
 
 command[check_fedmsg_cbacklog_busgateway_hub]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub Nommer 500 1000
 command[check_fedmsg_cbacklog_busgateway_relay]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-relay RelayConsumer 10 50
@@ -40,6 +42,7 @@ command[check_fedmsg_cbacklog_notifs_backend]={{libdir}}/nagios/plugins/check_fe
 command[check_fedmsg_cbacklog_bugzilla2fedmsg]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py moksha-hub BugzillaConsumer 10 100
 command[check_fedmsg_cbacklog_fedimg_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub KojiConsumer 2000 5000
 command[check_fedmsg_cbacklog_hotness_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub BugzillaTicketFiler 100 500
+command[check_fedmsg_cbacklog_bodhi_backend_hub]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub Masher 500 1000
 
 command[check_fedmsg_fmn_digest_last_ran]={{libdir}}/nagios/plugins/check_fedmsg_producer_last_ran.py fedmsg-hub DigestProducer 90 600
 command[check_fedmsg_fmn_confirm_last_ran]={{libdir}}/nagios/plugins/check_fedmsg_producer_last_ran.py fedmsg-hub ConfirmationProducer 90 600

roles/nagios_client/templates/check_fedmsg_masher_proc.cfg.j2

@@ -0,0 +1 @@
+command[check_fedmsg_hub_proc]={{ libdir }}/nagios/plugins/check_procs -c 1:1 -C 'fedmsg-hub' -u masher

roles/nagios_server/files/nagios/hosts/bodhi-backend01.cfg

@@ -0,0 +1,7 @@
+define host {
+  host_name                    bodhi-backend01
+  alias                        bodhi-backend01.phx2.fedoraproject.org
+  use                          defaulttemplate
+  address                      bodhi-backend01.phx2.fedoraproject.org
+  parents                      bvirthost10
+}

roles/nagios_server/files/nagios/hosts/bodhi-backend02.cfg

@@ -0,0 +1,7 @@
+define host {
+  host_name                    bodhi-backend02
+  alias                        bodhi-backend02.phx2.fedoraproject.org
+  use                          defaulttemplate
+  address                      bodhi-backend02.phx2.fedoraproject.org
+  parents                      bvirthost06
+}

roles/nagios_server/files/nagios/hosts/bodhi03.cfg

@@ -0,0 +1,7 @@
+define host {
+  host_name                    bodhi03
+  alias                        bodhi03.phx2.fedoraproject.org
+  use                          defaulttemplate
+  address                      bodhi03.phx2.fedoraproject.org
+  parents                      virthost01
+}

roles/nagios_server/files/nagios/hosts/bodhi04.cfg

@@ -0,0 +1,7 @@
+define host {
+  host_name                    bodhi04
+  alias                        bodhi04.phx2.fedoraproject.org
+  use                          defaulttemplate
+  address                      bodhi04.phx2.fedoraproject.org
+  parents                      virthost02
+}

roles/nagios_server/files/nagios/services/fedmsg.cfg

@@ -87,6 +87,20 @@ define service {
   use                   defaulttemplate
 }
 
+define service {
+  host_name             bodhi-backend01
+  service_description   Check for fedmsg-hub proc
+  check_command         check_by_nrpe!check_fedmsg_masher_proc
+  use                   defaulttemplate
+}
+
+define service {
+  host_name             bodhi-backend02
+  service_description   Check for fedmsg-hub proc
+  check_command         check_by_nrpe!check_fedmsg_masher_proc
+  use                   defaulttemplate
+}
+
 
 # Odd one, check for the supybot fedmsg plugin
 define service {
@@ -366,6 +380,19 @@ define service {
   use                   defaulttemplate
 }
 
+define service {
+  host_name             bodhi-backend01
+  service_description   Check fedmsg consumers and producers hub
+  check_command         check_by_nrpe!check_fedmsg_cp_bodhi_backend_hub
+  use                   defaulttemplate
+}
+
+define service {
+  host_name             bodhi-backend02
+  service_description   Check fedmsg consumers and producers hub
+  check_command         check_by_nrpe!check_fedmsg_cp_bodhi_backend_hub
+  use                   defaulttemplate
+}
 
 # BEGIN exceptions counter
 define service {
@@ -459,6 +486,20 @@ define service {
   use                   defaulttemplate
 }
 
+define service {
+  host_name             bodhi-backend01
+  service_description   Check fedmsg-hub consumers exceptions
+  check_command         check_by_nrpe!check_fedmsg_cexceptions_bodhi_backend_hub
+  use                   defaulttemplate
+}
+
+define service {
+  host_name             bodhi-backend02
+  service_description   Check fedmsg-hub consumers exceptions
+  check_command         check_by_nrpe!check_fedmsg_cexceptions_bodhi_backend_hub
+  use                   defaulttemplate
+}
+
 
 # BEGIN backlog checking
 define service {
@@ -552,6 +593,20 @@ define service {
   use                   defaulttemplate
 }
 
+define service {
+  host_name             bodhi-backend01
+  service_description   Check fedmsg-hub consumers backlog
+  check_command         check_by_nrpe!check_fedmsg_cbacklog_bodhi_backend_hub
+  use                   defaulttemplate
+}
+
+define service {
+  host_name             bodhi-backend02
+  service_description   Check fedmsg-hub consumers backlog
+  check_command         check_by_nrpe!check_fedmsg_cbacklog_bodhi_backend_hub
+  use                   defaulttemplate
+}
+
 
 # A few producer checks go below here
 define service {

roles/nagios_server/files/nagios/services/websites.cfg

@@ -87,6 +87,22 @@ define service {
   use                   internalwebsitetemplate
 }
 
+define service {
+  host_name             proxy01-wildcard, proxy02-wildcard, proxy03-fpo, proxy04-fpo, proxy06-fpo, proxy07-wildcard, proxy08-wildcard, proxy09-wildcard, proxy05-fpo, proxy10-fpo, proxy11-fpo
+  service_description   bodhi2
+  check_command         check_website_ssl!bodhi.fedoraproject.org!/!top testers
+  max_check_attempts    8
+  use                   websitetemplate
+}
+
+define service {
+  host_name             bodhi03,bodhi04
+  service_description   bodhi2-internal
+  check_command         check_website!localhost!/!top testers
+  max_check_attempts    8
+  use                   internalwebsitetemplate
+}
+
 define service {
   host_name		github2fedmsg01
   service_description	github2fedmsg-internal

roles/nagios_server/files/nrpe.cfg

@@ -231,6 +231,7 @@ command[check_fedmsg_hub_proc]=/usr/lib64/nagios/plugins/check_procs -c 1:1 -C '
 command[check_fedmsg_gateway_proc]=/usr/lib64/nagios/plugins/check_procs -c 1:1 -C 'fedmsg-gateway' -u fedmsg
 command[check_fedmsg_irc_proc]=/usr/lib64/nagios/plugins/check_procs -c 1:1 -C 'fedmsg-irc' -u fedmsg
 command[check_fedmsg_tweet_proc]=/usr/lib64/nagios/plugins/check_procs -c 1:1 -C 'fedmsg-tweet' -u fedmsg
+command[check_fedmsg_masher_proc]=/usr/lib64/nagios/plugins/check_procs -c 1:1 -C 'fedmsg-hub' -u masher
 command[check_supybot_fedmsg_plugin]=/usr/lib64/nagios/plugins/check_supybot_plugin -t fedmsg
 command[check_haproxy_conns]=/usr/lib64/nagios/plugins/check_haproxy_conns.py
 command[check_fcomm_cache_worker_proc]=/usr/lib64/nagios/plugins/check_procs -c 1:1 -C 'fcomm-cache-wor' -u apache
@@ -292,6 +293,7 @@ command[check_fedmsg_cp_notifs_backend]=/usr/lib64/nagios/plugins/check_fedmsg_p
 command[check_fedmsg_cp_bugzilla2fedmsg]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py moksha-hub BugzillaConsumer MonitoringProducer
 command[check_fedmsg_cp_fedimg_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub KojiConsumer MonitoringProducer
 command[check_fedmsg_cp_hotness_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub BugzillaTicketFiler MonitoringProducer
+command[check_fedmsg_cp_bodhi_backend_hub]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub Masher MonitoringProducer
 
 command[check_fedmsg_cexceptions_busgateway_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub Nommer 1 10
 command[check_fedmsg_cexceptions_busgateway_relay]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-relay RelayConsumer 1 10
@@ -305,6 +307,7 @@ command[check_fedmsg_cexceptions_notifs_backend]=/usr/lib64/nagios/plugins/check
 command[check_fedmsg_cexceptions_bugzilla2fedmsg]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py moksha-hub BugzillaConsumer 1 10
 command[check_fedmsg_cexceptions_fedimg_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub KojiConsumer 1 10
 command[check_fedmsg_cexceptions_hotness_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub BugzillaTicketFiler 1 10
+command[check_fedmsg_cexceptions_bodhi_backend_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub Masher 1 10
 
 command[check_fedmsg_cbacklog_busgateway_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub Nommer 500 1000
 command[check_fedmsg_cbacklog_busgateway_relay]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-relay RelayConsumer 10 50
@@ -318,6 +321,7 @@ command[check_fedmsg_cbacklog_notifs_backend]=/usr/lib64/nagios/plugins/check_fe
 command[check_fedmsg_cbacklog_bugzilla2fedmsg]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py moksha-hub BugzillaConsumer 10 100
 command[check_fedmsg_cbacklog_fedimg_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub KojiConsumer 2000 5000
 command[check_fedmsg_cbacklog_hotness_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub BugzillaTicketFiler 100 500
+command[check_fedmsg_cbacklog_bodhi_backend_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub Masher 500 1000
 
 command[check_fedmsg_fmn_digest_last_ran]={{libdir}}/nagios/plugins/check_fedmsg_producer_last_ran.py fedmsg-hub DigestProducer 90 600
 command[check_fedmsg_fmn_confirm_last_ran]={{libdir}}/nagios/plugins/check_fedmsg_producer_last_ran.py fedmsg-hub ConfirmationProducer 30 300