Added zabbix playbook

playbooks/groups/bodhi-backend.yml

@@ -111,6 +111,11 @@
       warning: 10
       critical: 100
 
+  - role: zabbix/zabbix-agent
+    zabbix_server: apache
+    zabbix_server_ip: apache
+    when: env == "staging"
+
   tasks:
   - name: create secondary volume dir for stg bodhi
     file: dest=/mnt/koji/vol state=directory owner=apache group=apache mode=0755

roles/zabbix/zabbix-agent/defaults/main.yml

@@ -0,0 +1,8 @@
+# Defaults variables for role zabbix-agent
+zabbix_server: zabbix01.stg.iad2.fedoraproject.org
+
+# TLS/PSK settings to encrypt between agent and proxy/server
+zabbix_agent_tls: False
+zabbix_agent_tls_psk:           # gen with `openssl rand -hex 32` and also known by zabbix server
+zabbix_agent_tls_psk_identity:  # Whatever makes sense to you and known by zabbix server
+

roles/zabbix/zabbix-agent/files/scripts/zabbix-check-eth-settings.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+export PATH=$PATH:/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin
+uname -a|grep -q xen >/dev/null
+if [ "$?" -eq "0" ] ;then
+ eth_dev=p$(ip route|grep default|awk '{print $5}')
+else
+ eth_dev=$(ip route|grep default|awk '{print $5}')
+fi
+
+lsmod |grep -q -E 'virtio_net|xen_net'
+if [ "$?" -eq "1" ] ;then
+  zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k eth_dev.speed -o $(ethtool $eth_dev|grep Speed|awk '{print $2}'|tr -d [:alpha:]|tr -d '/') >/dev/null
+  zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k eth_dev.duplex -o $(ethtool $eth_dev|grep Duplex|awk '{print $2}') >/dev/null
+fi
+
+
+
+

roles/zabbix/zabbix-agent/files/scripts/zabbix-check-iptables.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+# called by Zabbix to see if iptables is running
+PATH=$PATH:/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin
+
+if [ $( iptables -n -L INPUT|wc -l ) -gt 6 ] ; then
+  iptablesstatus="0"
+else
+  iptablesstatus="1"
+fi
+
+zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k net.iptables.status -o $iptablesstatus >/dev/null

roles/zabbix/zabbix-agent/files/scripts/zabbix-check-ro.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+logfile="/var/log/centos-ro-devices"
+
+ro_mounted_devices=$(awk '$4 ~ "^ro[,$]" && $3 !~ "(squashfs|iso9660|tmpfs|nfs)" {print $0}' /proc/mounts | wc -l)
+zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k devices.ro -o $ro_mounted_devices >/dev/null
+echo "=== $(date) === Read-Only devices on this system $(hostname)" >> $logfile
+awk '$4 ~ "^ro[,$]" && $3 !~ "(squashfs|iso9660|tmpfs|nfs)" {print $0}    ' /proc/mounts >> $logfile

roles/zabbix/zabbix-agent/files/scripts/zabbix-hw-raid-check.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+PATH=$PATH:/usr/local/bin:/sbin:/usr/sbin/
+
+function init_log() {
+logfile=/var/log/centos-hw-raid.log
+echo "=========================================================" > $logfile
+echo " CentOS Hardware Raid check - $(date +%Y%m%d-%H%M)" >> $logfile
+echo "=========================================================" >> $logfile
+}
+
+function 3w_xxxx_check() {
+echo "3ware controller found .. launching raid check" >> $logfile
+for controller in $(tw_cli show |grep ^c|awk '{print $1}') ;
+do
+  for disk in $(tw_cli /${controller} show |grep ^p|awk '{print $1}') ;
+  do
+    tw_cli /${controller} show |grep ^${disk}|egrep -q 'OK|NOT-PRESENT'
+    if [ "$?" -ne "0" ] ;then
+      tw_cli /${controller} show >> $logfile
+      zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k hwraid.3ware -o 1 >/dev/null
+      exit 1
+    else
+      echo "3ware controller ${controller} / array ${array} status : OK" >> $logfile
+      zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k hwraid.3ware -o 0 >/dev/null  
+    fi
+  done
+done
+}
+
+function 3w_9xxx_check() {
+ 3w_xxxx_check
+}
+
+function arcmsr_check() {
+echo "ARECA controller found .. launching raid check" >> $logfile
+for array in $(areca-cli rsf info|egrep -v 'Name|=|GuiErr'|awk '{print $1}') ;
+do
+  areca-cli rsf info raid=${array}|grep -q Normal
+  if [ "$?" -ne "0" ] ;then
+    areca-cli rsf info raid=${array} >> $logfile
+    zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k hwraid.arcmsr -o 1 >/dev/null
+    exit 1
+  else
+    echo "Areca array ${array} status : OK" >> $logfile
+    zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k hwraid.arcmsr -o 0 >/dev/null  
+  fi
+done
+}
+
+function megaraid_sas_check() {
+echo "Megaraid_sas controller found .. launching raid check" >> $logfile
+for LDid in $(/opt/MegaRAID/MegaCli/MegaCli64 -LDInfo -Lall -aALL|grep "Virtual Drive:"|awk '{print $3}') ;
+do
+  /opt/MegaRAID/MegaCli/MegaCli64 -LDInfo -L${LDid} -aALL|grep -q Optimal 
+  if [ "$?" -ne "0" ] ;then
+    /opt/MegaRAID/MegaCli/MegaCli64 -ShowSummary -aALL >>$logfile
+    zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k hwraid.megaraid -o 1 >/dev/null
+    exit 1
+  else
+    echo "Megaraid_sas array ${array} status : OK" >> $logfile
+    /opt/MegaRAID/MegaCli/MegaCli64 -ShowSummary -aALL >>$logfile
+    zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k hwraid.megaraid -o 0 >/dev/null  
+  fi
+done
+}
+
+init_log
+
+# Ensuring sg kmod is loaded, as needed by those tools ...
+lsmod|grep -q sg || modprobe sg
+
+for kmod in 3w_9xxx 3w_xxxx arcmsr megaraid_sas; do
+  /sbin/lsmod |grep -q ${kmod}
+  if [ "$?" = "0" ];then
+    ${kmod}_check
+  fi
+done

roles/zabbix/zabbix-agent/files/scripts/zabbix-mdstat-check.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+grep md /proc/mdstat >/dev/null 2>&1
+if [ $? = "0" ] ;then
+  for mddev in $( grep md /proc/mdstat |awk '{print $1}') ;
+  do
+    md_count=$(/sbin/mdadm --detail /dev/${mddev}|grep Failed|cut -f 2 -d ":"|tr -d [:blank:])
+    if [ $md_count > "0" ] ;then
+      zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k mdstat.failed -o $md_count >/dev/null
+      exit
+    else
+      zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k mdstat.failed -o 0 >/dev/null
+    fi
+  done
+fi
+

roles/zabbix/zabbix-agent/files/selinux/8/centos-zabbix-agent.te

@@ -0,0 +1,20 @@
+policy_module(centos-zabbix-agent, 1.1)
+
+require{
+ type zabbix_agent_t;
+ type zabbix_t;
+ type ping_t;
+ type zabbix_tmp_t;
+ class process setrlimit;
+}
+
+allow ping_t zabbix_tmp_t:file read_file_perms;
+allow ping_t zabbix_t:tcp_socket { read write };
+allow zabbix_agent_t self:process setrlimit;
+
+kernel_read_network_state(zabbix_agent_t)
+domain_read_all_domains_state(zabbix_agent_t)
+dev_read_sysfs(zabbix_agent_t)
+corenet_tcp_connect_all_ports(zabbix_agent_t)
+
+

roles/zabbix/zabbix-agent/handlers/main.yml

@@ -0,0 +1,6 @@
+- name: restart_zabbix_agent
+  service: name=zabbix-agent state=restarted
+
+- name: reload custom selinux files
+  shell: /usr/sbin/semodule -u "/etc/selinux/centos/centos-zabbix-agent.pp"
+  when: ansible_selinux.mode == "enforcing"

roles/zabbix/zabbix-agent/tasks/main.yml

@@ -0,0 +1,72 @@
+- name: Importing specific distro variables
+  include_vars: "{{ item }}"
+  with_first_found:
+    - "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yml"
+    - "{{ ansible_distribution }}.yml"
+    - common.yml
+
+- name: Installing Zabbix packages
+  yum:
+    name: "{{ zabbix_pkgs_list }}"
+    state: latest
+  register: pkg_install
+
+- name: Installing selinux dependency packages
+  yum:
+    name: "{{ pkgs_list }}"
+    state: installed
+  tags:
+    - packages
+
+- name: Distributing custom selinux policies
+  copy:
+    src: "selinux/{{ ansible_distribution_version[0] }}/{{ item }}"
+    dest: "/etc/selinux/centos/{{ item }}"
+  register: sepolicy
+  with_items:
+    - centos-zabbix-agent.pp
+
+- name: reload custom selinux files
+  shell: /usr/sbin/semodule -u "/etc/selinux/centos/centos-zabbix-agent.pp"
+  when: ansible_selinux.status == "enabled" and sepolicy.changed
+
+- name: Allowing zabbix to connect to network resources
+  seboolean:
+    name: zabbix_can_network
+    persistent: yes
+    state: yes
+  when: ansible_selinux.status == "enabled"
+
+- name: Configuring Zabbix agentd
+  template:
+    src: zabbix_agentd.conf.j2
+    dest: /etc/zabbix/zabbix_agentd.conf
+  notify: restart_zabbix_agent
+
+- name: Ensuring we have a directory to put zabbix scripts
+  file:
+    state: directory
+    path: /usr/lib/zabbix
+    mode: 0770
+    owner: zabbix
+    group: zabbix
+
+- name: Adding some other conf files under zabbix_agentd.d
+  template:
+    src: "{{ item }}.j2"
+    dest: "/etc/zabbix/zabbix_agentd.d/{{ item }}"
+    owner: zabbix
+    mode: 0666
+  notify: restart_zabbix_agent
+  with_items:
+    - interface-alias.conf
+
+- name: Enabling Zabbix service
+  service:
+    name: "{{ item }}"
+    state: started
+    enabled: yes
+  with_items:
+    - zabbix-agent
+
+- include_tasks: tools.yml

roles/zabbix/zabbix-agent/tasks/tools.yml

@@ -0,0 +1,25 @@
+- name: Some basic wrapper scripts for zabbix-sender
+  copy:
+    src: "scripts/{{ item }}"
+    dest: "/usr/lib/zabbix/{{ item }}"
+    mode: 0755
+  with_items:
+    - zabbix-hw-raid-check.sh
+    - zabbix-mdstat-check.sh
+    - zabbix-check-eth-settings.sh
+    - zabbix-check-iptables.sh
+    - zabbix-check-ro.sh
+
+- name: Ensuring we have some cron jobs for zabbix-sender wrapper scripts
+  cron:
+    name: "Zabbix sender wrapper script {{ item }}"
+    minute: "*/30"
+    job: "/usr/lib/zabbix/{{ item }}"
+    user: root
+  with_items:
+    - zabbix-hw-raid-check.sh
+    - zabbix-mdstat-check.sh
+    - zabbix-check-eth-settings.sh
+    - zabbix-check-iptables.sh
+    - zabbix-check-ro.sh
+

roles/zabbix/zabbix-agent/templates/interface-alias.conf.j2

@@ -0,0 +1,2 @@
+Alias=net.if.default.out:net.if.out[{{ ansible_default_ipv4.interface | default('eth0') }}]
+Alias=net.if.default.in:net.if.in[{{ ansible_default_ipv4.interface | default('eth0')}}]

roles/zabbix/zabbix-agent/templates/scripts/zabbix-conntrack-check.sh.j2

@@ -0,0 +1,4 @@
+#!/bin/bash
+zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k sys.net.ip_conntrack -o $(wc -l /proc/net/nf_conntrack|awk '{print $1}') > /dev/null
+
+

roles/zabbix/zabbix-agent/templates/zabbix_agentd.conf.j2

@@ -0,0 +1,19 @@
+Hostname={{ inventory_hostname }}
+Server={{ zabbix_server }}
+ServerActive={{ zabbix_server }}
+PidFile=/var/run/zabbix/zabbix_agentd.pid
+LogFile=/var/log/zabbix/zabbix_agentd.log
+LogFileSize=10
+EnableRemoteCommands=0
+Include=/etc/zabbix/zabbix_agentd.d/
+RefreshActiveChecks=180
+BufferSend=5
+BufferSize=100
+{% if zabbix_agent_tls %}
+# Settings for TLS/PSK between agent and proxy/server
+TLSAccept=psk
+TLSConnect=psk
+TLSPSKIdentity={{ zabbix_agent_tls_psk_identity }}
+TLSPSKFile=/etc/zabbix/zabbix_agent.psk
+{% endif %}
+

roles/zabbix/zabbix-agent/vars/CentOS-8.yml

@@ -0,0 +1,8 @@
+pkgs_list:
+  - policycoreutils-python-utils
+
+zabbix_pkgs_list:
+  - zabbix-agent
+  - zabbix-sender
+  - bc
+  - ncurses-compat-libs  # Needed for some megacli tools and raid monitoring checks through zabbix_sender

roles/zabbix/zabbix-agent/vars/Fedora.yml

@@ -0,0 +1,8 @@
+pkgs_list:
+  - policycoreutils-python-utils
+
+zabbix_pkgs_list:
+  - zabbix-agent
+  - zabbix # provides zabbix_sender
+  - bc
+  - ncurses-compat-libs  # Needed for some megacli tools and raid monitoring checks through zabbix_sender

roles/zabbix/zabbix-agent/vars/RedHat-8.yml

@@ -0,0 +1 @@
+CentOS-8.yml

roles/zabbix/zabbix-agent/vars/common.yml

@@ -0,0 +1,8 @@
+pkgs_list:
+  - libsemanage-python
+  - policycoreutils-python
+
+zabbix_pkgs_list:
+  - zabbix-agent
+  - zabbix-sender
+  - bc