diff --git a/playbooks/groups/bodhi-backend.yml b/playbooks/groups/bodhi-backend.yml index 9c838ec7fc..72a23b50e9 100644 --- a/playbooks/groups/bodhi-backend.yml +++ b/playbooks/groups/bodhi-backend.yml @@ -111,6 +111,11 @@ warning: 10 critical: 100 + - role: zabbix/zabbix-agent + zabbix_server: apache + zabbix_server_ip: apache + when: env == "staging" + tasks: - name: create secondary volume dir for stg bodhi file: dest=/mnt/koji/vol state=directory owner=apache group=apache mode=0755 diff --git a/roles/zabbix/zabbix-agent/defaults/main.yml b/roles/zabbix/zabbix-agent/defaults/main.yml new file mode 100644 index 0000000000..f068412946 --- /dev/null +++ b/roles/zabbix/zabbix-agent/defaults/main.yml @@ -0,0 +1,8 @@ +# Defaults variables for role zabbix-agent +zabbix_server: zabbix01.stg.iad2.fedoraproject.org + +# TLS/PSK settings to encrypt between agent and proxy/server +zabbix_agent_tls: False +zabbix_agent_tls_psk: # gen with `openssl rand -hex 32` and also known by zabbix server +zabbix_agent_tls_psk_identity: # Whatever makes sense to you and known by zabbix server + diff --git a/roles/zabbix/zabbix-agent/files/scripts/zabbix-check-eth-settings.sh b/roles/zabbix/zabbix-agent/files/scripts/zabbix-check-eth-settings.sh new file mode 100644 index 0000000000..ea7c5529f9 --- /dev/null +++ b/roles/zabbix/zabbix-agent/files/scripts/zabbix-check-eth-settings.sh @@ -0,0 +1,18 @@ +#!/bin/bash +export PATH=$PATH:/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin +uname -a|grep -q xen >/dev/null +if [ "$?" -eq "0" ] ;then + eth_dev=p$(ip route|grep default|awk '{print $5}') +else + eth_dev=$(ip route|grep default|awk '{print $5}') +fi + +lsmod |grep -q -E 'virtio_net|xen_net' +if [ "$?" -eq "1" ] ;then + zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k eth_dev.speed -o $(ethtool $eth_dev|grep Speed|awk '{print $2}'|tr -d [:alpha:]|tr -d '/') >/dev/null + zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k eth_dev.duplex -o $(ethtool $eth_dev|grep Duplex|awk '{print $2}') >/dev/null +fi + + + + diff --git a/roles/zabbix/zabbix-agent/files/scripts/zabbix-check-iptables.sh b/roles/zabbix/zabbix-agent/files/scripts/zabbix-check-iptables.sh new file mode 100644 index 0000000000..e709fb4a5d --- /dev/null +++ b/roles/zabbix/zabbix-agent/files/scripts/zabbix-check-iptables.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# called by Zabbix to see if iptables is running +PATH=$PATH:/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin + +if [ $( iptables -n -L INPUT|wc -l ) -gt 6 ] ; then + iptablesstatus="0" +else + iptablesstatus="1" +fi + +zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k net.iptables.status -o $iptablesstatus >/dev/null diff --git a/roles/zabbix/zabbix-agent/files/scripts/zabbix-check-ro.sh b/roles/zabbix/zabbix-agent/files/scripts/zabbix-check-ro.sh new file mode 100644 index 0000000000..6951e9ec7d --- /dev/null +++ b/roles/zabbix/zabbix-agent/files/scripts/zabbix-check-ro.sh @@ -0,0 +1,7 @@ +#!/bin/bash +logfile="/var/log/centos-ro-devices" + +ro_mounted_devices=$(awk '$4 ~ "^ro[,$]" && $3 !~ "(squashfs|iso9660|tmpfs|nfs)" {print $0}' /proc/mounts | wc -l) +zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k devices.ro -o $ro_mounted_devices >/dev/null +echo "=== $(date) === Read-Only devices on this system $(hostname)" >> $logfile +awk '$4 ~ "^ro[,$]" && $3 !~ "(squashfs|iso9660|tmpfs|nfs)" {print $0} ' /proc/mounts >> $logfile diff --git a/roles/zabbix/zabbix-agent/files/scripts/zabbix-hw-raid-check.sh b/roles/zabbix/zabbix-agent/files/scripts/zabbix-hw-raid-check.sh new file mode 100644 index 0000000000..b187fb8256 --- /dev/null +++ b/roles/zabbix/zabbix-agent/files/scripts/zabbix-hw-raid-check.sh @@ -0,0 +1,77 @@ +#!/bin/bash +PATH=$PATH:/usr/local/bin:/sbin:/usr/sbin/ + +function init_log() { +logfile=/var/log/centos-hw-raid.log +echo "=========================================================" > $logfile +echo " CentOS Hardware Raid check - $(date +%Y%m%d-%H%M)" >> $logfile +echo "=========================================================" >> $logfile +} + +function 3w_xxxx_check() { +echo "3ware controller found .. launching raid check" >> $logfile +for controller in $(tw_cli show |grep ^c|awk '{print $1}') ; +do + for disk in $(tw_cli /${controller} show |grep ^p|awk '{print $1}') ; + do + tw_cli /${controller} show |grep ^${disk}|egrep -q 'OK|NOT-PRESENT' + if [ "$?" -ne "0" ] ;then + tw_cli /${controller} show >> $logfile + zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k hwraid.3ware -o 1 >/dev/null + exit 1 + else + echo "3ware controller ${controller} / array ${array} status : OK" >> $logfile + zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k hwraid.3ware -o 0 >/dev/null + fi + done +done +} + +function 3w_9xxx_check() { + 3w_xxxx_check +} + +function arcmsr_check() { +echo "ARECA controller found .. launching raid check" >> $logfile +for array in $(areca-cli rsf info|egrep -v 'Name|=|GuiErr'|awk '{print $1}') ; +do + areca-cli rsf info raid=${array}|grep -q Normal + if [ "$?" -ne "0" ] ;then + areca-cli rsf info raid=${array} >> $logfile + zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k hwraid.arcmsr -o 1 >/dev/null + exit 1 + else + echo "Areca array ${array} status : OK" >> $logfile + zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k hwraid.arcmsr -o 0 >/dev/null + fi +done +} + +function megaraid_sas_check() { +echo "Megaraid_sas controller found .. launching raid check" >> $logfile +for LDid in $(/opt/MegaRAID/MegaCli/MegaCli64 -LDInfo -Lall -aALL|grep "Virtual Drive:"|awk '{print $3}') ; +do + /opt/MegaRAID/MegaCli/MegaCli64 -LDInfo -L${LDid} -aALL|grep -q Optimal + if [ "$?" -ne "0" ] ;then + /opt/MegaRAID/MegaCli/MegaCli64 -ShowSummary -aALL >>$logfile + zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k hwraid.megaraid -o 1 >/dev/null + exit 1 + else + echo "Megaraid_sas array ${array} status : OK" >> $logfile + /opt/MegaRAID/MegaCli/MegaCli64 -ShowSummary -aALL >>$logfile + zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k hwraid.megaraid -o 0 >/dev/null + fi +done +} + +init_log + +# Ensuring sg kmod is loaded, as needed by those tools ... +lsmod|grep -q sg || modprobe sg + +for kmod in 3w_9xxx 3w_xxxx arcmsr megaraid_sas; do + /sbin/lsmod |grep -q ${kmod} + if [ "$?" = "0" ];then + ${kmod}_check + fi +done diff --git a/roles/zabbix/zabbix-agent/files/scripts/zabbix-mdstat-check.sh b/roles/zabbix/zabbix-agent/files/scripts/zabbix-mdstat-check.sh new file mode 100644 index 0000000000..0134ca83bc --- /dev/null +++ b/roles/zabbix/zabbix-agent/files/scripts/zabbix-mdstat-check.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +grep md /proc/mdstat >/dev/null 2>&1 +if [ $? = "0" ] ;then + for mddev in $( grep md /proc/mdstat |awk '{print $1}') ; + do + md_count=$(/sbin/mdadm --detail /dev/${mddev}|grep Failed|cut -f 2 -d ":"|tr -d [:blank:]) + if [ $md_count > "0" ] ;then + zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k mdstat.failed -o $md_count >/dev/null + exit + else + zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k mdstat.failed -o 0 >/dev/null + fi + done +fi + diff --git a/roles/zabbix/zabbix-agent/files/selinux/8/centos-zabbix-agent.pp b/roles/zabbix/zabbix-agent/files/selinux/8/centos-zabbix-agent.pp new file mode 100644 index 0000000000..a79a0fa2e6 Binary files /dev/null and b/roles/zabbix/zabbix-agent/files/selinux/8/centos-zabbix-agent.pp differ diff --git a/roles/zabbix/zabbix-agent/files/selinux/8/centos-zabbix-agent.te b/roles/zabbix/zabbix-agent/files/selinux/8/centos-zabbix-agent.te new file mode 100644 index 0000000000..6533b045b2 --- /dev/null +++ b/roles/zabbix/zabbix-agent/files/selinux/8/centos-zabbix-agent.te @@ -0,0 +1,20 @@ +policy_module(centos-zabbix-agent, 1.1) + +require{ + type zabbix_agent_t; + type zabbix_t; + type ping_t; + type zabbix_tmp_t; + class process setrlimit; +} + +allow ping_t zabbix_tmp_t:file read_file_perms; +allow ping_t zabbix_t:tcp_socket { read write }; +allow zabbix_agent_t self:process setrlimit; + +kernel_read_network_state(zabbix_agent_t) +domain_read_all_domains_state(zabbix_agent_t) +dev_read_sysfs(zabbix_agent_t) +corenet_tcp_connect_all_ports(zabbix_agent_t) + + diff --git a/roles/zabbix/zabbix-agent/handlers/main.yml b/roles/zabbix/zabbix-agent/handlers/main.yml new file mode 100644 index 0000000000..ce03f98b54 --- /dev/null +++ b/roles/zabbix/zabbix-agent/handlers/main.yml @@ -0,0 +1,6 @@ +- name: restart_zabbix_agent + service: name=zabbix-agent state=restarted + +- name: reload custom selinux files + shell: /usr/sbin/semodule -u "/etc/selinux/centos/centos-zabbix-agent.pp" + when: ansible_selinux.mode == "enforcing" diff --git a/roles/zabbix/zabbix-agent/meta/main.yml b/roles/zabbix/zabbix-agent/meta/main.yml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/roles/zabbix/zabbix-agent/tasks/main.yml b/roles/zabbix/zabbix-agent/tasks/main.yml new file mode 100644 index 0000000000..7bfb2627f0 --- /dev/null +++ b/roles/zabbix/zabbix-agent/tasks/main.yml @@ -0,0 +1,72 @@ +- name: Importing specific distro variables + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yml" + - "{{ ansible_distribution }}.yml" + - common.yml + +- name: Installing Zabbix packages + yum: + name: "{{ zabbix_pkgs_list }}" + state: latest + register: pkg_install + +- name: Installing selinux dependency packages + yum: + name: "{{ pkgs_list }}" + state: installed + tags: + - packages + +- name: Distributing custom selinux policies + copy: + src: "selinux/{{ ansible_distribution_version[0] }}/{{ item }}" + dest: "/etc/selinux/centos/{{ item }}" + register: sepolicy + with_items: + - centos-zabbix-agent.pp + +- name: reload custom selinux files + shell: /usr/sbin/semodule -u "/etc/selinux/centos/centos-zabbix-agent.pp" + when: ansible_selinux.status == "enabled" and sepolicy.changed + +- name: Allowing zabbix to connect to network resources + seboolean: + name: zabbix_can_network + persistent: yes + state: yes + when: ansible_selinux.status == "enabled" + +- name: Configuring Zabbix agentd + template: + src: zabbix_agentd.conf.j2 + dest: /etc/zabbix/zabbix_agentd.conf + notify: restart_zabbix_agent + +- name: Ensuring we have a directory to put zabbix scripts + file: + state: directory + path: /usr/lib/zabbix + mode: 0770 + owner: zabbix + group: zabbix + +- name: Adding some other conf files under zabbix_agentd.d + template: + src: "{{ item }}.j2" + dest: "/etc/zabbix/zabbix_agentd.d/{{ item }}" + owner: zabbix + mode: 0666 + notify: restart_zabbix_agent + with_items: + - interface-alias.conf + +- name: Enabling Zabbix service + service: + name: "{{ item }}" + state: started + enabled: yes + with_items: + - zabbix-agent + +- include_tasks: tools.yml diff --git a/roles/zabbix/zabbix-agent/tasks/tools.yml b/roles/zabbix/zabbix-agent/tasks/tools.yml new file mode 100644 index 0000000000..999fcc6502 --- /dev/null +++ b/roles/zabbix/zabbix-agent/tasks/tools.yml @@ -0,0 +1,25 @@ +- name: Some basic wrapper scripts for zabbix-sender + copy: + src: "scripts/{{ item }}" + dest: "/usr/lib/zabbix/{{ item }}" + mode: 0755 + with_items: + - zabbix-hw-raid-check.sh + - zabbix-mdstat-check.sh + - zabbix-check-eth-settings.sh + - zabbix-check-iptables.sh + - zabbix-check-ro.sh + +- name: Ensuring we have some cron jobs for zabbix-sender wrapper scripts + cron: + name: "Zabbix sender wrapper script {{ item }}" + minute: "*/30" + job: "/usr/lib/zabbix/{{ item }}" + user: root + with_items: + - zabbix-hw-raid-check.sh + - zabbix-mdstat-check.sh + - zabbix-check-eth-settings.sh + - zabbix-check-iptables.sh + - zabbix-check-ro.sh + diff --git a/roles/zabbix/zabbix-agent/templates/interface-alias.conf.j2 b/roles/zabbix/zabbix-agent/templates/interface-alias.conf.j2 new file mode 100644 index 0000000000..656ff19c46 --- /dev/null +++ b/roles/zabbix/zabbix-agent/templates/interface-alias.conf.j2 @@ -0,0 +1,2 @@ +Alias=net.if.default.out:net.if.out[{{ ansible_default_ipv4.interface | default('eth0') }}] +Alias=net.if.default.in:net.if.in[{{ ansible_default_ipv4.interface | default('eth0')}}] diff --git a/roles/zabbix/zabbix-agent/templates/scripts/zabbix-conntrack-check.sh.j2 b/roles/zabbix/zabbix-agent/templates/scripts/zabbix-conntrack-check.sh.j2 new file mode 100644 index 0000000000..f1805ce868 --- /dev/null +++ b/roles/zabbix/zabbix-agent/templates/scripts/zabbix-conntrack-check.sh.j2 @@ -0,0 +1,4 @@ +#!/bin/bash +zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k sys.net.ip_conntrack -o $(wc -l /proc/net/nf_conntrack|awk '{print $1}') > /dev/null + + diff --git a/roles/zabbix/zabbix-agent/templates/zabbix_agentd.conf.j2 b/roles/zabbix/zabbix-agent/templates/zabbix_agentd.conf.j2 new file mode 100644 index 0000000000..3d2f4e5e8e --- /dev/null +++ b/roles/zabbix/zabbix-agent/templates/zabbix_agentd.conf.j2 @@ -0,0 +1,19 @@ +Hostname={{ inventory_hostname }} +Server={{ zabbix_server }} +ServerActive={{ zabbix_server }} +PidFile=/var/run/zabbix/zabbix_agentd.pid +LogFile=/var/log/zabbix/zabbix_agentd.log +LogFileSize=10 +EnableRemoteCommands=0 +Include=/etc/zabbix/zabbix_agentd.d/ +RefreshActiveChecks=180 +BufferSend=5 +BufferSize=100 +{% if zabbix_agent_tls %} +# Settings for TLS/PSK between agent and proxy/server +TLSAccept=psk +TLSConnect=psk +TLSPSKIdentity={{ zabbix_agent_tls_psk_identity }} +TLSPSKFile=/etc/zabbix/zabbix_agent.psk +{% endif %} + diff --git a/roles/zabbix/zabbix-agent/vars/CentOS-8.yml b/roles/zabbix/zabbix-agent/vars/CentOS-8.yml new file mode 100644 index 0000000000..045ea43ebd --- /dev/null +++ b/roles/zabbix/zabbix-agent/vars/CentOS-8.yml @@ -0,0 +1,8 @@ +pkgs_list: + - policycoreutils-python-utils + +zabbix_pkgs_list: + - zabbix-agent + - zabbix-sender + - bc + - ncurses-compat-libs # Needed for some megacli tools and raid monitoring checks through zabbix_sender diff --git a/roles/zabbix/zabbix-agent/vars/Fedora.yml b/roles/zabbix/zabbix-agent/vars/Fedora.yml new file mode 100644 index 0000000000..1a24c05ef6 --- /dev/null +++ b/roles/zabbix/zabbix-agent/vars/Fedora.yml @@ -0,0 +1,8 @@ +pkgs_list: + - policycoreutils-python-utils + +zabbix_pkgs_list: + - zabbix-agent + - zabbix # provides zabbix_sender + - bc + - ncurses-compat-libs # Needed for some megacli tools and raid monitoring checks through zabbix_sender diff --git a/roles/zabbix/zabbix-agent/vars/RedHat-8.yml b/roles/zabbix/zabbix-agent/vars/RedHat-8.yml new file mode 120000 index 0000000000..a38d21ef87 --- /dev/null +++ b/roles/zabbix/zabbix-agent/vars/RedHat-8.yml @@ -0,0 +1 @@ +CentOS-8.yml \ No newline at end of file diff --git a/roles/zabbix/zabbix-agent/vars/common.yml b/roles/zabbix/zabbix-agent/vars/common.yml new file mode 100644 index 0000000000..5c68df415b --- /dev/null +++ b/roles/zabbix/zabbix-agent/vars/common.yml @@ -0,0 +1,8 @@ +pkgs_list: + - libsemanage-python + - policycoreutils-python + +zabbix_pkgs_list: + - zabbix-agent + - zabbix-sender + - bc