zabbix: Updated SOP add zabbix montitoring releng compose
Signed-off-by: David Kirwan <davidkirwanirl@gmail.com>
This commit is contained in:
parent
3d9af9f4e1
commit
d54e61517c
1 changed files with 124 additions and 73 deletions
|
|
@ -100,99 +100,114 @@ Need at least one Zabbix check per cronjob. The Zabbix check should do the follo
|
|||
|
||||
|
||||
|
||||
=== Create a custom template
|
||||
Using the zabbix ansible role `community.zabbix.zabbix_template`, create a template:
|
||||
|
||||
----
|
||||
# Create template
|
||||
- name: Set API token
|
||||
tags: always
|
||||
set_fact:
|
||||
ansible_zabbix_auth_key: XXXXAPIKEYXXXX
|
||||
ansible_network_os: community.zabbix.zabbix
|
||||
ansible_connection: httpapi
|
||||
ansible_httpapi_port: 443
|
||||
ansible_httpapi_use_ssl: true
|
||||
ansible_httpapi_validate_certs: false
|
||||
ansible_host: ZABBIX_SERVER_HOSTNAME
|
||||
ansible_zabbix_url_path: "" # If Zabbix WebUI runs on non-default zabbix path (/),e.g. http://<FQDN>/zabbix
|
||||
|
||||
|
||||
# If implementing the template in the Zabbix GUI, can export the completed template with the following two
|
||||
- name: Get Zabbix template as JSON
|
||||
community.zabbix.zabbix_template_info:
|
||||
template_name: fedora releng compose cronjobs
|
||||
format: json
|
||||
omit_date: yes
|
||||
register: zabbix_template_json
|
||||
|
||||
- name: Write Zabbix templte to JSON file
|
||||
local_action:
|
||||
module: copy
|
||||
content: "{{ zabbix_template_json['template_json'] }}"
|
||||
dest: "roles/zabbix_server/files/zabbix_templates/releng_compose_cronjobs.json"
|
||||
|
||||
# Import the template JSON
|
||||
- name: Import Zabbix templates from JSON
|
||||
community.zabbix.zabbix_template:
|
||||
template_json: "{{ lookup('file', 'files/zabbix_templates/releng_compose_cronjobs.json') }}"
|
||||
state: present
|
||||
----
|
||||
|
||||
|
||||
=== Create a host group
|
||||
|
||||
----
|
||||
# Create host-group
|
||||
- name: Set API token
|
||||
tags: always
|
||||
set_fact:
|
||||
ansible_zabbix_auth_key: XXXXAPIKEYXXXX
|
||||
- name: Create host groups
|
||||
# set task level variables as we change ansible_connection plugin here
|
||||
community.zabbix.zabbix_group:
|
||||
state: present
|
||||
host_groups: "{{ item['hostgroup'] }}"
|
||||
with_items: "{{ zabbix_templates }}" # Hostgroups specific to an ansible group can be overridden in inventory/group_vars/group_name
|
||||
run_once: True
|
||||
tags:
|
||||
- zabbix_hostgroups
|
||||
vars:
|
||||
ansible_zabbix_auth_key: "{{ (env == 'staging')|ternary(zabbix_stg_apikey, zabbix_apikey) }}"
|
||||
ansible_network_os: community.zabbix.zabbix
|
||||
ansible_connection: httpapi
|
||||
ansible_httpapi_port: 443
|
||||
ansible_httpapi_use_ssl: true
|
||||
ansible_httpapi_validate_certs: false
|
||||
ansible_host: ZABBIX_SERVER_HOSTNAME
|
||||
ansible_zabbix_url_path: "" # If Zabbix WebUI runs on non-default zabbix path (/),e.g. http://<FQDN>/zabbix
|
||||
|
||||
- name: Create host groups
|
||||
# set task level variables as we change ansible_connection plugin here
|
||||
community.zabbix.zabbix_group:
|
||||
state: present
|
||||
host_groups:
|
||||
- fedora releng compose
|
||||
ansible_host: "{{ (env == 'staging')|ternary(zabbix_stg_hostname, zabbix_hostname) }}"
|
||||
ansible_zabbix_url_path: "" # If Zabbix WebUI runs on non-default (zabbix) path ,e.g. http://<FQDN>/zabbixeu
|
||||
----
|
||||
|
||||
|
||||
=== Add production releng_compose hosts to the Zabbix host group
|
||||
|
||||
----
|
||||
- name: Set API token
|
||||
tags: always
|
||||
set_fact:
|
||||
ansible_zabbix_auth_key: XXXXAPIKEYXXXX
|
||||
- name: Add hosts to hostgroups
|
||||
community.zabbix.zabbix_host:
|
||||
host_name: "{{ inventory_hostname }}"
|
||||
host_groups: "{{ item['hostgroup']}}"
|
||||
# link_templates: "{{ item['template'] }}" # We're adding the template to hostgroups in a seperate step, may not be required.
|
||||
force: false
|
||||
with_items: "{{ zabbix_templates }}"
|
||||
tags:
|
||||
- zabbix_add_hosts_to_hostgroups
|
||||
vars:
|
||||
ansible_zabbix_auth_key: "{{ (env == 'staging')|ternary(zabbix_stg_apikey, zabbix_apikey) }}"
|
||||
ansible_network_os: community.zabbix.zabbix
|
||||
ansible_connection: httpapi
|
||||
ansible_httpapi_port: 443
|
||||
ansible_httpapi_use_ssl: true
|
||||
ansible_httpapi_validate_certs: false
|
||||
ansible_host: ZABBIX_SERVER_HOSTNAME
|
||||
ansible_zabbix_url_path: "" # If Zabbix WebUI runs on non-default zabbix path (/),e.g. http://<FQDN>/zabbix
|
||||
ansible_host: "{{ (env == 'staging')|ternary(zabbix_stg_hostname, zabbix_hostname) }}"
|
||||
ansible_zabbix_url_path: "" # If Zabbix WebUI runs on non-default (zabbix) path ,e.g. http://<FQDN>/zabbixeu
|
||||
----
|
||||
|
||||
- name: Create host groups
|
||||
|
||||
=== Import a custom template
|
||||
Using the zabbix ansible role `community.zabbix.zabbix_template`, create a template:
|
||||
|
||||
Make sure to use JSON format. It might be best to use the Zabbix UI to configure initially, and then export the template. Make sure that the JSON template is minimised before importing back into Zabbix.
|
||||
|
||||
----
|
||||
#- name: Get Zabbix template as JSON
|
||||
# community.zabbix.zabbix_template_info:
|
||||
# template_name: fedora releng compose cronjobs
|
||||
# format: json
|
||||
# omit_date: yes
|
||||
# register: zabbix_template_json
|
||||
|
||||
#- name: Write Zabbix templte to JSON file
|
||||
# local_action:
|
||||
# module: copy
|
||||
# content: "{{ zabbix_template_json['template_json'] }}"
|
||||
# dest: "roles/zabbix_server/files/zabbix_templates/releng_compose_cronjobs.json"
|
||||
|
||||
- name: Import Zabbix templates from JSON
|
||||
community.zabbix.zabbix_template:
|
||||
template_json: "{{ lookup('file', item['template'] ) }}"
|
||||
state: present
|
||||
with_items: "{{ zabbix_templates }}" # Templates specific to an ansible group, can be overwridden in inventory/group_vars/group_name
|
||||
tags:
|
||||
- zabbix_templates
|
||||
vars:
|
||||
ansible_zabbix_auth_key: "{{ (env == 'staging')|ternary(zabbix_stg_apikey, zabbix_apikey) }}"
|
||||
ansible_network_os: community.zabbix.zabbix
|
||||
ansible_connection: httpapi
|
||||
ansible_httpapi_port: 443
|
||||
ansible_httpapi_use_ssl: true
|
||||
ansible_httpapi_validate_certs: false
|
||||
ansible_host: "{{ (env == 'staging')|ternary(zabbix_stg_hostname, zabbix_hostname) }}"
|
||||
ansible_zabbix_url_path: "" # If Zabbix WebUI runs on non-default (zabbix) path ,e.g. http://<FQDN>/zabbixeu
|
||||
----
|
||||
|
||||
=== Add template to host groups
|
||||
|
||||
----
|
||||
- name: Add templates to hosts
|
||||
community.zabbix.zabbix_host:
|
||||
host_name: "{{ item }}"
|
||||
host_groups:
|
||||
- fedora releng compose
|
||||
link_templates:
|
||||
- fedora releng compose cronjobs
|
||||
host_name: "{{ inventory_hostname }}"
|
||||
host_groups: "{{ item['hostgroup']}}"
|
||||
link_templates: "{{ item['template'] }}"
|
||||
force: false
|
||||
with_items:
|
||||
- compose-branched01.iad2.fedoraproject.org
|
||||
- compose-iot01.iad2.fedoraproject.org
|
||||
- compose-rawhide01.iad2.fedoraproject.org
|
||||
- compose-x86-01.iad2.fedoraproject.org
|
||||
with_items: "{{ zabbix_templates }}"
|
||||
tags:
|
||||
- zabbix_add_templates_to_hosts
|
||||
vars:
|
||||
ansible_zabbix_auth_key: "{{ (env == 'staging')|ternary(zabbix_stg_apikey, zabbix_apikey) }}"
|
||||
ansible_network_os: community.zabbix.zabbix
|
||||
ansible_connection: httpapi
|
||||
ansible_httpapi_port: 443
|
||||
ansible_httpapi_use_ssl: true
|
||||
ansible_httpapi_validate_certs: false
|
||||
ansible_host: "{{ (env == 'staging')|ternary(zabbix_stg_hostname, zabbix_hostname) }}"
|
||||
ansible_zabbix_url_path: "" # If Zabbix WebUI runs on non-default (zabbix) path ,e.g. http://<FQDN>/zabbixeu
|
||||
----
|
||||
|
||||
|
||||
|
|
@ -200,7 +215,7 @@ Using the zabbix ansible role `community.zabbix.zabbix_template`, create a templ
|
|||
|
||||
- Configure the type as zabbix agent active
|
||||
- Configure history to 7d for 1 week
|
||||
- Configure the resolution to 1m to check every minute
|
||||
- Configure the resolution to 60m to check every hour
|
||||
- Configure the key to match something like the following, changing the * to what ever the name of the cronjob is, eg rawhide
|
||||
|
||||
----
|
||||
|
|
@ -210,13 +225,13 @@ vfs.file.exists[/tmp/fedora-compose-*]
|
|||
|
||||
=== In this template create a trigger, one for each cronjob.
|
||||
|
||||
- Configure the trigger to 1 hour initially.
|
||||
- Configure the trigger to 8 hours.
|
||||
- Configure the severity to high
|
||||
- In this example the `fedora releng compose cronjobs` is the hostgroup that the hosts are members of, and have this template attached.
|
||||
- In this example the `releng_compose_cronjobs.json` is the name of the template, it makes it generic, and when the template is applied to a host, it gains the triggers and items contained in the template.
|
||||
- Configure the expression to something like the following, changing the * to what ever the name of the file in the key in the matching item
|
||||
|
||||
----
|
||||
last(/fedora releng compose cronjobs/vfs.file.exists[/tmp/fedora-compose-*])=1 and min(/fedora releng compose cronjobs/vfs.file.exists[/tmp/fedora-compose-*],1h)>0
|
||||
last(/releng_compose_cronjobs.json/vfs.file.exists[/tmp/fedora-compose-branched])=1 and min(/releng_compose_cronjobs.json/vfs.file.exists[/tmp/fedora-compose-branched],8h)>0
|
||||
----
|
||||
|
||||
|
||||
|
|
@ -229,6 +244,42 @@ last(/fedora releng compose cronjobs/vfs.file.exists[/tmp/fedora-compose-*])=1 a
|
|||
- If file exists, assume cron is running and if file exists for more than a set period, assume the cron job is stalled.
|
||||
|
||||
|
||||
=== Fedora Ansible Group Vars for Zabbix
|
||||
The following var structure is required to configure this new `zabbix_template` role. See the example structure in the `inventory/groups/releng_compose` for production:
|
||||
|
||||
----
|
||||
zabbix_templates:
|
||||
- group: "releng_compose"
|
||||
template: "releng_compose_cronjobs.json"
|
||||
hostgroup: "fedora releng compose"
|
||||
----
|
||||
|
||||
And for staging:
|
||||
|
||||
----
|
||||
zabbix_templates: "{{ [] }}"
|
||||
----
|
||||
|
||||
Currently we do not run composes in staging. So I've not activated this role on the staging machines. Ordinarially, make sure to add the same vars in prod and staging environments.
|
||||
|
||||
Each element in the list, should be used to link a single Zabbix template to a Zabbix hostgroup. group parameter is not currently used, but it should be set to the Ansible group name for documentation purposes.
|
||||
|
||||
To use this role going forward:
|
||||
|
||||
- Add template.json files to roles/zabbix/zabbix_templates/files
|
||||
- Add a `zabbix_templates` var to the `inventory/groups/groupname` file that matches the ansible group
|
||||
- Import the role in the template corresponding with this ansible group eg:
|
||||
|
||||
----
|
||||
# playbooks/groups/releng-compose.yml:33
|
||||
roles:
|
||||
...
|
||||
- zabbix/zabbix_templates
|
||||
...
|
||||
----
|
||||
|
||||
|
||||
|
||||
=== Future Work
|
||||
Replace these custom releng monitoring things with zabbix.
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue