2013-11-21 21:52:05 +00:00
|
|
|
#
|
2016-08-08 19:36:31 +00:00
|
|
|
# This playbook lets you safely reboot a virthost and all it's guests.
|
|
|
|
#
|
2013-03-04 22:37:13 +00:00
|
|
|
# requires --extra-vars="target=somevhost fqdn"
|
2015-07-22 21:52:30 +00:00
|
|
|
# Might add nodns=true or nonagios=true to the extra vars
|
2012-11-21 21:09:19 +00:00
|
|
|
|
|
|
|
#General overview:
|
2012-11-21 17:56:10 +00:00
|
|
|
# talk to the vhost
|
|
|
|
# get back list of instances
|
|
|
|
# add each of their hostnames to an addhoc group
|
|
|
|
# halt each of them in a second play
|
2012-11-21 21:09:19 +00:00
|
|
|
# wait for them to die
|
2012-11-21 17:56:10 +00:00
|
|
|
# third play, reboot the vhost
|
|
|
|
# wait for vhost to come back
|
2012-11-21 21:09:19 +00:00
|
|
|
|
2016-08-08 19:36:31 +00:00
|
|
|
# TODO: Figure out how to compare virt info pre and post boot.
|
2012-11-21 17:56:10 +00:00
|
|
|
|
|
|
|
- name: find instances
|
2013-11-21 21:52:05 +00:00
|
|
|
hosts: "{{ target }}"
|
|
|
|
gather_facts: False
|
2012-11-21 17:56:10 +00:00
|
|
|
user: root
|
|
|
|
|
|
|
|
tasks:
|
|
|
|
- name: get list of guests
|
2018-01-05 12:25:23 +00:00
|
|
|
virt: command=list_vms state=running
|
2012-11-21 17:56:10 +00:00
|
|
|
register: vmlist
|
2012-11-21 17:59:07 +00:00
|
|
|
|
2013-11-21 21:52:05 +00:00
|
|
|
# - name: get info on guests (prereboot)
|
|
|
|
# virt: command=info
|
|
|
|
# register: vminfo_pre
|
|
|
|
|
2012-11-21 21:09:19 +00:00
|
|
|
- name: add them to myvms_new group
|
2013-11-21 22:07:02 +00:00
|
|
|
local_action: add_host hostname={{ item }} groupname=myvms_new
|
2016-11-29 01:45:38 +00:00
|
|
|
with_items: "{{ vmlist.list_vms }}"
|
2012-11-21 17:56:10 +00:00
|
|
|
|
2014-12-15 21:36:50 +00:00
|
|
|
# Call out to another playbook. Disable any proxies that may live here
|
2023-05-19 09:53:27 -07:00
|
|
|
- import_playbook: update-proxy-dns.yml
|
|
|
|
vars:
|
|
|
|
status: enable
|
|
|
|
proxies: myvms_new:&proxies
|
2017-01-24 20:00:30 +00:00
|
|
|
when: nodns is not defined or not "true" in nodns
|
2014-12-15 21:36:50 +00:00
|
|
|
|
2012-11-21 17:56:10 +00:00
|
|
|
- name: halt instances
|
|
|
|
hosts: myvms_new
|
|
|
|
user: root
|
2013-11-21 21:52:05 +00:00
|
|
|
gather_facts: False
|
2013-01-28 21:27:18 +00:00
|
|
|
serial: 1
|
2012-11-21 17:56:10 +00:00
|
|
|
|
|
|
|
tasks:
|
2023-05-15 18:19:18 -07:00
|
|
|
# - name: delegate to on-control01 for ocp4 cluster
|
|
|
|
# set_fact:
|
|
|
|
# os_delegate_via: os-control01
|
|
|
|
# when: inventory_hostname in groups['ocp']
|
|
|
|
#
|
|
|
|
# - name: delegate to on-control01 for ocp4 cluster (stg)
|
|
|
|
# set_fact:
|
|
|
|
# os_delegate_via: os-control01.stg
|
|
|
|
# when: inventory_hostname in groups['ocp_stg']
|
|
|
|
#
|
|
|
|
# - name: drain OS node if necessary
|
|
|
|
# command: oc adm drain {{inventory_hostname }} --ignore-daemonsets --delete-local-data
|
|
|
|
# delegate_to: "{{os_delegate_via}}{{env_suffix}}.iad2.fedoraproject.org"
|
|
|
|
# when: inventory_hostname.startswith(('ocp', 'worker')) and hostvars[inventory_hostname].datacenter == 'iad2'
|
|
|
|
#
|
2013-11-21 21:52:05 +00:00
|
|
|
- name: schedule regular host downtime
|
2015-05-04 14:38:03 +00:00
|
|
|
nagios: action=downtime minutes=30 service=host host={{ inventory_hostname_short }}{{ env_suffix }}
|
2020-06-14 14:14:31 -07:00
|
|
|
delegate_to: noc01.iad2.fedoraproject.org
|
2013-05-07 21:50:45 +00:00
|
|
|
ignore_errors: true
|
2015-12-15 20:25:54 +00:00
|
|
|
when: nonagios is not defined or not nonagios
|
2013-01-28 21:27:18 +00:00
|
|
|
|
2022-03-29 10:15:31 -07:00
|
|
|
- name: shutdown vms
|
|
|
|
virt: command=shutdown name={{ inventory_hostname }}
|
2012-11-21 18:45:20 +00:00
|
|
|
ignore_errors: true
|
2022-03-29 10:15:31 -07:00
|
|
|
delegate_to: "{{ target }}"
|
2012-11-21 18:45:20 +00:00
|
|
|
|
2013-11-21 10:24:17 +00:00
|
|
|
- name: wait for the whole set to die.
|
|
|
|
hosts: myvms_new
|
2013-11-21 21:52:05 +00:00
|
|
|
gather_facts: False
|
2013-11-21 10:24:17 +00:00
|
|
|
user: root
|
|
|
|
|
|
|
|
tasks:
|
2012-11-21 18:45:20 +00:00
|
|
|
- name: wait for them to die
|
2013-11-21 21:52:05 +00:00
|
|
|
local_action: wait_for port=22 delay=30 timeout=300 state=stopped host={{ inventory_hostname }}
|
2013-11-21 10:24:17 +00:00
|
|
|
|
2012-11-21 18:22:03 +00:00
|
|
|
- name: reboot vhost
|
2013-11-21 21:52:05 +00:00
|
|
|
hosts: "{{ target }}"
|
|
|
|
gather_facts: False
|
2012-11-21 18:22:03 +00:00
|
|
|
user: root
|
|
|
|
|
|
|
|
tasks:
|
2013-03-01 16:24:16 +00:00
|
|
|
- name: tell nagios to shush
|
2015-05-04 14:38:03 +00:00
|
|
|
nagios: action=downtime minutes=60 service=host host={{ inventory_hostname_short }}{{ env_suffix }}
|
2020-06-14 14:14:31 -07:00
|
|
|
delegate_to: noc01.iad2.fedoraproject.org
|
2013-05-07 21:50:45 +00:00
|
|
|
ignore_errors: true
|
2015-12-15 20:25:54 +00:00
|
|
|
when: nonagios is not defined or not nonagios
|
2013-01-28 21:27:18 +00:00
|
|
|
|
2013-11-21 21:52:05 +00:00
|
|
|
- name: reboot the virthost
|
2019-04-09 17:50:37 +00:00
|
|
|
reboot:
|
2012-11-21 18:45:20 +00:00
|
|
|
|
2013-12-19 20:25:13 +00:00
|
|
|
- name: wait for libvirtd to come back on the virthost
|
2023-03-27 15:24:31 -07:00
|
|
|
wait_for: path=/run/libvirt/libvirt-sock state=present
|
2013-12-19 20:25:13 +00:00
|
|
|
|
2012-11-21 18:58:50 +00:00
|
|
|
- name: look up vmlist
|
2013-11-21 21:52:05 +00:00
|
|
|
virt: command=list_vms
|
2012-11-21 18:45:20 +00:00
|
|
|
register: newvmlist
|
2012-11-21 21:09:19 +00:00
|
|
|
|
2018-10-03 03:34:03 +00:00
|
|
|
- name: add them to myvms_postreboot group
|
|
|
|
local_action: add_host hostname={{ item }} groupname=myvms_postreboot
|
|
|
|
with_items: "{{ newvmlist.list_vms }}"
|
|
|
|
|
2018-11-06 21:22:45 +00:00
|
|
|
# - name: sync time
|
|
|
|
# command: ntpdate -u 1.rhel.pool.ntp.org
|
2013-11-21 21:52:05 +00:00
|
|
|
|
2014-10-01 01:53:33 +00:00
|
|
|
- name: tell nagios to unshush
|
2015-05-04 14:38:03 +00:00
|
|
|
nagios: action=unsilence service=host host={{ inventory_hostname_short }}{{ env_suffix }}
|
2020-06-14 14:14:31 -07:00
|
|
|
delegate_to: noc01.iad2.fedoraproject.org
|
2014-10-01 01:53:33 +00:00
|
|
|
ignore_errors: true
|
2015-12-15 20:25:54 +00:00
|
|
|
when: nonagios is not defined or not nonagios
|
2014-10-01 01:53:33 +00:00
|
|
|
|
2018-10-03 02:25:38 +00:00
|
|
|
- name: post reboot tasks
|
2018-10-03 03:34:03 +00:00
|
|
|
hosts: myvms_postreboot
|
2018-10-03 02:25:38 +00:00
|
|
|
user: root
|
|
|
|
gather_facts: False
|
|
|
|
serial: 1
|
|
|
|
|
2018-10-03 03:21:14 +00:00
|
|
|
tasks:
|
2020-05-31 13:33:53 -04:00
|
|
|
|
2022-03-29 09:47:37 -07:00
|
|
|
- name: delegate to on-control01 for ocp4 cluster
|
|
|
|
set_fact:
|
|
|
|
os_delegate_via: os-control01
|
2022-03-29 09:55:51 -07:00
|
|
|
when: inventory_hostname in groups['ocp']
|
|
|
|
|
|
|
|
- name: delegate to on-control01 for ocp4 cluster (stg)
|
|
|
|
set_fact:
|
|
|
|
os_delegate_via: os-control01.stg
|
|
|
|
when: inventory_hostname in groups['ocp_stg']
|
2022-03-29 09:47:37 -07:00
|
|
|
|
2020-05-31 13:33:53 -04:00
|
|
|
- name: Add back to openshift
|
|
|
|
command: oc adm uncordon {{inventory_hostname}}
|
|
|
|
delegate_to: "{{os_delegate_via}}{{env_suffix}}.iad2.fedoraproject.org"
|
2022-11-28 13:08:01 -08:00
|
|
|
when: inventory_hostname.startswith(('ocp', 'worker')) and hostvars[inventory_hostname].datacenter == 'iad2'
|
2018-10-03 02:25:38 +00:00
|
|
|
|
2019-05-21 15:56:08 +00:00
|
|
|
- name: restart gssproxy if we rebooted a ipa server
|
|
|
|
service: name=gssproxy state=restarted
|
|
|
|
when: inventory_hostname.startswith('ipa')
|
|
|
|
|
|
|
|
- name: restart rabbitmq if we rebooted a rabbit server
|
|
|
|
service: name=rabbitmq-server state=restarted
|
|
|
|
when: inventory_hostname.startswith('rabbitmq')
|
|
|
|
|
2014-12-15 21:36:50 +00:00
|
|
|
# Call out to that dns playbook. Put proxies back in now that they're back
|
2023-05-19 09:53:27 -07:00
|
|
|
- import_playbook: update-proxy-dns.yml
|
|
|
|
vars:
|
|
|
|
status: enable
|
|
|
|
proxies: myvms_new:&proxies
|
2015-12-15 20:25:54 +00:00
|
|
|
when: nodns is not defined or not nodns
|
2014-12-15 21:36:50 +00:00
|
|
|
|
2013-12-20 18:17:02 +00:00
|
|
|
vars_files:
|
|
|
|
- /srv/web/infra/ansible/vars/global.yml
|
2014-01-06 18:22:18 +00:00
|
|
|
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
|
2013-12-20 18:17:02 +00:00
|
|
|
|
2013-11-21 21:52:05 +00:00
|
|
|
# - name: get info on guests (postreboot)
|
|
|
|
# virt: command=info
|
|
|
|
# register: vminfo_post
|
|
|
|
|