First take on a datanommer upgrade playbook.

playbooks/manual/upgrade/datanommer.yml

@@ -0,0 +1,157 @@
+- name: push packages out
+  hosts:
+  - badges-backend
+  - badges-backend-stg
+  - datagrepper
+  - datagrepper-stg
+  - notifs-frontend
+  - notifs-frontend-stg
+  - busgateway
+  - busgateway-stg
+  user: root
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "/srv/private/ansible/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+  vars:
+    testing: False
+  handlers:
+  - include: "{{ handlers }}/restart_services.yml"
+
+  tasks:
+  - name: clean all metadata {%if testing%}(with infrastructure-testing on){%endif%}
+    command: yum clean all {%if testing%} --enablerepo=infrastructure-testing {%endif%}
+    always_run: yes
+  - name: yum update datanommer packages from main repo
+    yum: name="*datanommer*" state=latest
+    when: not testing
+  - name: yum update datanommer packages from testing repo
+    yum: name="*datanommer*" state=latest enablerepo=infrastructure-testing
+    when: testing
+
+- name: verify the badges backend and stop it
+  hosts: badges-backend;badges-backend-stg
+  user: root
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "/srv/private/ansible/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+  handlers:
+  - include: "{{ handlers }}/restart_services.yml"
+  pre_tasks:
+  - name: tell nagios to shush
+    nagios: action=downtime minutes=120 service=host host={{ inventory_hostname }}
+    delegate_to: noc01.phx2.fedoraproject.org
+    ignore_errors: true
+  roles:
+  - badges/backend
+  tasks:
+  - service: name="fedmsg-hub" state=stopped
+
+- name: verify the datagrepper frontend and stop it
+  hosts: datagrepper;datagrepper-stg
+  user: root
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "/srv/private/ansible/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+  handlers:
+  - include: "{{ handlers }}/restart_services.yml"
+  pre_tasks:
+  - name: tell nagios to shush
+    nagios: action=downtime minutes=120 service=host host={{ inventory_hostname }}
+    delegate_to: noc01.phx2.fedoraproject.org
+    ignore_errors: true
+  roles:
+  - datagrepper
+  tasks:
+  - service: name="httpd" state=stopped
+
+- name: verify the notifs frontend and stop it
+  hosts: notifs-frontend;notifs-frontend-stg
+  user: root
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "/srv/private/ansible/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+  handlers:
+  - include: "{{ handlers }}/restart_services.yml"
+  pre_tasks:
+  - name: tell nagios to shush
+    nagios: action=downtime minutes=120 service=host host={{ inventory_hostname }}
+    delegate_to: noc01.phx2.fedoraproject.org
+    ignore_errors: true
+  roles:
+  - notifs/frontend
+  tasks:
+  - service: name="httpd" state=stopped
+
+- name: verify the datanommer backend, stop it, and then upgrade the db
+  hosts: busgateway;busgateway-stg
+  user: root
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "/srv/private/ansible/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+  handlers:
+  - include: "{{ handlers }}/restart_services.yml"
+  pre_tasks:
+  - name: tell nagios to shush
+    nagios: action=downtime minutes=120 service=host host={{ inventory_hostname }}
+    delegate_to: noc01.phx2.fedoraproject.org
+    ignore_errors: true
+  roles:
+  - fedmsg/datanommer
+  tasks:
+  - name: Stop the notification backend
+    service: name="fedmsg-hub" state=stopped
+
+  - name: Upgrade the database
+    command: /usr/bin/alembic upgrade head
+    args:
+      chdir: /usr/share/datanommer.models/
+
+  - name: And... start the backend again
+    service: name="fedmsg-hub" state=started
+
+  post_tasks:
+  - name: tell nagios to unshush
+    nagios: action=unsilence service=host host={{ inventory_hostname }}
+    delegate_to: noc01.phx2.fedoraproject.org
+    ignore_errors: true
+
+- name: restart the frontend pieces (fmn.web and datagrepper)
+  hosts:
+  - datagrepper
+  - datagrepper-stg
+  - notifs-frontend
+  - notifs-frontend-stg
+  user: root
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "/srv/private/ansible/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+  tasks:
+  - service: name="httpd" state=started
+  post_tasks:
+  - name: tell nagios to unshush
+    nagios: action=unsilence service=host host={{ inventory_hostname }}
+    delegate_to: noc01.phx2.fedoraproject.org
+    ignore_errors: true
+
+- name: restart the last backend piece (badges)
+  hosts:
+  - badges-backend
+  - badges-backend-stg
+  user: root
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "/srv/private/ansible/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+  tasks:
+  - service: name="fedmsg-hub" state=started
+  post_tasks:
+  - name: tell nagios to unshush
+    nagios: action=unsilence service=host host={{ inventory_hostname }}
+    delegate_to: noc01.phx2.fedoraproject.org
+    ignore_errors: true