From 58a9e4d4e5af3d51d65a4f41b58b27505da2a098 Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Mon, 25 Feb 2019 19:35:27 +0000 Subject: [PATCH] ok let us try and make proxy01/10 have 3 engines --- roles/haproxy/templates/haproxy.cfg | 3 + .../mirrorlist_proxy/tasks/main.yml | 22 ++- .../restart-mirrorlist-containers.j2 | 132 ++++++++++++++++++ 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 roles/mirrormanager/mirrorlist_proxy/templates/restart-mirrorlist-containers.j2 diff --git a/roles/haproxy/templates/haproxy.cfg b/roles/haproxy/templates/haproxy.cfg index d5d2c4f89b..29bb567b7c 100644 --- a/roles/haproxy/templates/haproxy.cfg +++ b/roles/haproxy/templates/haproxy.cfg @@ -58,6 +58,9 @@ backend mirror-lists-backend timeout connect 30s server mirrorlist-local1 127.0.0.1:18081 check inter 1s rise 2 fall 3 weight 100 server mirrorlist-local2 127.0.0.1:18082 check inter 1s rise 2 fall 3 weight 100 +{% if datacenter == 'phx2' %} + server mirrorlist-local3 127.0.0.1:18083 check inter 1s rise 2 fall 3 weight 100 +{% endif %} option httpchk GET /metalink?repo=epel-7&arch=x86_64 option allbackups diff --git a/roles/mirrormanager/mirrorlist_proxy/tasks/main.yml b/roles/mirrormanager/mirrorlist_proxy/tasks/main.yml index 1efb50b821..7921d48a73 100644 --- a/roles/mirrormanager/mirrorlist_proxy/tasks/main.yml +++ b/roles/mirrormanager/mirrorlist_proxy/tasks/main.yml @@ -22,6 +22,7 @@ - /srv/mirrorlist/data - /srv/mirrorlist/data/mirrorlist1 - /srv/mirrorlist/data/mirrorlist2 + - /srv/mirrorlist/data/mirrorlist3 - /var/log/mirrormanager tags: - mirrorlist_proxy @@ -32,6 +33,7 @@ - /srv/mirrorlist/data - /srv/mirrorlist/data/mirrorlist1 - /srv/mirrorlist/data/mirrorlist2 + - /srv/mirrorlist/data/mirrorlist3 tags: - mirrorlist_proxy @@ -70,6 +72,18 @@ notify: - reload systemd +# We deploy two service files. Both listen on a different port, so that we can switch +# them out as part of the pkl deployment without having any local downtime. +- name: Deploy service files + template: src=mirrorlist.service.j2 dest=/etc/systemd/system/mirrorlist{{ item }}.service + with_items: + - 3 + tags: + - mirrorlist_proxy + notify: + - reload systemd + when: datacenter == 'phx2' + # enable both of them to run on boot - name: Enable mirrorlist1 service: name=mirrorlist1 enabled=yes @@ -81,9 +95,15 @@ tags: - mirrorlist_proxy +- name: Enable mirrorlist3 + service: name=mirrorlist3 enabled=yes + tags: + - mirrorlist_proxy + when: datacenter == 'phx2' + # install our cron script to handle hourly new pkl changes. - name: install script to restart mirrorlist containers on pkl changes - copy: src=restart-mirrorlist-containers dest=/usr/local/bin/restart-mirrorlist-containers mode=0755 + template: src=restart-mirrorlist-containers.j2 dest=/usr/local/bin/restart-mirrorlist-containers mode=0755 tags: - mirrorlist_proxy diff --git a/roles/mirrormanager/mirrorlist_proxy/templates/restart-mirrorlist-containers.j2 b/roles/mirrormanager/mirrorlist_proxy/templates/restart-mirrorlist-containers.j2 new file mode 100644 index 0000000000..f3f1f67426 --- /dev/null +++ b/roles/mirrormanager/mirrorlist_proxy/templates/restart-mirrorlist-containers.j2 @@ -0,0 +1,132 @@ +#!/bin/bash + +# +# This job handles hourly restarting mirrorlist container(s) and making sure all is well. +# + +## +## Docker ports for mirrors +mirrorlist1="http://localhost:18081/metalink?repo=rawhide&arch=x86_64" +mirrorlist2="http://localhost:18082/metalink?repo=rawhide&arch=x86_64" +{% if datacenter == 'phx2' %} +mirrorlist3="http://localhost:18083/metalink?repo=rawhide&arch=x86_64" +{% endif %} + +TIME_DRAIN=30 +TIME_RESTART=5 +TIME_DISABLE=5 + +## + +# Initial expected state is mirrorlist1 running, mirrorlist2 running and new pkl + +if [ ! -f /srv/mirrorlist/data/mirrorlist2/global_netblocks.txt ]; +then + cp /srv/mirrorlist/data/mirrorlist1/* /srv/mirrorlist/data/mirrorlist2/ +fi + +{% if datacenter == 'phx2' %} +# We have more mirror containers here +if [ ! -f /srv/mirrorlist/data/mirrorlist3/global_netblocks.txt ]; +then + cp /srv/mirrorlist/data/mirrorlist1/* /srv/mirrorlist/data/mirrorlist3/ +fi +{% endif %} + + +## Check that pkl is newer than old pkl +if [ /srv/mirrorlist/data/mirrorlist1/mirrorlist_cache.pkl -nt /srv/mirrorlist/data/mirrorlist2/mirrorlist_cache.pkl ]; then + # new pkl + : +else + # No new pkl + exit 0 +fi +## check mirrorlist1 running +if [ `systemctl show mirrorlist1 -p ActiveState` != 'ActiveState=active' ]; then + # mirrorlist1 not running, there is a problem + echo "Error: mirrorlist1 is not running as expected" + exit 1 +fi + +# check mirrorlist2 (old pkl and see that it's processing ok) +curl -q -H mirrors.fedoraproject.org ${mirrorlist2} -s -f --retry 50 --retry-delay 10 --retry-connrefused --retry-max-time 180 | grep "sha512" >/dev/null +if [ $? != 0 ]; then + echo "ERROR: mirrorlist2 not processing correctly" + exit 1 +fi + +{% if datacenter == 'phx2' %} +# check mirrorlist3 (old pkl and see that it's processing ok) +curl -q -H mirrors.fedoraproject.org ${mirrorlist3} -s -f --retry 50 --retry-delay 10 --retry-connrefused --retry-max-time 180 | grep "sha512" >/dev/null +if [ $? != 0 ]; then + echo "ERROR: mirrorlist3 not processing correctly" + exit 1 +fi +{% endif %} + + +# Drain mirrorlist1. This is safe since we assured that local2 is serving +echo "enable server mirror-lists-backend/mirrorlist-local2" | nc -U /var/run/haproxy-admin >& /dev/null +echo "set server mirror-lists-backend/mirrorlist-local1 state drain" | nc -U /var/run/haproxy-admin >& /dev/null +sleep ${TIME_DRAIN} +echo "disable server mirror-lists-backend/mirrorlist-local1" | nc -U /var/run/haproxy-admin >& /dev/null +sleep ${TIME_DISABLE} + +# restart mirrorlist1 (new pkl and make sure it's processing ok) +systemctl restart mirrorlist1 + +sleep ${TIME_RESTART} +curl -q -H mirrors.fedoraproject.org ${mirrorlist1} -s -f --retry 50 --retry-delay 10 --retry-connrefused --retry-max-time 180 | grep "sha512" >/dev/null +if [ $? != 0 ]; then + echo "ERROR: mirrorlist1 did not restart correctly" + exit 1 +fi + +# New mirrorlist seems to be working, put it back into service +echo "enable server mirror-lists-backend/mirrorlist-local1" | nc -U /var/run/haproxy-admin >& /dev/null +sleep ${TIME_RESTART} + +# copy new pkl to mirrorlist2 +cp -a /srv/mirrorlist/data/mirrorlist1/* /srv/mirrorlist/data/mirrorlist2/ + +# Drain mirrorlist2 +echo "set server mirror-lists-backend/mirrorlist-local2 state drain" | nc -U /var/run/haproxy-admin >& /dev/null +sleep ${TIME_DRAIN} +echo "disable server mirror-lists-backend/mirrorlist-local2" | nc -U /var/run/haproxy-admin >& /dev/null +sleep ${TIME_DISABLE} + +# restart mirrorlist2 +systemctl restart mirrorlist2 + +sleep ${TIME_RESTART} +curl -q -H mirrors.fedoraproject.org ${mirrorlist2} -o/dev/null -s -f --retry 50 --retry-delay 10 --retry-connrefused --retry-max-time 180 +if [ $? != 0 ]; then + echo "ERROR: mirrorlist2 did not restart correctly" + exit 1 +fi + +echo "enable server mirror-lists-backend/mirrorlist-local2" | nc -U /var/run/haproxy-admin >& /dev/null + +{% if datacenter == 'phx2' %} +# copy new pkl to mirrorlist3 +cp -a /srv/mirrorlist/data/mirrorlist1/* /srv/mirrorlist/data/mirrorlist3/ + +# Drain mirrorlist3 +echo "set server mirror-lists-backend/mirrorlist-local3 state drain" | nc -U /var/run/haproxy-admin >& /dev/null +sleep ${TIME_DRAIN} +echo "disable server mirror-lists-backend/mirrorlist-local3" | nc -U /var/run/haproxy-admin >& /dev/null +sleep ${TIME_DISABLE} + +# restart mirrorlist3 +systemctl restart mirrorlist3 + +sleep ${TIME_RESTART} +curl -q -H mirrors.fedoraproject.org ${mirrorlist3} -o/dev/null -s -f --retry 50 --retry-delay 10 --retry-connrefused --retry-max-time 180 +if [ $? != 0 ]; then + echo "ERROR: mirrorlist3 did not restart correctly" + exit 1 +fi + +echo "enable server mirror-lists-backend/mirrorlist-local3" | nc -U /var/run/haproxy-admin >& /dev/null +{% endif %}