ok let us try and make proxy01/10 have 3 engines

roles/haproxy/templates/haproxy.cfg

@@ -58,6 +58,9 @@ backend mirror-lists-backend
     timeout connect 30s
     server  mirrorlist-local1 127.0.0.1:18081 check inter 1s rise 2 fall 3 weight 100
     server  mirrorlist-local2 127.0.0.1:18082 check inter 1s rise 2 fall 3 weight 100
+{% if datacenter == 'phx2' %}
+    server  mirrorlist-local3 127.0.0.1:18083 check inter 1s rise 2 fall 3 weight 100
+{% endif %}
     option  httpchk GET /metalink?repo=epel-7&arch=x86_64
     option  allbackups
 

roles/mirrormanager/mirrorlist_proxy/tasks/main.yml

@@ -22,6 +22,7 @@
   - /srv/mirrorlist/data
   - /srv/mirrorlist/data/mirrorlist1
   - /srv/mirrorlist/data/mirrorlist2
+  - /srv/mirrorlist/data/mirrorlist3
   - /var/log/mirrormanager
   tags:
   - mirrorlist_proxy
@@ -32,6 +33,7 @@
   - /srv/mirrorlist/data
   - /srv/mirrorlist/data/mirrorlist1
   - /srv/mirrorlist/data/mirrorlist2
+  - /srv/mirrorlist/data/mirrorlist3
   tags:
   - mirrorlist_proxy
 
@@ -70,6 +72,18 @@
   notify:
   - reload systemd
 
+# We deploy two service files. Both listen on a different port, so that we can switch
+# them out as part of the pkl deployment without having any local downtime.
+- name: Deploy service files
+  template: src=mirrorlist.service.j2 dest=/etc/systemd/system/mirrorlist{{ item }}.service
+  with_items:
+  - 3
+  tags:
+  - mirrorlist_proxy
+  notify:
+  - reload systemd
+  when: datacenter == 'phx2'
+
 # enable both of them to run on boot
 - name: Enable mirrorlist1
   service: name=mirrorlist1 enabled=yes
@@ -81,9 +95,15 @@
   tags:
   - mirrorlist_proxy
 
+- name: Enable mirrorlist3
+  service: name=mirrorlist3 enabled=yes
+  tags:
+  - mirrorlist_proxy
+  when: datacenter == 'phx2'
+
 # install our cron script to handle hourly new pkl changes. 
 - name: install script to restart mirrorlist containers on pkl changes
-  copy: src=restart-mirrorlist-containers dest=/usr/local/bin/restart-mirrorlist-containers mode=0755
+  template: src=restart-mirrorlist-containers.j2 dest=/usr/local/bin/restart-mirrorlist-containers mode=0755
   tags:
   - mirrorlist_proxy
 

roles/mirrormanager/mirrorlist_proxy/templates/restart-mirrorlist-containers.j2

@@ -0,0 +1,132 @@
+#!/bin/bash
+
+#
+# This job handles hourly restarting mirrorlist container(s) and making sure all is well. 
+#
+
+##
+## Docker ports for mirrors
+mirrorlist1="http://localhost:18081/metalink?repo=rawhide&arch=x86_64"
+mirrorlist2="http://localhost:18082/metalink?repo=rawhide&arch=x86_64"
+{% if datacenter == 'phx2' %}
+mirrorlist3="http://localhost:18083/metalink?repo=rawhide&arch=x86_64"
+{% endif %}
+
+TIME_DRAIN=30
+TIME_RESTART=5
+TIME_DISABLE=5
+
+##
+
+# Initial expected state is mirrorlist1 running, mirrorlist2 running and new pkl
+
+if [ ! -f /srv/mirrorlist/data/mirrorlist2/global_netblocks.txt ];
+then
+	cp /srv/mirrorlist/data/mirrorlist1/* /srv/mirrorlist/data/mirrorlist2/
+fi
+
+{% if datacenter == 'phx2' %}
+# We have more mirror containers here
+if [ ! -f /srv/mirrorlist/data/mirrorlist3/global_netblocks.txt ];
+then
+	cp /srv/mirrorlist/data/mirrorlist1/* /srv/mirrorlist/data/mirrorlist3/
+fi
+{% endif %}
+
+
+## Check that pkl is newer than old pkl
+if [ /srv/mirrorlist/data/mirrorlist1/mirrorlist_cache.pkl -nt /srv/mirrorlist/data/mirrorlist2/mirrorlist_cache.pkl ]; then
+	# new pkl
+        :
+else
+	# No new pkl
+        exit 0
+fi
+## check mirrorlist1 running
+if [ `systemctl show mirrorlist1 -p ActiveState` != 'ActiveState=active' ]; then
+	# mirrorlist1 not running, there is a problem
+	echo "Error: mirrorlist1 is not running as expected"
+	exit 1
+fi
+
+# check mirrorlist2 (old pkl and see that it's processing ok)
+curl -q -H mirrors.fedoraproject.org ${mirrorlist2} -s -f --retry 50 --retry-delay 10 --retry-connrefused --retry-max-time 180 | grep "sha512" >/dev/null
+if [ $? != 0 ]; then
+	echo "ERROR: mirrorlist2 not processing correctly"
+	exit 1
+fi
+
+{% if datacenter == 'phx2' %}
+# check mirrorlist3 (old pkl and see that it's processing ok)
+curl -q -H mirrors.fedoraproject.org ${mirrorlist3} -s -f --retry 50 --retry-delay 10 --retry-connrefused --retry-max-time 180 | grep "sha512" >/dev/null
+if [ $? != 0 ]; then
+	echo "ERROR: mirrorlist3 not processing correctly"
+	exit 1
+fi
+{% endif %}
+
+
+# Drain mirrorlist1. This is safe since we assured that local2 is serving
+echo "enable server mirror-lists-backend/mirrorlist-local2" | nc -U /var/run/haproxy-admin >& /dev/null
+echo "set server mirror-lists-backend/mirrorlist-local1 state drain" | nc -U /var/run/haproxy-admin >& /dev/null
+sleep ${TIME_DRAIN}
+echo "disable server mirror-lists-backend/mirrorlist-local1" | nc -U /var/run/haproxy-admin >& /dev/null
+sleep ${TIME_DISABLE}
+
+# restart mirrorlist1 (new pkl and make sure it's processing ok)
+systemctl restart mirrorlist1
+
+sleep ${TIME_RESTART}
+curl -q -H mirrors.fedoraproject.org ${mirrorlist1} -s -f --retry 50 --retry-delay 10 --retry-connrefused --retry-max-time 180 | grep "sha512" >/dev/null
+if [ $? != 0 ]; then
+	echo "ERROR: mirrorlist1 did not restart correctly"
+	exit 1
+fi
+
+# New mirrorlist seems to be working, put it back into service
+echo "enable server mirror-lists-backend/mirrorlist-local1" | nc -U /var/run/haproxy-admin >& /dev/null
+sleep ${TIME_RESTART}
+
+# copy new pkl to mirrorlist2
+cp -a /srv/mirrorlist/data/mirrorlist1/* /srv/mirrorlist/data/mirrorlist2/
+
+# Drain mirrorlist2
+echo "set server mirror-lists-backend/mirrorlist-local2 state drain" | nc -U /var/run/haproxy-admin >& /dev/null
+sleep ${TIME_DRAIN}
+echo "disable server mirror-lists-backend/mirrorlist-local2" | nc -U /var/run/haproxy-admin >& /dev/null
+sleep ${TIME_DISABLE}
+
+# restart mirrorlist2
+systemctl restart mirrorlist2
+
+sleep ${TIME_RESTART}
+curl -q -H mirrors.fedoraproject.org ${mirrorlist2} -o/dev/null -s -f --retry 50 --retry-delay 10 --retry-connrefused --retry-max-time 180
+if [ $? != 0 ]; then
+	echo "ERROR: mirrorlist2 did not restart correctly"
+	exit 1
+fi
+
+echo "enable server mirror-lists-backend/mirrorlist-local2" | nc -U /var/run/haproxy-admin >& /dev/null
+
+{% if datacenter == 'phx2' %}
+# copy new pkl to mirrorlist3
+cp -a /srv/mirrorlist/data/mirrorlist1/* /srv/mirrorlist/data/mirrorlist3/
+
+# Drain mirrorlist3
+echo "set server mirror-lists-backend/mirrorlist-local3 state drain" | nc -U /var/run/haproxy-admin >& /dev/null
+sleep ${TIME_DRAIN}
+echo "disable server mirror-lists-backend/mirrorlist-local3" | nc -U /var/run/haproxy-admin >& /dev/null
+sleep ${TIME_DISABLE}
+
+# restart mirrorlist3
+systemctl restart mirrorlist3
+
+sleep ${TIME_RESTART}
+curl -q -H mirrors.fedoraproject.org ${mirrorlist3} -o/dev/null -s -f --retry 50 --retry-delay 10 --retry-connrefused --retry-max-time 180
+if [ $? != 0 ]; then
+	echo "ERROR: mirrorlist3 did not restart correctly"
+	exit 1
+fi
+
+echo "enable server mirror-lists-backend/mirrorlist-local3" | nc -U /var/run/haproxy-admin >& /dev/null
+{% endif %}