mirrormanager: use the new mirrorlist cache generation tool

The new mirrorlist cache generation requires now less than one minute.
This includes pushing the data to the proxies. It also runs now every 15
minutes instead of once an hour.

The sync_pkl_to_mirrorlists.sh script has been renamed to
sync_cache_to_mirrorlists.sh as it no longer copies a Python pickle and
it also copies the data to only proxies at the same time instead of one
proxy at a time.

Signed-off-by: Adrian Reber <adrian@lisas.de>

roles/mirrormanager/backend/files/backend.cron

@@ -1,10 +1,8 @@
 MAILTO=root
 
-# Refresh the mirrorlist cache once every hour.
+# Refresh the mirrorlist cache every 15 minutes
-# This can take up to 35 minutes. The mirrorlist servers
+# This takes about 1 minute
-# are restarted :15 after the full hour. Starting at :30
+0,15,30,45 * * * * mirrormanager /usr/local/bin/update-mirrorlist-server && /usr/local/bin/sync_cache_to_mirrorlists.sh
-# should give us enough time.
-30 * * * * mirrormanager /usr/bin/mm2_update-mirrorlist-server -p -o None && /usr/local/bin/sync_pkl_to_mirrorlists.sh
 
 # update master directory
 # logs sent to /var/log/mirrormanager/umdl.log by default
@@ -14,7 +12,7 @@ MAILTO=root
 10,40 * * * * mirrormanager /usr/local/bin/umdl-required fedora /var/log/mirrormanager/umdl-required.log
 # check if category Fedora Secondary Arches needs updating every 30 minutes
 20,50 * * * * mirrormanager /usr/local/bin/umdl-required fedora-secondary /var/log/mirrormanager/umdl-required.log
-# check if category Fedora Codecs needs updating once a day 
+# check if category Fedora Codecs needs updating once a day
 15 20 * * * mirrormanager /usr/local/bin/umdl-required codecs /var/log/mirrormanager/umdl-required.log
 # the remaining categories are updated every two hours
 15 */2 * * * mirrormanager /usr/local/bin/umdl-required archive /var/log/mirrormanager/umdl-required.log

roles/mirrormanager/backend/files/update-mirrorlist-server

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+LOCKFILE=/var/lock/mirrormanager/update-mirrorlist-server.lock
+CACHEDIR=/var/lib/mirrormanager
+
+[ -e ${LOCKFILE} ] && kill -0 $(cat ${LOCKFILE}) && exit 2
+mkdir -p $(dirname ${LOCKFILE})
+echo $$ > ${LOCKFILE}
+trap "rm -f ${LOCKFILE}" QUIT TERM INT HUP EXIT
+
+cd ${MM_DIR}
+rm -rf ${CACHEDIR}/old
+mkdir -p ${CACHEDIR}/old
+cp -ar ${CACHEDIR}/*  ${CACHEDIR}/old/ 2>/dev/null
+
+/usr/local/bin/generate-mirrorlist-cache $@
+
+exit 0

roles/mirrormanager/backend/tasks/main.yml

@@ -51,10 +51,11 @@
 - name: install backend helper scripts
   template: src={{ item }} dest=/usr/local/bin/{{ item }} mode=0755
   with_items:
-  - sync_pkl_to_mirrorlists.sh
+  - sync_cache_to_mirrorlists.sh
   - handle_propagation.sh
   - create_maps.sh
   - create_statistics.sh
+  - update-mirrorlist-server
   tags:
   - mirrormanager/backend
 

roles/mirrormanager/backend/templates/sync_cache_to_mirrorlists.sh

@@ -5,5 +5,5 @@
 MIRRORLIST_PROXY="{% for host in groups['mirrorlist_proxies'] %} {{ host }} {% endfor %}"
 
 for s in ${MIRRORLIST_PROXY}; do
-	rsync -az --delete-delay --delay-updates --delete /usr/share/mirrormanager2/country_continent.csv /var/lib/mirrormanager/{*pkl,*txt,*proto} ${s}:/srv/mirrorlist/data/mirrorlist1/
+	rsync -az --delete-delay --delay-updates --delete /usr/share/mirrormanager2/country_continent.csv /var/lib/mirrormanager/{*txt,*proto} ${s}:/srv/mirrorlist/data/mirrorlist1/ &
 done