From 67fbd00bc30cc9b90426f98430fc2ce2d02e27d4 Mon Sep 17 00:00:00 2001
From: Adrian Reber <adrian@lisas.de>
Date: Wed, 11 Nov 2020 12:39:15 +0100
Subject: [PATCH] mirrormanager: use the new mirrorlist cache generation tool

The new mirrorlist cache generation requires now less than one minute.
This includes pushing the data to the proxies. It also runs now every 15
minutes instead of once an hour.

The sync_pkl_to_mirrorlists.sh script has been renamed to
sync_cache_to_mirrorlists.sh as it no longer copies a Python pickle and
it also copies the data to only proxies at the same time instead of one
proxy at a time.

Signed-off-by: Adrian Reber <adrian@lisas.de>
---
 roles/mirrormanager/backend/files/backend.cron | 10 ++++------
 .../backend/files/update-mirrorlist-server     | 18 ++++++++++++++++++
 roles/mirrormanager/backend/tasks/main.yml     |  3 ++-
 ...orlists.sh => sync_cache_to_mirrorlists.sh} |  2 +-
 4 files changed, 25 insertions(+), 8 deletions(-)
 create mode 100644 roles/mirrormanager/backend/files/update-mirrorlist-server
 rename roles/mirrormanager/backend/templates/{sync_pkl_to_mirrorlists.sh => sync_cache_to_mirrorlists.sh} (69%)

diff --git a/roles/mirrormanager/backend/files/backend.cron b/roles/mirrormanager/backend/files/backend.cron
index 957f7227a8..7e05275b30 100644
--- a/roles/mirrormanager/backend/files/backend.cron
+++ b/roles/mirrormanager/backend/files/backend.cron
@@ -1,10 +1,8 @@
 MAILTO=root
 
-# Refresh the mirrorlist cache once every hour.
-# This can take up to 35 minutes. The mirrorlist servers
-# are restarted :15 after the full hour. Starting at :30
-# should give us enough time.
-30 * * * * mirrormanager /usr/bin/mm2_update-mirrorlist-server -p -o None && /usr/local/bin/sync_pkl_to_mirrorlists.sh
+# Refresh the mirrorlist cache every 15 minutes
+# This takes about 1 minute
+0,15,30,45 * * * * mirrormanager /usr/local/bin/update-mirrorlist-server && /usr/local/bin/sync_cache_to_mirrorlists.sh
 
 # update master directory
 # logs sent to /var/log/mirrormanager/umdl.log by default
@@ -14,7 +12,7 @@ MAILTO=root
 10,40 * * * * mirrormanager /usr/local/bin/umdl-required fedora /var/log/mirrormanager/umdl-required.log
 # check if category Fedora Secondary Arches needs updating every 30 minutes
 20,50 * * * * mirrormanager /usr/local/bin/umdl-required fedora-secondary /var/log/mirrormanager/umdl-required.log
-# check if category Fedora Codecs needs updating once a day 
+# check if category Fedora Codecs needs updating once a day
 15 20 * * * mirrormanager /usr/local/bin/umdl-required codecs /var/log/mirrormanager/umdl-required.log
 # the remaining categories are updated every two hours
 15 */2 * * * mirrormanager /usr/local/bin/umdl-required archive /var/log/mirrormanager/umdl-required.log
diff --git a/roles/mirrormanager/backend/files/update-mirrorlist-server b/roles/mirrormanager/backend/files/update-mirrorlist-server
new file mode 100644
index 0000000000..4dc4998697
--- /dev/null
+++ b/roles/mirrormanager/backend/files/update-mirrorlist-server
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+LOCKFILE=/var/lock/mirrormanager/update-mirrorlist-server.lock
+CACHEDIR=/var/lib/mirrormanager
+
+[ -e ${LOCKFILE} ] && kill -0 $(cat ${LOCKFILE}) && exit 2
+mkdir -p $(dirname ${LOCKFILE})
+echo $$ > ${LOCKFILE}
+trap "rm -f ${LOCKFILE}" QUIT TERM INT HUP EXIT
+
+cd ${MM_DIR}
+rm -rf ${CACHEDIR}/old
+mkdir -p ${CACHEDIR}/old
+cp -ar ${CACHEDIR}/*  ${CACHEDIR}/old/ 2>/dev/null
+
+/usr/local/bin/generate-mirrorlist-cache $@
+
+exit 0
diff --git a/roles/mirrormanager/backend/tasks/main.yml b/roles/mirrormanager/backend/tasks/main.yml
index 01120cd862..eff2aa4a7f 100644
--- a/roles/mirrormanager/backend/tasks/main.yml
+++ b/roles/mirrormanager/backend/tasks/main.yml
@@ -51,10 +51,11 @@
 - name: install backend helper scripts
   template: src={{ item }} dest=/usr/local/bin/{{ item }} mode=0755
   with_items:
-  - sync_pkl_to_mirrorlists.sh
+  - sync_cache_to_mirrorlists.sh
   - handle_propagation.sh
   - create_maps.sh
   - create_statistics.sh
+  - update-mirrorlist-server
   tags:
   - mirrormanager/backend
 
diff --git a/roles/mirrormanager/backend/templates/sync_pkl_to_mirrorlists.sh b/roles/mirrormanager/backend/templates/sync_cache_to_mirrorlists.sh
similarity index 69%
rename from roles/mirrormanager/backend/templates/sync_pkl_to_mirrorlists.sh
rename to roles/mirrormanager/backend/templates/sync_cache_to_mirrorlists.sh
index ee98d7491b..de8c404c4b 100644
--- a/roles/mirrormanager/backend/templates/sync_pkl_to_mirrorlists.sh
+++ b/roles/mirrormanager/backend/templates/sync_cache_to_mirrorlists.sh
@@ -5,5 +5,5 @@
 MIRRORLIST_PROXY="{% for host in groups['mirrorlist_proxies'] %} {{ host }} {% endfor %}"
 
 for s in ${MIRRORLIST_PROXY}; do
-	rsync -az --delete-delay --delay-updates --delete /usr/share/mirrormanager2/country_continent.csv /var/lib/mirrormanager/{*pkl,*txt,*proto} ${s}:/srv/mirrorlist/data/mirrorlist1/
+	rsync -az --delete-delay --delay-updates --delete /usr/share/mirrormanager2/country_continent.csv /var/lib/mirrormanager/{*txt,*proto} ${s}:/srv/mirrorlist/data/mirrorlist1/ &
 done