Enable CentOS Stream 9 in MirrorManager

In addition to configuration file changes this also needs the latest
version of MirrorManager, scan-primary-mirror and mirrorlist-server.

The reason for the software updates are mainly that CentOS Stream is
using an empty topdir ('') and all around the code there were
assumptions that topdir always ends with a slash and an empty string
does not end with a slash.

Signed-off-by: Adrian Reber <adrian@lisas.de>

roles/mirrormanager/backend/files/backend.cron

@@ -18,6 +18,9 @@ MAILTO=root
 15 */2 * * * mirrormanager /usr/local/bin/umdl-required archive /var/log/mirrormanager/umdl-required.log
 45 */2  * * * mirrormanager /usr/local/bin/umdl-required alt /var/log/mirrormanager/umdl-required.log
 
+# check if category CentOS needs updating once a day
+15 20 * * * mirrormanager /usr/local/bin/umdl-required centos /var/log/mirrormanager/umdl-required.log
+
 # Sync netblocks list once a day
 30 0 * * * mirrormanager cd /usr/share/mirrormanager2 && /usr/bin/mm2_get_global_netblocks /var/lib/mirrormanager/global_netblocks.txt
 # Run the script for Internet2 netblocks very late to

roles/mirrormanager/backend/files/umdl-required

@@ -30,6 +30,19 @@ elif [ "${1}" ==  "archive" ]; then
 	CATEGORY="Archive"
 elif [ "${1}" ==  "codecs" ]; then
 	CATEGORY="Codecs"
+	/usr/local/bin/lock-wrapper umdl-${1} "${SCANNER} --category \"Fedora ${CATEGORY}\""
+	exit 0
+elif [ "${1}" ==  "centos" ]; then
+	CATEGORY="CentOS"
+	SCANNER="/usr/local/bin/scan-primary-mirror -c /etc/mirrormanager/scan-primary-mirror-centos.toml"
+	/usr/local/bin/lock-wrapper umdl-${1} "${SCANNER} --category \"${CATEGORY}\""
+	if [ "$?" -eq "0" ]; then
+		echo -n "Finished umdl for ${CATEGORY} successfully at "
+	else
+		echo -n "${SCANNER} for ${CATEGORY} returned non-zero. Something failed. Please check umdl.log. "
+	fi
+	date
+	exit 0
 fi
 
 if [ -e /var/run/mirrormanager/umdl-${1} ]; then
@@ -39,13 +52,7 @@ else
 	let LASTRUN=CURDATE-86400
 fi
 
-# FULLFILETIMELIST -> FFTL
+FFTL="/srv/pub/${1}/fullfiletimelist-${1}"
-if [ "${1}" ==  "codecs" ]; then
-    /usr/local/bin/lock-wrapper umdl-${1} "${SCANNER} --category \"Fedora ${CATEGORY}\""
-    exit 0
-else
-    FFTL="/srv/pub/${1}/fullfiletimelist-${1}"
-fi
 FILEDATE=`stat -c %Z ${FFTL} 2> /dev/null`
 
 if [ "$?" -eq "1" ]; then

roles/mirrormanager/backend/tasks/main.yml

@@ -161,6 +161,13 @@
     owner: mirrormanager
     group: mirrormanager
     mode: 0600
+- name: install scan-primary-mirror centos configuration file
+  template:
+    src: scan-primary-mirror-centos.toml
+    dest: /etc/mirrormanager/scan-primary-mirror-centos.toml
+    owner: mirrormanager
+    group: mirrormanager
+    mode: 0600
 - name: create /srv/mirrorlist-server.git
   file: path=/srv/mirrorlist-server.git state=directory owner=mirrormanager group=mirrormanager mode=0755
 - name: create /srv/scan-primary-mirror.git
@@ -168,7 +175,7 @@
 - name: checkout mirrorlist-server
   git:
     repo: https://github.com/adrianreber/mirrorlist-server.git
-    version: 3.0.4
+    version: 3.0.5
     dest: /srv/mirrorlist-server.git
   become: yes
   become_user: mirrormanager
@@ -176,7 +183,7 @@
 - name: checkout scan-primary-mirror
   git:
     repo: https://github.com/adrianreber/scan-primary-mirror.git
-    version: 0.1.1
+    version: 0.2.0
     dest: /srv/scan-primary-mirror.git
   become: yes
   become_user: mirrormanager

roles/mirrormanager/backend/templates/scan-primary-mirror-centos.toml

@@ -0,0 +1,41 @@
+common_rsync_options="--no-motd --timeout 14400"
+max_propagation_days = 2
+max_stale_days = 3
+skip_repository_paths = ["SIGs"]
+
+[database]
+url="postgresql://{{ mirrormanager_db_user }}:{{ mirrormanager_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_db_name }}"
+
+[[category]]
+name="CentOS"
+type="rsync"
+url="rsync://mirror.stream.centos.org/CentOS-Stream-All"
+checksum_base="http://mirror.stream.centos.org/"
+
+[[repository_mapping]]
+regex="^\\d+(?:-stream)?/AppStream/"
+prefix="centos-appstream"
+
+[[repository_mapping]]
+regex="^\\d+(?:-stream)?/BaseOS/"
+prefix="centos-baseos"
+
+[[repository_mapping]]
+regex="^\\d+(?:-stream)?/CRB/"
+prefix="centos-crb"
+
+[[repository_mapping]]
+regex="^\\d+(?:-stream)?/HighAvailability/"
+prefix="centos-highavailability"
+
+[[repository_mapping]]
+regex="^\\d+(?:-stream)?/NFV/"
+prefix="centos-nfv"
+
+[[repository_mapping]]
+regex="^\\d+(?:-stream)?/RT/"
+prefix="centos-rt"
+
+[[repository_mapping]]
+regex="^\\d+(?:-stream)?/ResilientStorage/"
+prefix="centos-resilientstorage"

roles/mirrormanager/crawler/files/crawler.cron

@@ -33,3 +33,6 @@
 
 # Category: 'Fedora Other'; once a day, 10 threads
 0 14 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; /usr/bin/mm2_crawler --category="Fedora Other" --threads 9 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
+
+# Category: 'CentOS'; twice a day, 20 threads
+0 5,11 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; /usr/bin/mm2_crawler --category="CentOS" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1