From e66473e17d37d29be2cee5954c3b71f1d196c6da Mon Sep 17 00:00:00 2001
From: Adrian Reber <adrian@lisas.de>
Date: Thu, 11 Nov 2021 17:14:36 +0100
Subject: [PATCH] Only scan centos stream 9 primary mirror if there are changes

Currently we still see some errors with scanning the primary centos
stream 9 mirror.

The current assumption is that because of '--delay-updates' rsync
updates the primary mirror timestamps even if the actual files are moved
to the final location much later. This breaks our assumption during
scanning that we only have to check files in directories with changed
timestamps.

Currently we might scan the primary mirror and see updated timestamps
but the files are not yet moved to the final location because of
'--delay-updates'. After our scan, rsync finishes on the primary mirror
and the files are moved to the final location, but the timestamps of the
directories do not change. That results in us not detecting new files.

This change checks the timestamp of the COMPOSE_ID file and only runs
the scan if the timestamp is newer than during the last scan. The hope
is that the timestamp of COMPOSE_ID is only updated once the actual new
file is replacing the old file.

In addition to hopefully fixing the primary mirror scanning this should
also reduce the I/O load on the primary mirror because we do not run a
full rsync directory listing if COMPOSE_ID has not changed.

Signed-off-by: Adrian Reber <adrian@lisas.de>
---
 .../mirrormanager/backend/files/umdl-required | 49 +++++++++----------
 1 file changed, 24 insertions(+), 25 deletions(-)

diff --git a/roles/mirrormanager/backend/files/umdl-required b/roles/mirrormanager/backend/files/umdl-required
index 5b24bddc08..b1f7114573 100644
--- a/roles/mirrormanager/backend/files/umdl-required
+++ b/roles/mirrormanager/backend/files/umdl-required
@@ -18,32 +18,22 @@ CURDATE=`date +%s`
 SCANNER="/usr/bin/mm2_update-master-directory-list"
 
 if [ "${1}" ==  "fedora" ]; then
-	CATEGORY="Linux"
+	CATEGORY="Fedora Linux"
 elif [ "${1}" ==  "epel" ]; then
-	CATEGORY="EPEL"
+	CATEGORY="Fedora EPEL"
 	SCANNER="/usr/local/bin/scan-primary-mirror"
 elif [ "${1}" ==  "alt" ]; then
-	CATEGORY="Other"
+	CATEGORY="Fedora Other"
 elif [ "${1}" ==  "fedora-secondary" ]; then
-	CATEGORY="Secondary Arches"
+	CATEGORY="Fedora Secondary Arches"
 elif [ "${1}" ==  "archive" ]; then
-	CATEGORY="Archive"
+	CATEGORY="Fedora Archive"
 elif [ "${1}" ==  "codecs" ]; then
-	CATEGORY="Codecs"
+	CATEGORY="Fedora Codecs"
 	SCANNER="/usr/local/bin/scan-primary-mirror"
-	/usr/local/bin/lock-wrapper umdl-${1} "${SCANNER} --category \"Fedora ${CATEGORY}\""
-	exit 0
 elif [ "${1}" ==  "centos" ]; then
 	CATEGORY="CentOS"
 	SCANNER="/usr/local/bin/scan-primary-mirror -c /etc/mirrormanager/scan-primary-mirror-centos.toml -d"
-	/usr/local/bin/lock-wrapper umdl-${1} "${SCANNER} --category \"${CATEGORY}\""
-	if [ "$?" -eq "0" ]; then
-		echo -n "Finished umdl for ${CATEGORY} successfully at "
-	else
-		echo -n "${SCANNER} for ${CATEGORY} returned non-zero. Something failed. Please check umdl.log. "
-	fi
-	date
-	exit 0
 fi
 
 if [ -e /var/run/mirrormanager/umdl-${1} ]; then
@@ -53,12 +43,21 @@ else
 	let LASTRUN=CURDATE-86400
 fi
 
-FFTL="/srv/pub/${1}/fullfiletimelist-${1}"
-FILEDATE=`stat -c %Z ${FFTL} 2> /dev/null`
+if [ "${1}" ==  "centos" ]; then
+	FFTL="http://mirror.stream.centos.org/9-stream/COMPOSE_ID"
+	FILEDATE=`date +%s -d"$( curl -s --head ${FFTL} | awk 'BEGIN {FS=": "}/^Last-Modified/{print $2}' )"`
 
-if [ "$?" -eq "1" ]; then
-	echo "Error stat() of ${FFTL} failed. This should not happen."
-	exit 1
+elif [ "${1}" ==  "codecs" ]; then
+	FFTL="${CATEGORY}"
+	FILEDATE=${CURDATE}
+else
+	FFTL="/srv/pub/${1}/fullfiletimelist-${1}"
+	FILEDATE=`stat -c %Z ${FFTL} 2> /dev/null`
+
+	if [ "$?" -eq "1" ]; then
+		echo "Error stat() of ${FFTL} failed. This should not happen."
+		exit 1
+	fi
 fi
 
 if [ "$LASTRUN" -gt "$FILEDATE" ]; then
@@ -67,18 +66,18 @@ if [ "$LASTRUN" -gt "$FILEDATE" ]; then
 	exit 0
 fi
 
-echo -n "${FFTL} has changed since last run. Running umdl for Fedora ${CATEGORY} at "
+echo -n "${FFTL} has changed since last run. Running umdl for ${CATEGORY} at "
 date
 
-/usr/local/bin/lock-wrapper umdl-${1} "${SCANNER} --category \"Fedora ${CATEGORY}\""
+/usr/local/bin/lock-wrapper umdl-${1} "${SCANNER} --category \"${CATEGORY}\""
 
 if [ "$?" -eq "0" ]; then
 	# success! remember the date of this run
 	echo "LASTRUN=${CURDATE}" > /var/run/mirrormanager/umdl-${1}
-	echo -n "Finished umdl for Fedora ${CATEGORY} successfully at "
+	echo -n "Finished umdl for ${CATEGORY} successfully at "
 	date
 	exit 0
 fi
 
-echo -n "umdl for Fedora ${CATEGORY} returned non-zero. Something failed. Please check umdl.log. "
+echo -n "umdl for ${CATEGORY} returned non-zero. Something failed. Please check umdl.log. "
 date