infrastructure/ansible
1
0
Fork 0
Code Pull requests 7 Activity Actions

Only scan centos stream 9 primary mirror if there are changes

Browse source 
Currently we still see some errors with scanning the primary centos
stream 9 mirror.

The current assumption is that because of '--delay-updates' rsync
updates the primary mirror timestamps even if the actual files are moved
to the final location much later. This breaks our assumption during
scanning that we only have to check files in directories with changed
timestamps.

Currently we might scan the primary mirror and see updated timestamps
but the files are not yet moved to the final location because of
'--delay-updates'. After our scan, rsync finishes on the primary mirror
and the files are moved to the final location, but the timestamps of the
directories do not change. That results in us not detecting new files.

This change checks the timestamp of the COMPOSE_ID file and only runs
the scan if the timestamp is newer than during the last scan. The hope
is that the timestamp of COMPOSE_ID is only updated once the actual new
file is replacing the old file.

In addition to hopefully fixing the primary mirror scanning this should
also reduce the I/O load on the primary mirror because we do not run a
full rsync directory listing if COMPOSE_ID has not changed.

Signed-off-by: Adrian Reber <adrian@lisas.de>
This commit is contained in:
Adrian Reber 2021-11-11 17:14:36 +01:00
parent d6f742aaf7
commit e66473e17d
1 changed files with 24 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

49
roles/mirrormanager/backend/files/umdl-required
View file

@ -18,32 +18,22 @@ CURDATE=`date +%s`
SCANNER="/usr/bin/mm2_update-master-directory-list"
if [ "${1}" == "fedora" ]; then
CATEGORY="Linux"
CATEGORY="Fedora Linux"
elif [ "${1}" == "epel" ]; then
CATEGORY="EPEL"
CATEGORY="Fedora EPEL"
SCANNER="/usr/local/bin/scan-primary-mirror"
elif [ "${1}" == "alt" ]; then
CATEGORY="Other"
CATEGORY="Fedora Other"
elif [ "${1}" == "fedora-secondary" ]; then
CATEGORY="Secondary Arches"
CATEGORY="Fedora Secondary Arches"
elif [ "${1}" == "archive" ]; then
CATEGORY="Archive"
CATEGORY="Fedora Archive"
elif [ "${1}" == "codecs" ]; then
CATEGORY="Codecs"
CATEGORY="Fedora Codecs"
SCANNER="/usr/local/bin/scan-primary-mirror"
/usr/local/bin/lock-wrapper umdl-${1} "${SCANNER} --category \"Fedora ${CATEGORY}\""
exit 0
elif [ "${1}" == "centos" ]; then
CATEGORY="CentOS"
SCANNER="/usr/local/bin/scan-primary-mirror -c /etc/mirrormanager/scan-primary-mirror-centos.toml -d"
/usr/local/bin/lock-wrapper umdl-${1} "${SCANNER} --category \"${CATEGORY}\""
if [ "$?" -eq "0" ]; then
echo -n "Finished umdl for ${CATEGORY} successfully at "
else
echo -n "${SCANNER} for ${CATEGORY} returned non-zero. Something failed. Please check umdl.log. "
fi
date
exit 0
fi
if [ -e /var/run/mirrormanager/umdl-${1} ]; then
@ -53,12 +43,21 @@ else
let LASTRUN=CURDATE-86400
fi
FFTL="/srv/pub/${1}/fullfiletimelist-${1}"
FILEDATE=`stat -c %Z ${FFTL} 2> /dev/null`
if [ "${1}" == "centos" ]; then
FFTL="http://mirror.stream.centos.org/9-stream/COMPOSE_ID"
FILEDATE=`date +%s -d"$( curl -s --head ${FFTL} | awk 'BEGIN {FS=": "}/^Last-Modified/{print $2}' )"`
if [ "$?" -eq "1" ]; then
echo "Error stat() of ${FFTL} failed. This should not happen."
exit 1
elif [ "${1}" == "codecs" ]; then
FFTL="${CATEGORY}"
FILEDATE=${CURDATE}
else
FFTL="/srv/pub/${1}/fullfiletimelist-${1}"
FILEDATE=`stat -c %Z ${FFTL} 2> /dev/null`
if [ "$?" -eq "1" ]; then
echo "Error stat() of ${FFTL} failed. This should not happen."
exit 1
fi
fi
if [ "$LASTRUN" -gt "$FILEDATE" ]; then
@ -67,18 +66,18 @@ if [ "$LASTRUN" -gt "$FILEDATE" ]; then
exit 0
fi
echo -n "${FFTL} has changed since last run. Running umdl for Fedora ${CATEGORY} at "
echo -n "${FFTL} has changed since last run. Running umdl for ${CATEGORY} at "
date
/usr/local/bin/lock-wrapper umdl-${1} "${SCANNER} --category \"Fedora ${CATEGORY}\""
/usr/local/bin/lock-wrapper umdl-${1} "${SCANNER} --category \"${CATEGORY}\""
if [ "$?" -eq "0" ]; then
# success! remember the date of this run
echo "LASTRUN=${CURDATE}" > /var/run/mirrormanager/umdl-${1}
echo -n "Finished umdl for Fedora ${CATEGORY} successfully at "
echo -n "Finished umdl for ${CATEGORY} successfully at "
date
exit 0
fi
echo -n "umdl for Fedora ${CATEGORY} returned non-zero. Something failed. Please check umdl.log. "
echo -n "umdl for ${CATEGORY} returned non-zero. Something failed. Please check umdl.log. "
date