From f3c9f3bb9d92c8946c0e725e1dbc039639a3de41 Mon Sep 17 00:00:00 2001 From: Adrian Reber Date: Mon, 6 Sep 2021 19:40:36 +0200 Subject: [PATCH] Enable CentOS Stream 9 in MirrorManager In addition to configuration file changes this also needs the latest version of MirrorManager, scan-primary-mirror and mirrorlist-server. The reason for the software updates are mainly that CentOS Stream is using an empty topdir ('') and all around the code there were assumptions that topdir always ends with a slash and an empty string does not end with a slash. Signed-off-by: Adrian Reber --- .../mirrormanager/backend/files/backend.cron | 3 ++ .../mirrormanager/backend/files/umdl-required | 21 ++++++---- roles/mirrormanager/backend/tasks/main.yml | 11 ++++- .../templates/scan-primary-mirror-centos.toml | 41 +++++++++++++++++++ .../mirrormanager/crawler/files/crawler.cron | 3 ++ 5 files changed, 70 insertions(+), 9 deletions(-) create mode 100644 roles/mirrormanager/backend/templates/scan-primary-mirror-centos.toml diff --git a/roles/mirrormanager/backend/files/backend.cron b/roles/mirrormanager/backend/files/backend.cron index 7e05275b30..e2cbee5ba8 100644 --- a/roles/mirrormanager/backend/files/backend.cron +++ b/roles/mirrormanager/backend/files/backend.cron @@ -18,6 +18,9 @@ MAILTO=root 15 */2 * * * mirrormanager /usr/local/bin/umdl-required archive /var/log/mirrormanager/umdl-required.log 45 */2 * * * mirrormanager /usr/local/bin/umdl-required alt /var/log/mirrormanager/umdl-required.log +# check if category CentOS needs updating once a day +15 20 * * * mirrormanager /usr/local/bin/umdl-required centos /var/log/mirrormanager/umdl-required.log + # Sync netblocks list once a day 30 0 * * * mirrormanager cd /usr/share/mirrormanager2 && /usr/bin/mm2_get_global_netblocks /var/lib/mirrormanager/global_netblocks.txt # Run the script for Internet2 netblocks very late to diff --git a/roles/mirrormanager/backend/files/umdl-required b/roles/mirrormanager/backend/files/umdl-required index 5ed7b3e1a8..2a695d3417 100644 --- a/roles/mirrormanager/backend/files/umdl-required +++ b/roles/mirrormanager/backend/files/umdl-required @@ -30,6 +30,19 @@ elif [ "${1}" == "archive" ]; then CATEGORY="Archive" elif [ "${1}" == "codecs" ]; then CATEGORY="Codecs" + /usr/local/bin/lock-wrapper umdl-${1} "${SCANNER} --category \"Fedora ${CATEGORY}\"" + exit 0 +elif [ "${1}" == "centos" ]; then + CATEGORY="CentOS" + SCANNER="/usr/local/bin/scan-primary-mirror -c /etc/mirrormanager/scan-primary-mirror-centos.toml" + /usr/local/bin/lock-wrapper umdl-${1} "${SCANNER} --category \"${CATEGORY}\"" + if [ "$?" -eq "0" ]; then + echo -n "Finished umdl for ${CATEGORY} successfully at " + else + echo -n "${SCANNER} for ${CATEGORY} returned non-zero. Something failed. Please check umdl.log. " + fi + date + exit 0 fi if [ -e /var/run/mirrormanager/umdl-${1} ]; then @@ -39,13 +52,7 @@ else let LASTRUN=CURDATE-86400 fi -# FULLFILETIMELIST -> FFTL -if [ "${1}" == "codecs" ]; then - /usr/local/bin/lock-wrapper umdl-${1} "${SCANNER} --category \"Fedora ${CATEGORY}\"" - exit 0 -else - FFTL="/srv/pub/${1}/fullfiletimelist-${1}" -fi +FFTL="/srv/pub/${1}/fullfiletimelist-${1}" FILEDATE=`stat -c %Z ${FFTL} 2> /dev/null` if [ "$?" -eq "1" ]; then diff --git a/roles/mirrormanager/backend/tasks/main.yml b/roles/mirrormanager/backend/tasks/main.yml index 7db2b79e23..afd2a724d9 100644 --- a/roles/mirrormanager/backend/tasks/main.yml +++ b/roles/mirrormanager/backend/tasks/main.yml @@ -161,6 +161,13 @@ owner: mirrormanager group: mirrormanager mode: 0600 +- name: install scan-primary-mirror centos configuration file + template: + src: scan-primary-mirror-centos.toml + dest: /etc/mirrormanager/scan-primary-mirror-centos.toml + owner: mirrormanager + group: mirrormanager + mode: 0600 - name: create /srv/mirrorlist-server.git file: path=/srv/mirrorlist-server.git state=directory owner=mirrormanager group=mirrormanager mode=0755 - name: create /srv/scan-primary-mirror.git @@ -168,7 +175,7 @@ - name: checkout mirrorlist-server git: repo: https://github.com/adrianreber/mirrorlist-server.git - version: 3.0.4 + version: 3.0.5 dest: /srv/mirrorlist-server.git become: yes become_user: mirrormanager @@ -176,7 +183,7 @@ - name: checkout scan-primary-mirror git: repo: https://github.com/adrianreber/scan-primary-mirror.git - version: 0.1.1 + version: 0.2.0 dest: /srv/scan-primary-mirror.git become: yes become_user: mirrormanager diff --git a/roles/mirrormanager/backend/templates/scan-primary-mirror-centos.toml b/roles/mirrormanager/backend/templates/scan-primary-mirror-centos.toml new file mode 100644 index 0000000000..4cac003ffd --- /dev/null +++ b/roles/mirrormanager/backend/templates/scan-primary-mirror-centos.toml @@ -0,0 +1,41 @@ +common_rsync_options="--no-motd --timeout 14400" +max_propagation_days = 2 +max_stale_days = 3 +skip_repository_paths = ["SIGs"] + +[database] +url="postgresql://{{ mirrormanager_db_user }}:{{ mirrormanager_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_db_name }}" + +[[category]] +name="CentOS" +type="rsync" +url="rsync://mirror.stream.centos.org/CentOS-Stream-All" +checksum_base="http://mirror.stream.centos.org/" + +[[repository_mapping]] +regex="^\\d+(?:-stream)?/AppStream/" +prefix="centos-appstream" + +[[repository_mapping]] +regex="^\\d+(?:-stream)?/BaseOS/" +prefix="centos-baseos" + +[[repository_mapping]] +regex="^\\d+(?:-stream)?/CRB/" +prefix="centos-crb" + +[[repository_mapping]] +regex="^\\d+(?:-stream)?/HighAvailability/" +prefix="centos-highavailability" + +[[repository_mapping]] +regex="^\\d+(?:-stream)?/NFV/" +prefix="centos-nfv" + +[[repository_mapping]] +regex="^\\d+(?:-stream)?/RT/" +prefix="centos-rt" + +[[repository_mapping]] +regex="^\\d+(?:-stream)?/ResilientStorage/" +prefix="centos-resilientstorage" diff --git a/roles/mirrormanager/crawler/files/crawler.cron b/roles/mirrormanager/crawler/files/crawler.cron index a86b4c1068..b2fb775abf 100644 --- a/roles/mirrormanager/crawler/files/crawler.cron +++ b/roles/mirrormanager/crawler/files/crawler.cron @@ -33,3 +33,6 @@ # Category: 'Fedora Other'; once a day, 10 threads 0 14 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; /usr/bin/mm2_crawler --category="Fedora Other" --threads 9 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1 + +# Category: 'CentOS'; twice a day, 20 threads +0 5,11 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; /usr/bin/mm2_crawler --category="CentOS" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1