infrastructure/ansible
1
0
Fork 0
Code Pull requests 7 Activity Actions

s3-mirror: sync s3-sync-path script with ideas from s3.sh

Browse source 
1. sync everything except for repomd.xml
2. then sync repomd.xml files only, and invalidate caches
3. gently wait a bit to give current downloads a chance
4. delete outdated RPMs and metadata, shouldn't be needed

Also make the sleep/cache configurable.
This commit is contained in:
Pavel Raiskup 2020-03-27 09:16:12 +01:00 • committed by Pierre-Yves Chibon
parent 1936dcfd09
commit f4538051df
2 changed files with 65 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

100
roles/s3-mirror/files/s3-sync-path.sh
View file

@ -9,58 +9,64 @@ if [[ "$1" == "" ]] || [[ $1 != /pub* ]] || [[ $1 != */ ]]; then
exit 1 exit 1
fi fi
# first run do not delete anything or copy the repodata. aws_sync=( aws s3 sync --no-follow-symlinks )
CMD1="aws s3 sync \
--exclude */repodata/* \
--exclude *.snapshot/* \
--exclude *source/* \
--exclude *SRPMS/* \
--exclude *debug/* \
--exclude *beta/* \
--exclude *ppc/* \
--exclude *ppc64/* \
--exclude *repoview/* \
--exclude *Fedora/* \
--exclude *EFI/* \
--exclude *core/* \
--exclude *extras/* \
--exclude *LiveOS/* \
--exclude *development/rawhide/* \
--no-follow-symlinks \
--only-show-errors \
"
#--dryrun \
# second we delete old content and also copy the repodata # first run do not delete anything or copy the repodata.
CMD2="aws s3 sync \ exclude=(
--delete \ --exclude "*.snapshot/*"
--exclude *.snapshot/* \ --exclude "*source/*"
--exclude *source/* \ --exclude "*SRPMS/*"
--exclude *SRPMS/* \ --exclude "*debug/*"
--exclude *debug/* \ --exclude "*beta/*"
--exclude *beta/* \ --exclude "*ppc/*"
--exclude *ppc/* \ --exclude "*ppc64/*"
--exclude *ppc64/* \ --exclude "*repoview/*"
--exclude *repoview/* \ --exclude "*Fedora/*"
--exclude *Fedora/* \ --exclude "*EFI/*"
--exclude *EFI/* \ --exclude "*core/*"
--exclude *core/* \ --exclude "*extras/*"
--exclude *extras/* \ --exclude "*LiveOS/*"
--exclude *LiveOS/* \ --exclude "*development/rawhide/*"
--exclude *development/rawhide/* \ --only-show-errors
--no-follow-symlinks \ )
--only-show-errors \
" S3_MIRROR=s3-mirror-us-west-1-02.fedoraproject.org
#--dryrun \ DIST_ID=E2KJMDC0QAJDMU
MAX_CACHE_SEC=60
DNF_GENTLY_TIMEOUT=120
# First run this command that syncs, but does not delete.
# It also excludes repomd.xml.
CMD1=( "${aws_sync[@]}" "${excludes[@]}" --exclude "*/repomd.xml" )
# Next we run this command which syncs repomd.xml files. Include must precede
# the large set of excludes. Make sure that the 'max-age' isn't too large so
# we know that we can start removing old data ASAP.
CMD2=( "${aws_sync[@]}" --exclude "*" --include "*/repomd.xml" "${excludes[@]}"
--cache-control "max-age=$MAX_CACHE_SEC" )
# Then we delete old RPMs and old metadata (but after invalidating caches).
CMD3=( "${aws_sync[@]}" "${excludes[@]}" --delete )
#echo "$CMD /srv$1 s3://s3-mirror-us-west-1-02.fedoraproject.org$1" #echo "$CMD /srv$1 s3://s3-mirror-us-west-1-02.fedoraproject.org$1"
echo "Starting $1 sync at $(date)" >> /var/log/s3-mirror/timestamps echo "Starting $1 sync at $(date)" >> /var/log/s3-mirror/timestamps
$CMD1 /srv$1 s3://s3-mirror-us-west-1-02.fedoraproject.org$1 "${CMD1[@]}" "/srv$1" "s3://$S3_MIRROR$1"
$CMD1 /srv$1/repodata/ s3://s3-mirror-us-west-1-02.fedoraproject.org$1/repodata/ "${CMD2[@]}" "/srv$1" "s3://$S3_MIRROR$1"
# Always do the invalidations because they are quick and prevent issues # Always do the invalidations because they are quick and prevent issues
# depending on which path is synced. # depending on which path is synced.
for file in $(echo $1/repodata/* ); do for file in $(echo $1/repodata/repomd.xml ); do
aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file" > /dev/null aws cloudfront create-invalidation --distribution-id $DIST_ID --paths "$file" > /dev/null
done done
$CMD2 /srv$1 s3://s3-mirror-us-west-1-02.fedoraproject.org$1
SLEEP=$(( MAX_CACHE_SEC + DNF_GENTLY_TIMEOUT ))
echo "Ready $1 sync, giving dnf downloads ${SLEEP}s before delete, at $(date)" >> /var/log/s3-mirror/timestamps
# Consider some DNF processes started downloading metadata before we invalidated
# caches, and started with outdated repomd.xml file. Give it few more seconds
# so they have chance to download the rest of metadata and RPMs.
sleep $SLEEP
"${CMD3[@]}" "/srv$1" "s3://$S3_MIRROR$1"
echo "Ending $1 sync at $(date)" >> /var/log/s3-mirror/timestamps echo "Ending $1 sync at $(date)" >> /var/log/s3-mirror/timestamps

19
roles/s3-mirror/files/s3.sh
View file

@ -88,6 +88,11 @@ excludes=(
--exclude "*/updates/testing/29/*" --exclude "*/updates/testing/29/*"
) )
S3_MIRROR=s3-mirror-us-west-1-02.fedoraproject.org
DIST_ID=E2KJMDC0QAJDMU
MAX_CACHE_SEC=60
DNF_GENTLY_TIMEOUT=120
# First run this command that syncs, but does not delete. # First run this command that syncs, but does not delete.
# It also excludes repomd.xml. # It also excludes repomd.xml.
CMD1=( "${aws_sync[@]}" "${excludes[@]}" --exclude "*/repomd.xml" ) CMD1=( "${aws_sync[@]}" "${excludes[@]}" --exclude "*/repomd.xml" )
@ -95,14 +100,12 @@ CMD1=( "${aws_sync[@]}" "${excludes[@]}" --exclude "*/repomd.xml" )
# Next we run this command which syncs repomd.xml files. Include must precede # Next we run this command which syncs repomd.xml files. Include must precede
# the large set of excludes. Make sure that the 'max-age' isn't too large so # the large set of excludes. Make sure that the 'max-age' isn't too large so
# we know that we can start removing old data ASAP. # we know that we can start removing old data ASAP.
CMD2=( "${aws_sync[@]}" --exclude "*" --include "*/repomd.xml" "${excludes[@]}" --cache-control max-age=300 ) CMD2=( "${aws_sync[@]}" --exclude "*" --include "*/repomd.xml" "${excludes[@]}"
--cache-control "max-age=$MAX_CACHE_SEC" )
# Then we delete old RPMs and old metadata (but after invalidating caches). # Then we delete old RPMs and old metadata (but after invalidating caches).
CMD3=( "${aws_sync[@]}" "${excludes[@]}" --delete ) CMD3=( "${aws_sync[@]}" "${excludes[@]}" --delete )
S3_MIRROR=s3-mirror-us-west-1-02.fedoraproject.org
DIST_ID=E2KJMDC0QAJDMU
# Sync EPEL # Sync EPEL
#echo $CMD /srv/pub/epel/ s3://$S3_MIRROR/pub/epel/ #echo $CMD /srv/pub/epel/ s3://$S3_MIRROR/pub/epel/
echo "Starting EPEL sync at $(date)" >> /var/log/s3-mirror/timestamps echo "Starting EPEL sync at $(date)" >> /var/log/s3-mirror/timestamps
@ -132,10 +135,12 @@ for file in $(echo /srv/pub/fedora/linux/updates/*/*/*/repodata/repomd.xml | sed
aws cloudfront create-invalidation --distribution-id "$DIST_ID" --paths "$file" aws cloudfront create-invalidation --distribution-id "$DIST_ID" --paths "$file"
done done
SLEEP=$(( MAX_CACHE_SEC + DNF_GENTLY_TIMEOUT ))
# Consider some DNF processes started downloading metadata before we invalidated # Consider some DNF processes started downloading metadata before we invalidated
# caches, and started with outdated repomd.xml file. Give it 10 minutes so they # caches, and started with outdated repomd.xml file. Give it few more seconds
# have chance to download the rest of metadata and RPMs. # so they have chance to download the rest of metadata and RPMs.
sleep 600 sleep $SLEEP
"${CMD3[@]}" /srv/pub/epel/ "s3://$S3_MIRROR/pub/epel/" "${CMD3[@]}" /srv/pub/epel/ "s3://$S3_MIRROR/pub/epel/"
"${CMD3[@]}" /srv/pub/fedora/ s3://$S3_MIRROR/pub/fedora/ "${CMD3[@]}" /srv/pub/fedora/ s3://$S3_MIRROR/pub/fedora/