Infrastructure/ansible
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

s3-mirror: Split things into 2 sync runs, one without repodata and delete, the other with both.

Browse source 
In order to make sure the s3 mirror always is consistent, split out the commands
to make it sync without repodata and delete, then do another run with those, then finally
invalidate all the repodata/* files.

Some of the cron jobs are adjusted to allow the repodata invalidation.

Signed-off-by: Kevin Fenzi <kevin@scrye.com>
This commit is contained in:
Kevin Fenzi 2020-03-05 20:46:53 +00:00 • committed by Pierre-Yves Chibon
parent 4164c240d8
commit 899d1767eb
3 changed files with 140 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

42
roles/s3-mirror/files/s3-sync-path.sh
View file

@ -9,7 +9,30 @@ if [[ "$1" == "" ]] || [[ $1 != /pub* ]] || [[ $1 != */ ]]; then
exit 1
fi
CMD="aws s3 sync \
# first run do not delete anything or copy the repodata.
CMD1="aws s3 sync \
--exclude */repodata/* \
--exclude *.snapshot/* \
--exclude *source/* \
--exclude *SRPMS/* \
--exclude *debug/* \
--exclude *beta/* \
--exclude *ppc/* \
--exclude *ppc64/* \
--exclude *repoview/* \
--exclude *Fedora/* \
--exclude *EFI/* \
--exclude *core/* \
--exclude *extras/* \
--exclude *LiveOS/* \
--exclude *development/rawhide/* \
--no-follow-symlinks \
--only-show-errors \
"
#--dryrun \
# second we delete old content and also copy the repodata
CMD2="aws s3 sync \
--delete \
--exclude *.snapshot/* \
--exclude *source/* \
@ -32,19 +55,12 @@ CMD="aws s3 sync \
#echo "$CMD /srv$1 s3://s3-mirror-us-west-1-02.fedoraproject.org$1"
echo "Starting $1 sync at $(date)" >> /var/log/s3-mirror/timestamps
$CMD /srv$1 s3://s3-mirror-us-west-1-02.fedoraproject.org$1
echo "Ending $1 sync at $(date)" >> /var/log/s3-mirror/timestamps
$CMD1 /srv$1 s3://s3-mirror-us-west-1-02.fedoraproject.org$1
$CMD1 /srv$1/repodata/ s3://s3-mirror-us-west-1-02.fedoraproject.org$1/repodata/
# Always do the invalidations because they are quick and prevent issues
# depending on which path is synced.
for file in $(echo /srv/pub/epel/6/*/repodata/repomd.xml | sed 's#/srv##g'); do
aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file" > /dev/null
done
for file in $(echo /srv/pub/epel/7/*/repodata/repomd.xml | sed 's#/srv##g'); do
aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file" > /dev/null
done
for file in $(echo /srv/pub/fedora/linux/updates/*/*/*/repodata/repomd.xml | sed 's#/srv##g'); do
for file in $(echo $1/repodata/* ); do
aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file" > /dev/null
done
$CMD2 /srv$1 s3://s3-mirror-us-west-1-02.fedoraproject.org$1
echo "Ending $1 sync at $(date)" >> /var/log/s3-mirror/timestamps

114
roles/s3-mirror/files/s3.sh
View file

@ -3,8 +3,10 @@
# LGPL
# Author: Rick Elrod <relrod@redhat.com>
CMD="aws s3 sync \
--delete \
# first run this command that syncs, but does not delete.
# It also excludes repodata.
CMD1="aws s3 sync \
--exclude */repodata/* \
--exclude */.snapshot/* \
--exclude */source/* \
--exclude */SRPMS/* \
@ -38,6 +40,9 @@ CMD="aws s3 sync \
--exclude */releases/24/* \
--exclude */releases/25/* \
--exclude */releases/26/* \
--exclude */releases/27/* \
--exclude */releases/28/* \
--exclude */releases/29/* \
--exclude */updates/8/* \
--exclude */updates/9/* \
--exclude */updates/10/* \
@ -57,6 +62,9 @@ CMD="aws s3 sync \
--exclude */updates/24/* \
--exclude */updates/25/* \
--exclude */updates/26/* \
--exclude */updates/27/* \
--exclude */updates/28/* \
--exclude */updates/29/* \
--exclude */updates/testing/8/* \
--exclude */updates/testing/9/* \
--exclude */updates/testing/10/* \
@ -76,6 +84,95 @@ CMD="aws s3 sync \
--exclude */updates/testing/24/* \
--exclude */updates/testing/25/* \
--exclude */updates/testing/26/* \
--exclude */updates/testing/27/* \
--exclude */updates/testing/28/* \
--exclude */updates/testing/29/* \
--no-follow-symlinks \
"
#--dryrun \
# Next we run this command which also includes repodata.
CMD2="aws s3 sync \
--exclude */.snapshot/* \
--exclude */source/* \
--exclude */SRPMS/* \
--exclude */debug/* \
--exclude */beta/* \
--exclude */ppc/* \
--exclude */ppc64/* \
--exclude */repoview/* \
--exclude */Fedora/* \
--exclude */EFI/* \
--exclude */core/* \
--exclude */extras/* \
--exclude */LiveOS/* \
--exclude */development/rawhide/* \
--exclude */releases/8/* \
--exclude */releases/9/* \
--exclude */releases/10/* \
--exclude */releases/11/* \
--exclude */releases/12/* \
--exclude */releases/13/* \
--exclude */releases/14/* \
--exclude */releases/15/* \
--exclude */releases/16/* \
--exclude */releases/17/* \
--exclude */releases/18/* \
--exclude */releases/19/* \
--exclude */releases/20/* \
--exclude */releases/21/* \
--exclude */releases/22/* \
--exclude */releases/23/* \
--exclude */releases/24/* \
--exclude */releases/25/* \
--exclude */releases/26/* \
--exclude */releases/27/* \
--exclude */releases/28/* \
--exclude */releases/29/* \
--exclude */updates/8/* \
--exclude */updates/9/* \
--exclude */updates/10/* \
--exclude */updates/11/* \
--exclude */updates/12/* \
--exclude */updates/13/* \
--exclude */updates/14/* \
--exclude */updates/15/* \
--exclude */updates/16/* \
--exclude */updates/17/* \
--exclude */updates/18/* \
--exclude */updates/19/* \
--exclude */updates/20/* \
--exclude */updates/21/* \
--exclude */updates/22/* \
--exclude */updates/23/* \
--exclude */updates/24/* \
--exclude */updates/25/* \
--exclude */updates/26/* \
--exclude */updates/27/* \
--exclude */updates/28/* \
--exclude */updates/29/* \
--exclude */updates/testing/8/* \
--exclude */updates/testing/9/* \
--exclude */updates/testing/10/* \
--exclude */updates/testing/11/* \
--exclude */updates/testing/12/* \
--exclude */updates/testing/13/* \
--exclude */updates/testing/14/* \
--exclude */updates/testing/15/* \
--exclude */updates/testing/16/* \
--exclude */updates/testing/17/* \
--exclude */updates/testing/18/* \
--exclude */updates/testing/19/* \
--exclude */updates/testing/20/* \
--exclude */updates/testing/21/* \
--exclude */updates/testing/22/* \
--exclude */updates/testing/23/* \
--exclude */updates/testing/24/* \
--exclude */updates/testing/25/* \
--exclude */updates/testing/26/* \
--exclude */updates/testing/27/* \
--exclude */updates/testing/28/* \
--exclude */updates/testing/29/* \
--no-follow-symlinks \
"
#--dryrun \
@ -83,9 +180,9 @@ CMD="aws s3 sync \
# Sync EPEL
#echo $CMD /srv/pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
echo "Starting EPEL sync at $(date)" >> /var/log/s3-mirror/timestamps
$CMD /srv/pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
$CMD1 /srv/pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
$CMD2 /srv/pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
echo "Ending EPEL sync at $(date)" >> /var/log/s3-mirror/timestamps
for file in $(echo /srv/pub/epel/6/*/repodata/repomd.xml | sed 's#/srv##g'); do
aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file"
done
@ -94,12 +191,19 @@ for file in $(echo /srv/pub/epel/7/*/repodata/repomd.xml | sed 's#/srv##g'); do
aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file"
done
for file in $(echo /srv/pub/epel/8/*/repodata/repomd.xml | sed 's#/srv##g'); do
aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file"
done
$CMD2 --delete /srv/pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
# Sync Fedora
#echo $CMD /srv/pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/
echo "Starting Fedora sync at $(date)" >> /var/log/s3-mirror/timestamps
$CMD /srv/pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/
$CMD1 /srv/pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/
$CMD2 /srv/pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/
echo "Ending Fedora sync at $(date)" >> /var/log/s3-mirror/timestamps
for file in $(echo /srv/pub/fedora/linux/updates/*/*/*/repodata/repomd.xml | sed 's#/srv##g'); do
aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file"
done
$CMD2 --delete /srv/pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/

4
roles/s3-mirror/tasks/main.yml
View file

@ -69,7 +69,7 @@
- name: s3sync cron - updates for current
cron: name="s3sync-updates-current" minute="0" hour="3,9,15,21" user="s3-mirror"
job='/usr/local/bin/lock-wrapper s3sync-updates-current "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/updates/{{ FedoraCycleNumber|int }}/" 2>&1 | /usr/local/bin/nag-once s3-updates-current.sh 1d 2>&1'
job='/usr/local/bin/lock-wrapper s3sync-updates-current "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/updates/{{ FedoraCycleNumber|int }}/Everything/x86_64/os" 2>&1 | /usr/local/bin/nag-once s3-updates-current.sh 1d 2>&1'
cron_file=s3-updates-current.sh
when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
tags:
@ -95,7 +95,7 @@
- name: s3sync cron - updates for current-1
cron: name="s3sync-updates-previous" minute="30" hour="0,6,12,18" user="s3-mirror"
job='/usr/local/bin/lock-wrapper s3sync-updates-previous "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/updates/{{ FedoraCycleNumber|int - 1 }}/" 2>&1 | /usr/local/bin/nag-once s3-updates-previous.sh 1d 2>&1'
job='/usr/local/bin/lock-wrapper s3sync-updates-previous "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/updates/{{ FedoraCycleNumber|int - 1 }}/Everything/x86_64/" 2>&1 | /usr/local/bin/nag-once s3-updates-previous.sh 1d 2>&1'
cron_file=s3-updates-previous.sh
when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
tags: