Infrastructure/ansible
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

Revert "s3-mirror: Split things into 2 sync runs, one without repodata and delete, the other with both."

Browse source 
This reverts commit 626c976f8ce616750a56cd0de7847beeaf3924f7.
This commit is contained in:
Kevin Fenzi 2020-03-06 23:16:57 +00:00 • committed by Pierre-Yves Chibon
parent be03aa4abf
commit 76fb2e0871
3 changed files with 20 additions and 140 deletions
Download patch file Download diff file Expand all files Collapse all files

42
roles/s3-mirror/files/s3-sync-path.sh
View file

@ -9,30 +9,7 @@ if [[ "$1" == "" ]] || [[ $1 != /pub* ]] || [[ $1 != */ ]]; then
exit 1
fi
# first run do not delete anything or copy the repodata.
CMD1="aws s3 sync \
--exclude */repodata/* \
--exclude *.snapshot/* \
--exclude *source/* \
--exclude *SRPMS/* \
--exclude *debug/* \
--exclude *beta/* \
--exclude *ppc/* \
--exclude *ppc64/* \
--exclude *repoview/* \
--exclude *Fedora/* \
--exclude *EFI/* \
--exclude *core/* \
--exclude *extras/* \
--exclude *LiveOS/* \
--exclude *development/rawhide/* \
--no-follow-symlinks \
--only-show-errors \
"
#--dryrun \
# second we delete old content and also copy the repodata
CMD2="aws s3 sync \
CMD="aws s3 sync \
--delete \
--exclude *.snapshot/* \
--exclude *source/* \
@ -55,12 +32,19 @@ CMD2="aws s3 sync \
#echo "$CMD /srv$1 s3://s3-mirror-us-west-1-02.fedoraproject.org$1"
echo "Starting $1 sync at $(date)" >> /var/log/s3-mirror/timestamps
$CMD1 /srv$1 s3://s3-mirror-us-west-1-02.fedoraproject.org$1
$CMD1 /srv$1/repodata/ s3://s3-mirror-us-west-1-02.fedoraproject.org$1/repodata/
$CMD /srv$1 s3://s3-mirror-us-west-1-02.fedoraproject.org$1
echo "Ending $1 sync at $(date)" >> /var/log/s3-mirror/timestamps
# Always do the invalidations because they are quick and prevent issues
# depending on which path is synced.
for file in $(echo $1/repodata/* ); do
for file in $(echo /srv/pub/epel/6/*/repodata/repomd.xml | sed 's#/srv##g'); do
aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file" > /dev/null
done
for file in $(echo /srv/pub/epel/7/*/repodata/repomd.xml | sed 's#/srv##g'); do
aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file" > /dev/null
done
for file in $(echo /srv/pub/fedora/linux/updates/*/*/*/repodata/repomd.xml | sed 's#/srv##g'); do
aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file" > /dev/null
done
$CMD2 /srv$1 s3://s3-mirror-us-west-1-02.fedoraproject.org$1
echo "Ending $1 sync at $(date)" >> /var/log/s3-mirror/timestamps

114
roles/s3-mirror/files/s3.sh
View file

@ -3,10 +3,8 @@
# LGPL
# Author: Rick Elrod <relrod@redhat.com>
# first run this command that syncs, but does not delete.
# It also excludes repodata.
CMD1="aws s3 sync \
--exclude */repodata/* \
CMD="aws s3 sync \
--delete \
--exclude */.snapshot/* \
--exclude */source/* \
--exclude */SRPMS/* \
@ -40,9 +38,6 @@ CMD1="aws s3 sync \
--exclude */releases/24/* \
--exclude */releases/25/* \
--exclude */releases/26/* \
--exclude */releases/27/* \
--exclude */releases/28/* \
--exclude */releases/29/* \
--exclude */updates/8/* \
--exclude */updates/9/* \
--exclude */updates/10/* \
@ -62,9 +57,6 @@ CMD1="aws s3 sync \
--exclude */updates/24/* \
--exclude */updates/25/* \
--exclude */updates/26/* \
--exclude */updates/27/* \
--exclude */updates/28/* \
--exclude */updates/29/* \
--exclude */updates/testing/8/* \
--exclude */updates/testing/9/* \
--exclude */updates/testing/10/* \
@ -84,95 +76,6 @@ CMD1="aws s3 sync \
--exclude */updates/testing/24/* \
--exclude */updates/testing/25/* \
--exclude */updates/testing/26/* \
--exclude */updates/testing/27/* \
--exclude */updates/testing/28/* \
--exclude */updates/testing/29/* \
--no-follow-symlinks \
"
#--dryrun \
# Next we run this command which also includes repodata.
CMD2="aws s3 sync \
--exclude */.snapshot/* \
--exclude */source/* \
--exclude */SRPMS/* \
--exclude */debug/* \
--exclude */beta/* \
--exclude */ppc/* \
--exclude */ppc64/* \
--exclude */repoview/* \
--exclude */Fedora/* \
--exclude */EFI/* \
--exclude */core/* \
--exclude */extras/* \
--exclude */LiveOS/* \
--exclude */development/rawhide/* \
--exclude */releases/8/* \
--exclude */releases/9/* \
--exclude */releases/10/* \
--exclude */releases/11/* \
--exclude */releases/12/* \
--exclude */releases/13/* \
--exclude */releases/14/* \
--exclude */releases/15/* \
--exclude */releases/16/* \
--exclude */releases/17/* \
--exclude */releases/18/* \
--exclude */releases/19/* \
--exclude */releases/20/* \
--exclude */releases/21/* \
--exclude */releases/22/* \
--exclude */releases/23/* \
--exclude */releases/24/* \
--exclude */releases/25/* \
--exclude */releases/26/* \
--exclude */releases/27/* \
--exclude */releases/28/* \
--exclude */releases/29/* \
--exclude */updates/8/* \
--exclude */updates/9/* \
--exclude */updates/10/* \
--exclude */updates/11/* \
--exclude */updates/12/* \
--exclude */updates/13/* \
--exclude */updates/14/* \
--exclude */updates/15/* \
--exclude */updates/16/* \
--exclude */updates/17/* \
--exclude */updates/18/* \
--exclude */updates/19/* \
--exclude */updates/20/* \
--exclude */updates/21/* \
--exclude */updates/22/* \
--exclude */updates/23/* \
--exclude */updates/24/* \
--exclude */updates/25/* \
--exclude */updates/26/* \
--exclude */updates/27/* \
--exclude */updates/28/* \
--exclude */updates/29/* \
--exclude */updates/testing/8/* \
--exclude */updates/testing/9/* \
--exclude */updates/testing/10/* \
--exclude */updates/testing/11/* \
--exclude */updates/testing/12/* \
--exclude */updates/testing/13/* \
--exclude */updates/testing/14/* \
--exclude */updates/testing/15/* \
--exclude */updates/testing/16/* \
--exclude */updates/testing/17/* \
--exclude */updates/testing/18/* \
--exclude */updates/testing/19/* \
--exclude */updates/testing/20/* \
--exclude */updates/testing/21/* \
--exclude */updates/testing/22/* \
--exclude */updates/testing/23/* \
--exclude */updates/testing/24/* \
--exclude */updates/testing/25/* \
--exclude */updates/testing/26/* \
--exclude */updates/testing/27/* \
--exclude */updates/testing/28/* \
--exclude */updates/testing/29/* \
--no-follow-symlinks \
"
#--dryrun \
@ -180,9 +83,9 @@ CMD2="aws s3 sync \
# Sync EPEL
#echo $CMD /srv/pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
echo "Starting EPEL sync at $(date)" >> /var/log/s3-mirror/timestamps
$CMD1 /srv/pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
$CMD2 /srv/pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
$CMD /srv/pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
echo "Ending EPEL sync at $(date)" >> /var/log/s3-mirror/timestamps
for file in $(echo /srv/pub/epel/6/*/repodata/repomd.xml | sed 's#/srv##g'); do
aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file"
done
@ -191,19 +94,12 @@ for file in $(echo /srv/pub/epel/7/*/repodata/repomd.xml | sed 's#/srv##g'); do
aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file"
done
for file in $(echo /srv/pub/epel/8/*/repodata/repomd.xml | sed 's#/srv##g'); do
aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file"
done
$CMD2 --delete /srv/pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
# Sync Fedora
#echo $CMD /srv/pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/
echo "Starting Fedora sync at $(date)" >> /var/log/s3-mirror/timestamps
$CMD1 /srv/pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/
$CMD2 /srv/pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/
$CMD /srv/pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/
echo "Ending Fedora sync at $(date)" >> /var/log/s3-mirror/timestamps
for file in $(echo /srv/pub/fedora/linux/updates/*/*/*/repodata/repomd.xml | sed 's#/srv##g'); do
aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file"
done
$CMD2 --delete /srv/pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/

4
roles/s3-mirror/tasks/main.yml
View file

@ -69,7 +69,7 @@
- name: s3sync cron - updates for current
cron: name="s3sync-updates-current" minute="0" hour="3,9,15,21" user="s3-mirror"
job='/usr/local/bin/lock-wrapper s3sync-updates-current "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/updates/{{ FedoraCycleNumber|int }}/Everything/x86_64/os" 2>&1 | /usr/local/bin/nag-once s3-updates-current.sh 1d 2>&1'
job='/usr/local/bin/lock-wrapper s3sync-updates-current "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/updates/{{ FedoraCycleNumber|int }}/" 2>&1 | /usr/local/bin/nag-once s3-updates-current.sh 1d 2>&1'
cron_file=s3-updates-current.sh
when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
tags:
@ -95,7 +95,7 @@
- name: s3sync cron - updates for current-1
cron: name="s3sync-updates-previous" minute="30" hour="0,6,12,18" user="s3-mirror"
job='/usr/local/bin/lock-wrapper s3sync-updates-previous "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/updates/{{ FedoraCycleNumber|int - 1 }}/Everything/x86_64/" 2>&1 | /usr/local/bin/nag-once s3-updates-previous.sh 1d 2>&1'
job='/usr/local/bin/lock-wrapper s3sync-updates-previous "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/updates/{{ FedoraCycleNumber|int - 1 }}/" 2>&1 | /usr/local/bin/nag-once s3-updates-previous.sh 1d 2>&1'
cron_file=s3-updates-previous.sh
when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
tags: