s3-mirror: avoid potential unexpected shell-expansion, dedup
Doing this has risk that '*' expands to directories:
$> x='echo *'
$> $x
bin boot dev etc home
So it is better to use bash array (we have bash shebang anyways):
$> x=( echo '*' )
$> "${x[@]}"
*
Also de-duplicate few things so it is easier to concentrate on the code.
This commit is contained in:
Pavel Raiskup
Pierre-Yves Chibon
parent
0fe0e70b92
commit
19f148c842
1 changed files with 106 additions and 182 deletions
|
|
@ -3,207 +3,131 @@
|
|||
# LGPL
|
||||
# Author: Rick Elrod <relrod@redhat.com>
|
||||
|
||||
# first run this command that syncs, but does not delete.
|
||||
base_cmd=(
|
||||
aws s3 sync
|
||||
--no-follow-symlinks
|
||||
--exclude "*/.snapshot/*"
|
||||
--exclude "*/source/*"
|
||||
--exclude "*/SRPMS/*"
|
||||
--exclude "*/debug/*"
|
||||
--exclude "*/beta/*"
|
||||
--exclude "*/ppc/*"
|
||||
--exclude "*/ppc64/*"
|
||||
--exclude "*/repoview/*"
|
||||
--exclude "*/Fedora/*"
|
||||
--exclude "*/EFI/*"
|
||||
--exclude "*/core/*"
|
||||
--exclude "*/extras/*"
|
||||
--exclude "*/LiveOS/*"
|
||||
--exclude "*/development/rawhide/*"
|
||||
--exclude "*/releases/8/*"
|
||||
--exclude "*/releases/9/*"
|
||||
--exclude "*/releases/10/*"
|
||||
--exclude "*/releases/11/*"
|
||||
--exclude "*/releases/12/*"
|
||||
--exclude "*/releases/13/*"
|
||||
--exclude "*/releases/14/*"
|
||||
--exclude "*/releases/15/*"
|
||||
--exclude "*/releases/16/*"
|
||||
--exclude "*/releases/17/*"
|
||||
--exclude "*/releases/18/*"
|
||||
--exclude "*/releases/19/*"
|
||||
--exclude "*/releases/20/*"
|
||||
--exclude "*/releases/21/*"
|
||||
--exclude "*/releases/22/*"
|
||||
--exclude "*/releases/23/*"
|
||||
--exclude "*/releases/24/*"
|
||||
--exclude "*/releases/25/*"
|
||||
--exclude "*/releases/26/*"
|
||||
--exclude "*/releases/27/*"
|
||||
--exclude "*/releases/28/*"
|
||||
--exclude "*/releases/29/*"
|
||||
--exclude "*/updates/8/*"
|
||||
--exclude "*/updates/9/*"
|
||||
--exclude "*/updates/10/*"
|
||||
--exclude "*/updates/11/*"
|
||||
--exclude "*/updates/12/*"
|
||||
--exclude "*/updates/13/*"
|
||||
--exclude "*/updates/14/*"
|
||||
--exclude "*/updates/15/*"
|
||||
--exclude "*/updates/16/*"
|
||||
--exclude "*/updates/17/*"
|
||||
--exclude "*/updates/18/*"
|
||||
--exclude "*/updates/19/*"
|
||||
--exclude "*/updates/20/*"
|
||||
--exclude "*/updates/21/*"
|
||||
--exclude "*/updates/22/*"
|
||||
--exclude "*/updates/23/*"
|
||||
--exclude "*/updates/24/*"
|
||||
--exclude "*/updates/25/*"
|
||||
--exclude "*/updates/26/*"
|
||||
--exclude "*/updates/27/*"
|
||||
--exclude "*/updates/28/*"
|
||||
--exclude "*/updates/29/*"
|
||||
--exclude "*/updates/testing/8/*"
|
||||
--exclude "*/updates/testing/9/*"
|
||||
--exclude "*/updates/testing/10/*"
|
||||
--exclude "*/updates/testing/11/*"
|
||||
--exclude "*/updates/testing/12/*"
|
||||
--exclude "*/updates/testing/13/*"
|
||||
--exclude "*/updates/testing/14/*"
|
||||
--exclude "*/updates/testing/15/*"
|
||||
--exclude "*/updates/testing/16/*"
|
||||
--exclude "*/updates/testing/17/*"
|
||||
--exclude "*/updates/testing/18/*"
|
||||
--exclude "*/updates/testing/19/*"
|
||||
--exclude "*/updates/testing/20/*"
|
||||
--exclude "*/updates/testing/21/*"
|
||||
--exclude "*/updates/testing/22/*"
|
||||
--exclude "*/updates/testing/23/*"
|
||||
--exclude "*/updates/testing/24/*"
|
||||
--exclude "*/updates/testing/25/*"
|
||||
--exclude "*/updates/testing/26/*"
|
||||
--exclude "*/updates/testing/27/*"
|
||||
--exclude "*/updates/testing/28/*"
|
||||
--exclude "*/updates/testing/29/*"
|
||||
)
|
||||
|
||||
# First run this command that syncs, but does not delete.
|
||||
# It also excludes repodata.
|
||||
CMD1="aws s3 sync \
|
||||
--exclude */repodata/* \
|
||||
--exclude */.snapshot/* \
|
||||
--exclude */source/* \
|
||||
--exclude */SRPMS/* \
|
||||
--exclude */debug/* \
|
||||
--exclude */beta/* \
|
||||
--exclude */ppc/* \
|
||||
--exclude */ppc64/* \
|
||||
--exclude */repoview/* \
|
||||
--exclude */Fedora/* \
|
||||
--exclude */EFI/* \
|
||||
--exclude */core/* \
|
||||
--exclude */extras/* \
|
||||
--exclude */LiveOS/* \
|
||||
--exclude */development/rawhide/* \
|
||||
--exclude */releases/8/* \
|
||||
--exclude */releases/9/* \
|
||||
--exclude */releases/10/* \
|
||||
--exclude */releases/11/* \
|
||||
--exclude */releases/12/* \
|
||||
--exclude */releases/13/* \
|
||||
--exclude */releases/14/* \
|
||||
--exclude */releases/15/* \
|
||||
--exclude */releases/16/* \
|
||||
--exclude */releases/17/* \
|
||||
--exclude */releases/18/* \
|
||||
--exclude */releases/19/* \
|
||||
--exclude */releases/20/* \
|
||||
--exclude */releases/21/* \
|
||||
--exclude */releases/22/* \
|
||||
--exclude */releases/23/* \
|
||||
--exclude */releases/24/* \
|
||||
--exclude */releases/25/* \
|
||||
--exclude */releases/26/* \
|
||||
--exclude */releases/27/* \
|
||||
--exclude */releases/28/* \
|
||||
--exclude */releases/29/* \
|
||||
--exclude */updates/8/* \
|
||||
--exclude */updates/9/* \
|
||||
--exclude */updates/10/* \
|
||||
--exclude */updates/11/* \
|
||||
--exclude */updates/12/* \
|
||||
--exclude */updates/13/* \
|
||||
--exclude */updates/14/* \
|
||||
--exclude */updates/15/* \
|
||||
--exclude */updates/16/* \
|
||||
--exclude */updates/17/* \
|
||||
--exclude */updates/18/* \
|
||||
--exclude */updates/19/* \
|
||||
--exclude */updates/20/* \
|
||||
--exclude */updates/21/* \
|
||||
--exclude */updates/22/* \
|
||||
--exclude */updates/23/* \
|
||||
--exclude */updates/24/* \
|
||||
--exclude */updates/25/* \
|
||||
--exclude */updates/26/* \
|
||||
--exclude */updates/27/* \
|
||||
--exclude */updates/28/* \
|
||||
--exclude */updates/29/* \
|
||||
--exclude */updates/testing/8/* \
|
||||
--exclude */updates/testing/9/* \
|
||||
--exclude */updates/testing/10/* \
|
||||
--exclude */updates/testing/11/* \
|
||||
--exclude */updates/testing/12/* \
|
||||
--exclude */updates/testing/13/* \
|
||||
--exclude */updates/testing/14/* \
|
||||
--exclude */updates/testing/15/* \
|
||||
--exclude */updates/testing/16/* \
|
||||
--exclude */updates/testing/17/* \
|
||||
--exclude */updates/testing/18/* \
|
||||
--exclude */updates/testing/19/* \
|
||||
--exclude */updates/testing/20/* \
|
||||
--exclude */updates/testing/21/* \
|
||||
--exclude */updates/testing/22/* \
|
||||
--exclude */updates/testing/23/* \
|
||||
--exclude */updates/testing/24/* \
|
||||
--exclude */updates/testing/25/* \
|
||||
--exclude */updates/testing/26/* \
|
||||
--exclude */updates/testing/27/* \
|
||||
--exclude */updates/testing/28/* \
|
||||
--exclude */updates/testing/29/* \
|
||||
--no-follow-symlinks \
|
||||
"
|
||||
#--dryrun \
|
||||
CMD1=( "${base_cmd[@]}" --exclude "*/repodata/*" )
|
||||
|
||||
# Next we run this command which also includes repodata.
|
||||
CMD2="aws s3 sync \
|
||||
--exclude */.snapshot/* \
|
||||
--exclude */source/* \
|
||||
--exclude */SRPMS/* \
|
||||
--exclude */debug/* \
|
||||
--exclude */beta/* \
|
||||
--exclude */ppc/* \
|
||||
--exclude */ppc64/* \
|
||||
--exclude */repoview/* \
|
||||
--exclude */Fedora/* \
|
||||
--exclude */EFI/* \
|
||||
--exclude */core/* \
|
||||
--exclude */extras/* \
|
||||
--exclude */LiveOS/* \
|
||||
--exclude */development/rawhide/* \
|
||||
--exclude */releases/8/* \
|
||||
--exclude */releases/9/* \
|
||||
--exclude */releases/10/* \
|
||||
--exclude */releases/11/* \
|
||||
--exclude */releases/12/* \
|
||||
--exclude */releases/13/* \
|
||||
--exclude */releases/14/* \
|
||||
--exclude */releases/15/* \
|
||||
--exclude */releases/16/* \
|
||||
--exclude */releases/17/* \
|
||||
--exclude */releases/18/* \
|
||||
--exclude */releases/19/* \
|
||||
--exclude */releases/20/* \
|
||||
--exclude */releases/21/* \
|
||||
--exclude */releases/22/* \
|
||||
--exclude */releases/23/* \
|
||||
--exclude */releases/24/* \
|
||||
--exclude */releases/25/* \
|
||||
--exclude */releases/26/* \
|
||||
--exclude */releases/27/* \
|
||||
--exclude */releases/28/* \
|
||||
--exclude */releases/29/* \
|
||||
--exclude */updates/8/* \
|
||||
--exclude */updates/9/* \
|
||||
--exclude */updates/10/* \
|
||||
--exclude */updates/11/* \
|
||||
--exclude */updates/12/* \
|
||||
--exclude */updates/13/* \
|
||||
--exclude */updates/14/* \
|
||||
--exclude */updates/15/* \
|
||||
--exclude */updates/16/* \
|
||||
--exclude */updates/17/* \
|
||||
--exclude */updates/18/* \
|
||||
--exclude */updates/19/* \
|
||||
--exclude */updates/20/* \
|
||||
--exclude */updates/21/* \
|
||||
--exclude */updates/22/* \
|
||||
--exclude */updates/23/* \
|
||||
--exclude */updates/24/* \
|
||||
--exclude */updates/25/* \
|
||||
--exclude */updates/26/* \
|
||||
--exclude */updates/27/* \
|
||||
--exclude */updates/28/* \
|
||||
--exclude */updates/29/* \
|
||||
--exclude */updates/testing/8/* \
|
||||
--exclude */updates/testing/9/* \
|
||||
--exclude */updates/testing/10/* \
|
||||
--exclude */updates/testing/11/* \
|
||||
--exclude */updates/testing/12/* \
|
||||
--exclude */updates/testing/13/* \
|
||||
--exclude */updates/testing/14/* \
|
||||
--exclude */updates/testing/15/* \
|
||||
--exclude */updates/testing/16/* \
|
||||
--exclude */updates/testing/17/* \
|
||||
--exclude */updates/testing/18/* \
|
||||
--exclude */updates/testing/19/* \
|
||||
--exclude */updates/testing/20/* \
|
||||
--exclude */updates/testing/21/* \
|
||||
--exclude */updates/testing/22/* \
|
||||
--exclude */updates/testing/23/* \
|
||||
--exclude */updates/testing/24/* \
|
||||
--exclude */updates/testing/25/* \
|
||||
--exclude */updates/testing/26/* \
|
||||
--exclude */updates/testing/27/* \
|
||||
--exclude */updates/testing/28/* \
|
||||
--exclude */updates/testing/29/* \
|
||||
--no-follow-symlinks \
|
||||
"
|
||||
#--dryrun \
|
||||
CMD2=( "${base_cmd[@]}" )
|
||||
|
||||
# Then we delete old RPMs and old metadata (but after invalidating caches).
|
||||
CMD3=( "${base_cmd[@]}" --delete )
|
||||
|
||||
S3_MIRROR=s3-mirror-us-west-1-02.fedoraproject.org
|
||||
DIST_ID=E2KJMDC0QAJDMU
|
||||
|
||||
# Sync EPEL
|
||||
#echo $CMD /srv/pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
|
||||
#echo $CMD /srv/pub/epel/ s3://$S3_MIRROR/pub/epel/
|
||||
echo "Starting EPEL sync at $(date)" >> /var/log/s3-mirror/timestamps
|
||||
$CMD1 /srv/pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
|
||||
$CMD2 /srv/pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
|
||||
"${CMD1[@]}" /srv/pub/epel/ "s3://$S3_MIRROR/pub/epel/"
|
||||
"${CMD2[@]}" /srv/pub/epel/ "s3://$S3_MIRROR/pub/epel/"
|
||||
echo "Ending EPEL sync at $(date)" >> /var/log/s3-mirror/timestamps
|
||||
for file in $(echo /srv/pub/epel/6/*/repodata/repomd.xml | sed 's#/srv##g'); do
|
||||
aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file"
|
||||
aws cloudfront create-invalidation --distribution-id "$DIST_ID" --paths "$file"
|
||||
done
|
||||
|
||||
for file in $(echo /srv/pub/epel/7/*/repodata/repomd.xml | sed 's#/srv##g'); do
|
||||
aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file"
|
||||
aws cloudfront create-invalidation --distribution-id "$DIST_ID" --paths "$file"
|
||||
done
|
||||
|
||||
for file in $(echo /srv/pub/epel/8/*/repodata/repomd.xml | sed 's#/srv##g'); do
|
||||
aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file"
|
||||
aws cloudfront create-invalidation --distribution-id "$DIST_ID" --paths "$file"
|
||||
done
|
||||
$CMD2 --delete /srv/pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
|
||||
"${CMD3[@]}" /srv/pub/epel/ "s3://$S3_MIRROR/pub/epel/"
|
||||
|
||||
# Sync Fedora
|
||||
#echo $CMD /srv/pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/
|
||||
#echo $CMD /srv/pub/fedora/ s3://$S3_MIRROR/pub/fedora/
|
||||
echo "Starting Fedora sync at $(date)" >> /var/log/s3-mirror/timestamps
|
||||
$CMD1 /srv/pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/
|
||||
$CMD2 /srv/pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/
|
||||
"${CMD1[@]}" /srv/pub/fedora/ "s3://$S3_MIRROR/pub/fedora/"
|
||||
"${CMD2[@]}" /srv/pub/fedora/ "s3://$S3_MIRROR/pub/fedora/"
|
||||
echo "Ending Fedora sync at $(date)" >> /var/log/s3-mirror/timestamps
|
||||
|
||||
for file in $(echo /srv/pub/fedora/linux/updates/*/*/*/repodata/repomd.xml | sed 's#/srv##g'); do
|
||||
aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file"
|
||||
aws cloudfront create-invalidation --distribution-id "$DIST_ID" --paths "$file"
|
||||
done
|
||||
$CMD2 --delete /srv/pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/
|
||||
"${CMD3[@]}" /srv/pub/fedora/ s3://$S3_MIRROR/pub/fedora/
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue