s3-mirror: Split things into 2 sync runs, one without repodata and delete, the other with both.

In order to make sure the s3 mirror always is consistent, split out the commands
to make it sync without repodata and delete, then do another run with those, then finally
invalidate all the repodata/* files.

Some of the cron jobs are adjusted to allow the repodata invalidation.

Signed-off-by: Kevin Fenzi <kevin@scrye.com>

roles/s3-mirror/files/s3-sync-path.sh

@@ -9,7 +9,30 @@ if [[ "$1" == "" ]] || [[ $1 != /pub* ]] || [[ $1 != */ ]]; then
   exit 1
 fi
 
-CMD="aws s3 sync                   \
+# first run do not delete anything or copy the repodata.
+CMD1="aws s3 sync                   \
+  --exclude */repodata/*         \
+  --exclude *.snapshot/*          \
+  --exclude *source/*             \
+  --exclude *SRPMS/*              \
+  --exclude *debug/*              \
+  --exclude *beta/*               \
+  --exclude *ppc/*                \
+  --exclude *ppc64/*              \
+  --exclude *repoview/*           \
+  --exclude *Fedora/*             \
+  --exclude *EFI/*                \
+  --exclude *core/*               \
+  --exclude *extras/*             \
+  --exclude *LiveOS/*             \
+  --exclude *development/rawhide/* \
+  --no-follow-symlinks            \
+  --only-show-errors              \
+  "
+  #--dryrun                         \
+
+# second we delete old content and also copy the repodata
+CMD2="aws s3 sync                   \
   --delete                         \
   --exclude *.snapshot/*          \
   --exclude *source/*             \
@@ -32,19 +55,12 @@ CMD="aws s3 sync                   \
 
 #echo "$CMD /srv$1 s3://s3-mirror-us-west-1-02.fedoraproject.org$1"
 echo "Starting $1 sync at $(date)" >> /var/log/s3-mirror/timestamps
-$CMD /srv$1 s3://s3-mirror-us-west-1-02.fedoraproject.org$1
-echo "Ending $1 sync at $(date)" >> /var/log/s3-mirror/timestamps
-
+$CMD1 /srv$1 s3://s3-mirror-us-west-1-02.fedoraproject.org$1
+$CMD1 /srv$1/repodata/ s3://s3-mirror-us-west-1-02.fedoraproject.org$1/repodata/
 # Always do the invalidations because they are quick and prevent issues
 # depending on which path is synced.
-for file in $(echo /srv/pub/epel/6/*/repodata/repomd.xml | sed 's#/srv##g'); do
-  aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file" > /dev/null
-done
-
-for file in $(echo /srv/pub/epel/7/*/repodata/repomd.xml | sed 's#/srv##g'); do
-  aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file" > /dev/null
-done
-
-for file in $(echo /srv/pub/fedora/linux/updates/*/*/*/repodata/repomd.xml | sed 's#/srv##g'); do
+for file in $(echo $1/repodata/* ); do
   aws cloudfront create-invalidation --distribution-id E2KJMDC0QAJDMU --paths "$file" > /dev/null
 done
+$CMD2 /srv$1 s3://s3-mirror-us-west-1-02.fedoraproject.org$1
+echo "Ending $1 sync at $(date)" >> /var/log/s3-mirror/timestamps