33 lines
1.3 KiB
Bash
Executable file
33 lines
1.3 KiB
Bash
Executable file
#!/bin/sh
|
|
|
|
s3cmd=/usr/bin/s3cmd
|
|
|
|
S3CMD_ARGS="sync \
|
|
--recursive \
|
|
--delete-removed \
|
|
--verbose \
|
|
"
|
|
targets="s3-mirror-us-east-1 s3-mirror-us-west-1 s3-mirror-us-west-2 s3-mirror-eu-west-1 s3-mirror-ap-northeast-1"
|
|
for t in ${targets}; do
|
|
mkdir -p /var/log/s3-mirror-logs/bucket/${t}
|
|
done
|
|
|
|
echo "=============================================================="
|
|
echo -n "Starting at "
|
|
date
|
|
|
|
time $s3cmd $S3CMD_ARGS s3://s3-mirror-logs.fedoraproject.org/s3-mirror-us-east-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-us-east-1/ &
|
|
time $s3cmd $S3CMD_ARGS s3://aws-logs-us-west-1.fedoraproject.org/s3-mirror-us-west-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-us-west-1/ &
|
|
time $s3cmd $S3CMD_ARGS s3://aws-logs-us-west-2.fedoraproject.org/s3-mirror-us-west-2/ /var/log/s3-mirror-logs/bucket/s3-mirror-us-west-2/ &
|
|
time $s3cmd $S3CMD_ARGS s3://aws-logs-eu-west-1.fedoraproject.org/s3-mirror-eu-west-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-eu-west-1/ &
|
|
time $s3cmd $S3CMD_ARGS s3://aws-logs-ap-northeast-1.fedoraproject.org/s3-mirror-s3-northeast-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-ap-northeast-1/ &
|
|
wait
|
|
|
|
# concatenate them for processing
|
|
rm /var/log/s3-mirror-logs/combined.log
|
|
find /var/log/s3-mirror-logs/bucket/ -type f -mtime -2 | sort -n | while read f; do
|
|
cat $f >> /var/log/s3-mirror-logs/combined.log
|
|
done
|
|
|
|
echo -n "Ending at "
|
|
date
|