#!/bin/sh s3cmd=/usr/bin/s3cmd S3CMD_ARGS="sync \ --recursive \ --delete-removed \ --verbose \ " targets="s3-mirror-us-east-1 s3-mirror-us-west-1 s3-mirror-us-west-2 s3-mirror-eu-west-1 s3-mirror-ap-northeast-1" for t in ${targets}; do mkdir -p /var/log/s3-mirror-logs/bucket/${t} done echo "==============================================================" echo -n "Starting at " date time $s3cmd $S3CMD_ARGS s3://s3-mirror-logs.fedoraproject.org/s3-mirror-us-east-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-us-east-1/ & time $s3cmd $S3CMD_ARGS s3://aws-logs-us-west-1.fedoraproject.org/s3-mirror-us-west-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-us-west-1/ & time $s3cmd $S3CMD_ARGS s3://aws-logs-us-west-2.fedoraproject.org/s3-mirror-us-west-2/ /var/log/s3-mirror-logs/bucket/s3-mirror-us-west-2/ & time $s3cmd $S3CMD_ARGS s3://aws-logs-eu-west-1.fedoraproject.org/s3-mirror-eu-west-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-eu-west-1/ & time $s3cmd $S3CMD_ARGS s3://aws-logs-ap-northeast-1.fedoraproject.org/s3-mirror-s3-northeast-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-ap-northeast-1/ & wait # concatenate them for processing rm /var/log/s3-mirror-logs/combined.log find /var/log/s3-mirror-logs/bucket/ -type f -mtime -2 | sort -n | while read f; do cat $f >> /var/log/s3-mirror-logs/combined.log done echo -n "Ending at " date