Infrastructure/ansible
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

most of s3-mirror role

Browse source 
Signed-off-by: Ricky Elrod <codeblock@fedoraproject.org>
This commit is contained in:
Rick Elrod 2015-07-27 19:14:17 +00:00
parent c0e7b9f1c5
commit 8c4931e0f9
8 changed files with 366 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

33
roles/s3-mirror/files/s3sync-logs Executable file
View file

@ -0,0 +1,33 @@
#!/bin/sh
s3cmd=/usr/bin/s3cmd
S3CMD_ARGS="sync \
--recursive \
--delete-removed \
--verbose \
"
targets="s3-mirror-us-east-1 s3-mirror-us-west-1 s3-mirror-us-west-2 s3-mirror-eu-west-1 s3-mirror-ap-northeast-1"
for t in ${targets}; do
mkdir -p /var/log/s3-mirror-logs/bucket/${t}
done
echo "=============================================================="
echo -n "Starting at "
date
time $s3cmd $S3CMD_ARGS s3://s3-mirror-logs.fedoraproject.org/s3-mirror-us-east-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-us-east-1/ &
time $s3cmd $S3CMD_ARGS s3://aws-logs-us-west-1.fedoraproject.org/s3-mirror-us-west-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-us-west-1/ &
time $s3cmd $S3CMD_ARGS s3://aws-logs-us-west-2.fedoraproject.org/s3-mirror-us-west-2/ /var/log/s3-mirror-logs/bucket/s3-mirror-us-west-2/ &
time $s3cmd $S3CMD_ARGS s3://aws-logs-eu-west-1.fedoraproject.org/s3-mirror-eu-west-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-eu-west-1/ &
time $s3cmd $S3CMD_ARGS s3://aws-logs-ap-northeast-1.fedoraproject.org/s3-mirror-s3-northeast-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-ap-northeast-1/ &
wait
# concatenate them for processing
rm /var/log/s3-mirror-logs/combined.log
find /var/log/s3-mirror-logs/bucket/ -type f -mtime -2 | sort -n | while read f; do
cat $f >> /var/log/s3-mirror-logs/combined.log
done
echo -n "Ending at "
date