attempt at cron-ify-ing new s3 sync
Signed-off-by: Rick Elrod <relrod@redhat.com>
This commit is contained in:
Rick Elrod
parent
ca306d59d9
commit
7984e52034
9 changed files with 135 additions and 247 deletions
3
roles/s3-mirror/files/config
Normal file
3
roles/s3-mirror/files/config
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
[default]
|
||||
region = us-west-2
|
||||
output = text
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
.snapshot/
|
||||
source/
|
||||
SRPMS/
|
||||
debug/
|
||||
beta/
|
||||
ppc/
|
||||
ppc64/
|
||||
repoview/
|
||||
Fedora/
|
||||
Live/
|
||||
isolinux/
|
||||
images/
|
||||
EFI/
|
||||
drpms/
|
||||
core/
|
||||
extras/
|
||||
LiveOS/
|
||||
development/
|
||||
releases/test/
|
||||
releases/8/
|
||||
releases/9/
|
||||
releases/10/
|
||||
releases/11/
|
||||
releases/12/
|
||||
releases/13/
|
||||
releases/14/
|
||||
releases/15/
|
||||
releases/16/
|
||||
updates/8/
|
||||
updates/9/
|
||||
updates/10/
|
||||
updates/11/
|
||||
updates/12/
|
||||
updates/13/
|
||||
updates/14/
|
||||
updates/15/
|
||||
updates/16/
|
||||
updates/testing/8/
|
||||
updates/testing/9/
|
||||
updates/testing/10/
|
||||
updates/testing/11/
|
||||
updates/testing/12/
|
||||
updates/testing/13/
|
||||
updates/testing/14/
|
||||
updates/testing/15/
|
||||
updates/testing/16/
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
/var/log/s3-mirror/*log {
|
||||
daily
|
||||
rotate 7
|
||||
missingok
|
||||
ifempty
|
||||
compress
|
||||
compresscmd /usr/bin/bzip2
|
||||
uncompresscmd /usr/bin/bunzip2
|
||||
compressext .bz2
|
||||
dateext
|
||||
copytruncate
|
||||
}
|
||||
93
roles/s3-mirror/files/s3.sh
Normal file
93
roles/s3-mirror/files/s3.sh
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
#!/usr/bin/env bash
|
||||
# (c) 2018 Red Hat, Inc.
|
||||
# LGPL
|
||||
# Author: Rick Elrod <relrod@redhat.com>
|
||||
|
||||
CMD="aws s3 sync \
|
||||
--delete \
|
||||
--exclude */.snapshot/* \
|
||||
--exclude */source/* \
|
||||
--exclude */SRPMS/* \
|
||||
--exclude */debug/* \
|
||||
--exclude */beta/* \
|
||||
--exclude */ppc/* \
|
||||
--exclude */ppc64/* \
|
||||
--exclude */repoview/* \
|
||||
--exclude */Fedora/* \
|
||||
--exclude */Live/* \
|
||||
--exclude */isolinux/* \
|
||||
--exclude */images/* \
|
||||
--exclude */EFI/* \
|
||||
--exclude */core/* \
|
||||
--exclude */extras/* \
|
||||
--exclude */LiveOS/* \
|
||||
--exclude */development/* \
|
||||
--exclude */releases/test/* \
|
||||
--exclude */releases/8/* \
|
||||
--exclude */releases/9/* \
|
||||
--exclude */releases/10/* \
|
||||
--exclude */releases/11/* \
|
||||
--exclude */releases/12/* \
|
||||
--exclude */releases/13/* \
|
||||
--exclude */releases/14/* \
|
||||
--exclude */releases/15/* \
|
||||
--exclude */releases/16/* \
|
||||
--exclude */releases/17/* \
|
||||
--exclude */releases/18/* \
|
||||
--exclude */releases/19/* \
|
||||
--exclude */releases/20/* \
|
||||
--exclude */releases/21/* \
|
||||
--exclude */releases/22/* \
|
||||
--exclude */releases/23/* \
|
||||
--exclude */releases/24/* \
|
||||
--exclude */releases/25/* \
|
||||
--exclude */releases/26/* \
|
||||
--exclude */updates/8/* \
|
||||
--exclude */updates/9/* \
|
||||
--exclude */updates/10/* \
|
||||
--exclude */updates/11/* \
|
||||
--exclude */updates/12/* \
|
||||
--exclude */updates/13/* \
|
||||
--exclude */updates/14/* \
|
||||
--exclude */updates/15/* \
|
||||
--exclude */updates/16/* \
|
||||
--exclude */updates/17/* \
|
||||
--exclude */updates/18/* \
|
||||
--exclude */updates/19/* \
|
||||
--exclude */updates/20/* \
|
||||
--exclude */updates/21/* \
|
||||
--exclude */updates/22/* \
|
||||
--exclude */updates/23/* \
|
||||
--exclude */updates/24/* \
|
||||
--exclude */updates/25/* \
|
||||
--exclude */updates/26/* \
|
||||
--exclude */updates/testing/8/* \
|
||||
--exclude */updates/testing/9/* \
|
||||
--exclude */updates/testing/10/* \
|
||||
--exclude */updates/testing/11/* \
|
||||
--exclude */updates/testing/12/* \
|
||||
--exclude */updates/testing/13/* \
|
||||
--exclude */updates/testing/14/* \
|
||||
--exclude */updates/testing/15/* \
|
||||
--exclude */updates/testing/16/* \
|
||||
--exclude */updates/testing/17/* \
|
||||
--exclude */updates/testing/18/* \
|
||||
--exclude */updates/testing/19/* \
|
||||
--exclude */updates/testing/20/* \
|
||||
--exclude */updates/testing/21/* \
|
||||
--exclude */updates/testing/22/* \
|
||||
--exclude */updates/testing/23/* \
|
||||
--exclude */updates/testing/24/* \
|
||||
--exclude */updates/testing/25/* \
|
||||
--exclude */updates/testing/26/* \
|
||||
--no-follow-symlinks \
|
||||
"
|
||||
#--dryrun \
|
||||
|
||||
# Sync EPEL
|
||||
echo $CMD /pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
|
||||
$CMD /pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
|
||||
|
||||
# Sync Fedora
|
||||
echo $CMD /pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/
|
||||
$CMD /pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/
|
||||
|
|
@ -1,84 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
logfile="/var/log/s3-mirror/s3sync.log"
|
||||
tree="/srv/pub/"
|
||||
s3cmd=/usr/bin/s3cmd
|
||||
curdate=`date +%s`
|
||||
|
||||
if [ ! -d ${tree} ]; then
|
||||
echo "Specified root of the directory tree (${tree}) does not exist. Exiting." | tee ${logfile}
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exec 1>>${logfile} 2>&1
|
||||
|
||||
function newer()
|
||||
{
|
||||
if [ "$1" -nt "$2" ] ; then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
S3CMD_ARGS="sync \
|
||||
--verbose \
|
||||
--preserve \
|
||||
--recursive \
|
||||
--check-md5 \
|
||||
--delay-updates \
|
||||
--guess-mime-type --mime-type application/octet-stream \
|
||||
--reduced-redundancy \
|
||||
--acl-public \
|
||||
--exclude-from /usr/local/etc/s3-mirror-excludes.txt \
|
||||
"
|
||||
|
||||
content="epel"
|
||||
targets="s3-mirror-us-east-1 s3-mirror-us-west-1 s3-mirror-us-west-2 s3-mirror-eu-west-1 s3-mirror-ap-northeast-1"
|
||||
report=0
|
||||
|
||||
if [ -e /var/run/s3-mirror/lastrun ]; then
|
||||
. /var/run/s3-mirror/lastrun
|
||||
else
|
||||
# 48 hours -> 172800 seconds
|
||||
let lastrun=curdate-172800
|
||||
fi
|
||||
|
||||
let delta=curdate-lastrun
|
||||
|
||||
# hardcoded to epel only '-e'
|
||||
/usr/local/bin/last-sync -d ${delta} -e
|
||||
|
||||
if [ "$?" -ne "0" ]; then
|
||||
# no changes since the last sync
|
||||
# abort
|
||||
exit 0
|
||||
fi
|
||||
|
||||
for c in ${content}; do
|
||||
if $(newer ${tree}/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist) ; then
|
||||
echo "=============================================================="
|
||||
echo -n "Starting at "
|
||||
date
|
||||
time $s3cmd $S3CMD_ARGS \
|
||||
--cache-file /var/lib/s3-mirror/${c}.cache \
|
||||
--delete-removed \
|
||||
--delete-after \
|
||||
--add-destination s3://s3-mirror-us-west-1.fedoraproject.org/pub/${c}/ \
|
||||
--add-destination s3://s3-mirror-us-west-2.fedoraproject.org/pub/${c}/ \
|
||||
--add-destination s3://s3-mirror-eu-west-1.fedoraproject.org/pub/${c}/ \
|
||||
--add-destination s3://s3-mirror-ap-northeast-1.fedoraproject.org/pub/${c}/ \
|
||||
${tree}/${c}/ s3://s3-mirror-us-east-1.fedoraproject.org/pub/${c}/
|
||||
report=1
|
||||
cp -a ${tree}/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist
|
||||
echo "lastrun=${curdate}" > /var/run/s3-mirror/lastrun
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
if [ ${report} -ne 0 ] ; then
|
||||
for target in ${targets} ; do
|
||||
report_mirror -c /etc/mirrormanager-client/report_mirror_${target}.conf --exclude-from /usr/local/etc/s3-mirror-excludes.txt
|
||||
done
|
||||
echo -n "Ending at "
|
||||
date
|
||||
fi
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
#!/usr/bin/python
|
||||
# Copyright 2013 by Matt Domsch
|
||||
# Licensed under the MIT/X11 license
|
||||
# usage: symmetric_diff <file a> <file b>
|
||||
# given two text files a and b, returns the symmetric difference of the lines changed between them.
|
||||
# The order in which lines appear in each file is irrelevant.
|
||||
|
||||
import sys
|
||||
import optparse
|
||||
|
||||
def read_file(filename):
|
||||
f = open(filename, 'r')
|
||||
lines = f.readlines()
|
||||
f.close()
|
||||
return set(lines)
|
||||
|
||||
def diff(a, b):
|
||||
file_a = read_file(a)
|
||||
file_b = read_file(b)
|
||||
sdiff = file_a.symmetric_difference(file_b)
|
||||
sdiff = list(sdiff)
|
||||
sdiff.sort()
|
||||
return sdiff
|
||||
|
||||
def main():
|
||||
parser = optparse.OptionParser(usage=sys.argv[0] + " [options]")
|
||||
(options, args) = parser.parse_args()
|
||||
if len(args) < 2:
|
||||
parser.print_help()
|
||||
sdiff = diff(args[0], args[1])
|
||||
for l in sdiff:
|
||||
sys.stdout.write(l)
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -1,26 +1,12 @@
|
|||
- name: Create s3-mirror group
|
||||
group: gid=442 name=s3-mirror
|
||||
tags:
|
||||
- s3-mirror
|
||||
|
||||
- name: Create s3-mirror user
|
||||
user: group=s3-mirror name=s3-mirror comment="s3-mirror user" home=/home/s3-mirror shell=/bin/bash
|
||||
|
||||
- name: s3cfg file
|
||||
template: src=s3cfg dest=/home/s3-mirror/.s3cfg owner=s3-mirror group=s3-mirror mode=0600
|
||||
|
||||
- name: Install packages
|
||||
package: name={{ item }} state=present
|
||||
with_items:
|
||||
- s3cmd
|
||||
- mirrormanager2-client
|
||||
|
||||
- name: Create report_mirror config files
|
||||
template: src=report_mirror.conf dest=/etc/mirrormanager-client/report_mirror_{{ item }}.conf owner=s3-mirror group=s3-mirror mode=0400
|
||||
with_items:
|
||||
- s3-mirror-us-east-1
|
||||
- s3-mirror-us-west-1
|
||||
- s3-mirror-us-west-2
|
||||
- s3-mirror-eu-west-1
|
||||
- s3-mirror-ap-northeast-1
|
||||
tags:
|
||||
- s3-mirror
|
||||
|
||||
- name: Create needed directories
|
||||
file: path={{item}} owner=s3-mirror group=s3-mirror state=directory mode=0755
|
||||
|
|
@ -28,29 +14,44 @@
|
|||
- /var/log/s3-mirror
|
||||
- /var/lib/s3-mirror
|
||||
- /var/run/s3-mirror
|
||||
- /var/log/s3-mirror-logs
|
||||
- /var/log/s3-mirror-logs/bucket
|
||||
- /home/s3-mirror/.aws
|
||||
#- /var/log/s3-mirror-logs
|
||||
#- /var/log/s3-mirror-logs/bucket
|
||||
tags:
|
||||
- s3-mirror
|
||||
|
||||
- name: s3sync and symmetric_diff scripts
|
||||
- name: aws credentials file
|
||||
template: src=credentials dest=/home/s3-mirror/.aws/credentials owner=s3-mirror group=s3-mirror mode=0600
|
||||
tags:
|
||||
- s3-mirror
|
||||
|
||||
- name: aws config file
|
||||
copy: src=config dest=/home/s3-mirror/.aws/config owner=s3-mirror group=s3-mirror mode=0600
|
||||
tags:
|
||||
- s3-mirror
|
||||
|
||||
- name: Install packages
|
||||
package: name={{ item }} state=present
|
||||
with_items:
|
||||
- awscli
|
||||
# - mirrormanager2-client
|
||||
tags:
|
||||
- s3-mirror
|
||||
|
||||
- name: s3 script
|
||||
copy: src={{item}} dest=/usr/local/bin/{{item}} owner=s3-mirror group=s3-mirror mode=0755
|
||||
with_items:
|
||||
- s3sync
|
||||
- symmetric_diff
|
||||
|
||||
- name: s3-mirror-excludes.txt
|
||||
copy: src=s3-excludes.txt dest=/usr/local/etc/s3-mirror-excludes.txt owner=s3-mirror group=s3-mirror mode=0644
|
||||
|
||||
- name: s3-mirror logrotate
|
||||
copy: src=s3-mirror.logrotate dest=/etc/logrotate.d/s3-mirror owner=root group=root mode=0644
|
||||
|
||||
- name: s3sync-logs
|
||||
copy: src=s3sync-logs dest=/usr/local/bin/s3sync-logs owner=s3-mirror group=s3-mirror mode=0755
|
||||
- s3.sh
|
||||
tags:
|
||||
- s3-mirror
|
||||
|
||||
- name: s3sync cron
|
||||
cron: name="s3sync" minute="3,18,33,48" user="s3-mirror"
|
||||
job='/usr/local/bin/lock-wrapper s3sync /usr/local/bin/s3sync 2>&1 | /usr/local/bin/nag-once s3sync 1d 2>&1'
|
||||
cron_file=s3sync
|
||||
cron: name="s3sync" minute="0" hour="0,11" user="s3-mirror"
|
||||
job='/usr/local/bin/lock-wrapper s3sync /usr/local/bin/s3.sh 2>&1 | /usr/local/bin/nag-once s3.sh 1d 2>&1'
|
||||
cron_file=s3.sh
|
||||
when: env != 'staging'
|
||||
tags:
|
||||
- s3-mirror
|
||||
|
||||
#- name: s3sync-logs cron
|
||||
# cron: name="s3sync-logs" hour="0" user="root"
|
||||
|
|
|
|||
3
roles/s3-mirror/templates/credentials
Normal file
3
roles/s3-mirror/templates/credentials
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
[default]
|
||||
aws_access_key_id = {{ s3_mirror_aws_access_key }}
|
||||
aws_secret_access_key = {{ s3_mirror_aws_secret_key }}
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
[default]
|
||||
access_key = {{ s3_mirror_aws_access_key }}
|
||||
secret_key = {{ s3_mirror_aws_secret_key }}
|
||||
acl_public = False
|
||||
bucket_location = US
|
||||
cloudfront_host = cloudfront.amazonaws.com
|
||||
cloudfront_resource = /2008-06-30/distribution
|
||||
default_mime_type = binary/octet-stream
|
||||
delete_removed = False
|
||||
dry_run = False
|
||||
encoding = UTF-8
|
||||
encrypt = False
|
||||
force = False
|
||||
get_continue = False
|
||||
gpg_command = /usr/bin/gpg
|
||||
gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
|
||||
gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
|
||||
gpg_passphrase =
|
||||
guess_mime_type = True
|
||||
host_base = s3.amazonaws.com
|
||||
host_bucket = %(bucket)s.s3.amazonaws.com
|
||||
human_readable_sizes = False
|
||||
list_md5 = False
|
||||
preserve_attrs = True
|
||||
progress_meter = True
|
||||
proxy_host =
|
||||
proxy_port = 0
|
||||
recursive = False
|
||||
recv_chunk = 4096
|
||||
send_chunk = 4096
|
||||
simpledb_host = sdb.amazonaws.com
|
||||
skip_existing = False
|
||||
urlencoding_mode = normal
|
||||
use_https = False
|
||||
verbosity = WARNING
|
||||
Loading…
Add table
Add a link
Reference in a new issue