Infrastructure/ansible
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

attempt at cron-ify-ing new s3 sync

Browse source 
Signed-off-by: Rick Elrod <relrod@redhat.com>
This commit is contained in:
Rick Elrod 2018-10-09 17:17:47 +00:00
parent ca306d59d9
commit 7984e52034
9 changed files with 135 additions and 247 deletions
Download patch file Download diff file Expand all files Collapse all files

3
roles/s3-mirror/files/config Normal file
View file

@ -0,0 +1,3 @@
[default]
region = us-west-2
output = text

46
roles/s3-mirror/files/s3-excludes.txt
View file

@ -1,46 +0,0 @@
.snapshot/
source/
SRPMS/
debug/
beta/
ppc/
ppc64/
repoview/
Fedora/
Live/
isolinux/
images/
EFI/
drpms/
core/
extras/
LiveOS/
development/
releases/test/
releases/8/
releases/9/
releases/10/
releases/11/
releases/12/
releases/13/
releases/14/
releases/15/
releases/16/
updates/8/
updates/9/
updates/10/
updates/11/
updates/12/
updates/13/
updates/14/
updates/15/
updates/16/
updates/testing/8/
updates/testing/9/
updates/testing/10/
updates/testing/11/
updates/testing/12/
updates/testing/13/
updates/testing/14/
updates/testing/15/
updates/testing/16/

12
roles/s3-mirror/files/s3-mirror.logrotate
View file

@ -1,12 +0,0 @@
/var/log/s3-mirror/*log {
daily
rotate 7
missingok
ifempty
compress
compresscmd /usr/bin/bzip2
uncompresscmd /usr/bin/bunzip2
compressext .bz2
dateext
copytruncate
}

93
roles/s3-mirror/files/s3.sh Normal file
View file

@ -0,0 +1,93 @@
#!/usr/bin/env bash
# (c) 2018 Red Hat, Inc.
# LGPL
# Author: Rick Elrod <relrod@redhat.com>
CMD="aws s3 sync \
--delete \
--exclude */.snapshot/* \
--exclude */source/* \
--exclude */SRPMS/* \
--exclude */debug/* \
--exclude */beta/* \
--exclude */ppc/* \
--exclude */ppc64/* \
--exclude */repoview/* \
--exclude */Fedora/* \
--exclude */Live/* \
--exclude */isolinux/* \
--exclude */images/* \
--exclude */EFI/* \
--exclude */core/* \
--exclude */extras/* \
--exclude */LiveOS/* \
--exclude */development/* \
--exclude */releases/test/* \
--exclude */releases/8/* \
--exclude */releases/9/* \
--exclude */releases/10/* \
--exclude */releases/11/* \
--exclude */releases/12/* \
--exclude */releases/13/* \
--exclude */releases/14/* \
--exclude */releases/15/* \
--exclude */releases/16/* \
--exclude */releases/17/* \
--exclude */releases/18/* \
--exclude */releases/19/* \
--exclude */releases/20/* \
--exclude */releases/21/* \
--exclude */releases/22/* \
--exclude */releases/23/* \
--exclude */releases/24/* \
--exclude */releases/25/* \
--exclude */releases/26/* \
--exclude */updates/8/* \
--exclude */updates/9/* \
--exclude */updates/10/* \
--exclude */updates/11/* \
--exclude */updates/12/* \
--exclude */updates/13/* \
--exclude */updates/14/* \
--exclude */updates/15/* \
--exclude */updates/16/* \
--exclude */updates/17/* \
--exclude */updates/18/* \
--exclude */updates/19/* \
--exclude */updates/20/* \
--exclude */updates/21/* \
--exclude */updates/22/* \
--exclude */updates/23/* \
--exclude */updates/24/* \
--exclude */updates/25/* \
--exclude */updates/26/* \
--exclude */updates/testing/8/* \
--exclude */updates/testing/9/* \
--exclude */updates/testing/10/* \
--exclude */updates/testing/11/* \
--exclude */updates/testing/12/* \
--exclude */updates/testing/13/* \
--exclude */updates/testing/14/* \
--exclude */updates/testing/15/* \
--exclude */updates/testing/16/* \
--exclude */updates/testing/17/* \
--exclude */updates/testing/18/* \
--exclude */updates/testing/19/* \
--exclude */updates/testing/20/* \
--exclude */updates/testing/21/* \
--exclude */updates/testing/22/* \
--exclude */updates/testing/23/* \
--exclude */updates/testing/24/* \
--exclude */updates/testing/25/* \
--exclude */updates/testing/26/* \
--no-follow-symlinks \
"
#--dryrun \
# Sync EPEL
echo $CMD /pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
$CMD /pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/
# Sync Fedora
echo $CMD /pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/
$CMD /pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/

84
roles/s3-mirror/files/s3sync
View file

@ -1,84 +0,0 @@
#!/bin/bash
logfile="/var/log/s3-mirror/s3sync.log"
tree="/srv/pub/"
s3cmd=/usr/bin/s3cmd
curdate=`date +%s`
if [ ! -d ${tree} ]; then
echo "Specified root of the directory tree (${tree}) does not exist. Exiting." | tee ${logfile}
exit 1
fi
exec 1>>${logfile} 2>&1
function newer()
{
if [ "$1" -nt "$2" ] ; then
return 0
fi
return 1
}
S3CMD_ARGS="sync \
--verbose \
--preserve \
--recursive \
--check-md5 \
--delay-updates \
--guess-mime-type --mime-type application/octet-stream \
--reduced-redundancy \
--acl-public \
--exclude-from /usr/local/etc/s3-mirror-excludes.txt \
"
content="epel"
targets="s3-mirror-us-east-1 s3-mirror-us-west-1 s3-mirror-us-west-2 s3-mirror-eu-west-1 s3-mirror-ap-northeast-1"
report=0
if [ -e /var/run/s3-mirror/lastrun ]; then
. /var/run/s3-mirror/lastrun
else
# 48 hours -> 172800 seconds
let lastrun=curdate-172800
fi
let delta=curdate-lastrun
# hardcoded to epel only '-e'
/usr/local/bin/last-sync -d ${delta} -e
if [ "$?" -ne "0" ]; then
# no changes since the last sync
# abort
exit 0
fi
for c in ${content}; do
if $(newer ${tree}/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist) ; then
echo "=============================================================="
echo -n "Starting at "
date
time $s3cmd $S3CMD_ARGS \
--cache-file /var/lib/s3-mirror/${c}.cache \
--delete-removed \
--delete-after \
--add-destination s3://s3-mirror-us-west-1.fedoraproject.org/pub/${c}/ \
--add-destination s3://s3-mirror-us-west-2.fedoraproject.org/pub/${c}/ \
--add-destination s3://s3-mirror-eu-west-1.fedoraproject.org/pub/${c}/ \
--add-destination s3://s3-mirror-ap-northeast-1.fedoraproject.org/pub/${c}/ \
${tree}/${c}/ s3://s3-mirror-us-east-1.fedoraproject.org/pub/${c}/
report=1
cp -a ${tree}/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist
echo "lastrun=${curdate}" > /var/run/s3-mirror/lastrun
fi
done
if [ ${report} -ne 0 ] ; then
for target in ${targets} ; do
report_mirror -c /etc/mirrormanager-client/report_mirror_${target}.conf --exclude-from /usr/local/etc/s3-mirror-excludes.txt
done
echo -n "Ending at "
date
fi

35
roles/s3-mirror/files/symmetric_diff
View file

@ -1,35 +0,0 @@
#!/usr/bin/python
# Copyright 2013 by Matt Domsch
# Licensed under the MIT/X11 license
# usage: symmetric_diff <file a> <file b>
# given two text files a and b, returns the symmetric difference of the lines changed between them.
# The order in which lines appear in each file is irrelevant.
import sys
import optparse
def read_file(filename):
f = open(filename, 'r')
lines = f.readlines()
f.close()
return set(lines)
def diff(a, b):
file_a = read_file(a)
file_b = read_file(b)
sdiff = file_a.symmetric_difference(file_b)
sdiff = list(sdiff)
sdiff.sort()
return sdiff
def main():
parser = optparse.OptionParser(usage=sys.argv[0] + " [options]")
(options, args) = parser.parse_args()
if len(args) < 2:
parser.print_help()
sdiff = diff(args[0], args[1])
for l in sdiff:
sys.stdout.write(l)
if __name__ == "__main__":
sys.exit(main())

71
roles/s3-mirror/tasks/main.yml
View file

@ -1,26 +1,12 @@
- name: Create s3-mirror group - name: Create s3-mirror group
group: gid=442 name=s3-mirror group: gid=442 name=s3-mirror
tags:
- s3-mirror
- name: Create s3-mirror user - name: Create s3-mirror user
user: group=s3-mirror name=s3-mirror comment="s3-mirror user" home=/home/s3-mirror shell=/bin/bash user: group=s3-mirror name=s3-mirror comment="s3-mirror user" home=/home/s3-mirror shell=/bin/bash
tags:
- name: s3cfg file - s3-mirror
template: src=s3cfg dest=/home/s3-mirror/.s3cfg owner=s3-mirror group=s3-mirror mode=0600
- name: Install packages
package: name={{ item }} state=present
with_items:
- s3cmd
- mirrormanager2-client
- name: Create report_mirror config files
template: src=report_mirror.conf dest=/etc/mirrormanager-client/report_mirror_{{ item }}.conf owner=s3-mirror group=s3-mirror mode=0400
with_items:
- s3-mirror-us-east-1
- s3-mirror-us-west-1
- s3-mirror-us-west-2
- s3-mirror-eu-west-1
- s3-mirror-ap-northeast-1
- name: Create needed directories - name: Create needed directories
file: path={{item}} owner=s3-mirror group=s3-mirror state=directory mode=0755 file: path={{item}} owner=s3-mirror group=s3-mirror state=directory mode=0755
@ -28,29 +14,44 @@
- /var/log/s3-mirror - /var/log/s3-mirror
- /var/lib/s3-mirror - /var/lib/s3-mirror
- /var/run/s3-mirror - /var/run/s3-mirror
- /var/log/s3-mirror-logs - /home/s3-mirror/.aws
- /var/log/s3-mirror-logs/bucket #- /var/log/s3-mirror-logs
#- /var/log/s3-mirror-logs/bucket
tags:
- s3-mirror
- name: s3sync and symmetric_diff scripts - name: aws credentials file
template: src=credentials dest=/home/s3-mirror/.aws/credentials owner=s3-mirror group=s3-mirror mode=0600
tags:
- s3-mirror
- name: aws config file
copy: src=config dest=/home/s3-mirror/.aws/config owner=s3-mirror group=s3-mirror mode=0600
tags:
- s3-mirror
- name: Install packages
package: name={{ item }} state=present
with_items:
- awscli
# - mirrormanager2-client
tags:
- s3-mirror
- name: s3 script
copy: src={{item}} dest=/usr/local/bin/{{item}} owner=s3-mirror group=s3-mirror mode=0755 copy: src={{item}} dest=/usr/local/bin/{{item}} owner=s3-mirror group=s3-mirror mode=0755
with_items: with_items:
- s3sync - s3.sh
- symmetric_diff tags:
- s3-mirror
- name: s3-mirror-excludes.txt
copy: src=s3-excludes.txt dest=/usr/local/etc/s3-mirror-excludes.txt owner=s3-mirror group=s3-mirror mode=0644
- name: s3-mirror logrotate
copy: src=s3-mirror.logrotate dest=/etc/logrotate.d/s3-mirror owner=root group=root mode=0644
- name: s3sync-logs
copy: src=s3sync-logs dest=/usr/local/bin/s3sync-logs owner=s3-mirror group=s3-mirror mode=0755
- name: s3sync cron - name: s3sync cron
cron: name="s3sync" minute="3,18,33,48" user="s3-mirror" cron: name="s3sync" minute="0" hour="0,11" user="s3-mirror"
job='/usr/local/bin/lock-wrapper s3sync /usr/local/bin/s3sync 2>&1 | /usr/local/bin/nag-once s3sync 1d 2>&1' job='/usr/local/bin/lock-wrapper s3sync /usr/local/bin/s3.sh 2>&1 | /usr/local/bin/nag-once s3.sh 1d 2>&1'
cron_file=s3sync cron_file=s3.sh
when: env != 'staging' when: env != 'staging'
tags:
- s3-mirror
#- name: s3sync-logs cron #- name: s3sync-logs cron
# cron: name="s3sync-logs" hour="0" user="root" # cron: name="s3sync-logs" hour="0" user="root"

3
roles/s3-mirror/templates/credentials Normal file
View file

@ -0,0 +1,3 @@
[default]
aws_access_key_id = {{ s3_mirror_aws_access_key }}
aws_secret_access_key = {{ s3_mirror_aws_secret_key }}

35
roles/s3-mirror/templates/s3cfg
View file

@ -1,35 +0,0 @@
[default]
access_key = {{ s3_mirror_aws_access_key }}
secret_key = {{ s3_mirror_aws_secret_key }}
acl_public = False
bucket_location = US
cloudfront_host = cloudfront.amazonaws.com
cloudfront_resource = /2008-06-30/distribution
default_mime_type = binary/octet-stream
delete_removed = False
dry_run = False
encoding = UTF-8
encrypt = False
force = False
get_continue = False
gpg_command = /usr/bin/gpg
gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
gpg_passphrase =
guess_mime_type = True
host_base = s3.amazonaws.com
host_bucket = %(bucket)s.s3.amazonaws.com
human_readable_sizes = False
list_md5 = False
preserve_attrs = True
progress_meter = True
proxy_host =
proxy_port = 0
recursive = False
recv_chunk = 4096
send_chunk = 4096
simpledb_host = sdb.amazonaws.com
skip_existing = False
urlencoding_mode = normal
use_https = False
verbosity = WARNING