From 8c4931e0f99d747731db2bfb00a9635bd1208f31 Mon Sep 17 00:00:00 2001 From: Ricky Elrod Date: Mon, 27 Jul 2015 19:14:17 +0000 Subject: [PATCH] most of s3-mirror role Signed-off-by: Ricky Elrod --- roles/s3-mirror/files/s3-excludes.txt | 46 +++++++++++++ roles/s3-mirror/files/s3-mirror.init | 77 +++++++++++++++++++++ roles/s3-mirror/files/s3-mirror.logrotate | 15 ++++ roles/s3-mirror/files/s3sync | 84 +++++++++++++++++++++++ roles/s3-mirror/files/s3sync-logs | 33 +++++++++ roles/s3-mirror/files/symmetric_diff | 35 ++++++++++ roles/s3-mirror/tasks/main.yml | 41 +++++++++++ roles/s3-mirror/templates/s3cfg | 35 ++++++++++ 8 files changed, 366 insertions(+) create mode 100644 roles/s3-mirror/files/s3-excludes.txt create mode 100644 roles/s3-mirror/files/s3-mirror.init create mode 100644 roles/s3-mirror/files/s3-mirror.logrotate create mode 100755 roles/s3-mirror/files/s3sync create mode 100755 roles/s3-mirror/files/s3sync-logs create mode 100755 roles/s3-mirror/files/symmetric_diff create mode 100644 roles/s3-mirror/tasks/main.yml create mode 100644 roles/s3-mirror/templates/s3cfg diff --git a/roles/s3-mirror/files/s3-excludes.txt b/roles/s3-mirror/files/s3-excludes.txt new file mode 100644 index 0000000000..ab82e6f188 --- /dev/null +++ b/roles/s3-mirror/files/s3-excludes.txt @@ -0,0 +1,46 @@ +.snapshot/ +source/ +SRPMS/ +debug/ +beta/ +ppc/ +ppc64/ +repoview/ +Fedora/ +Live/ +isolinux/ +images/ +EFI/ +drpms/ +core/ +extras/ +LiveOS/ +development/ +releases/test/ +releases/8/ +releases/9/ +releases/10/ +releases/11/ +releases/12/ +releases/13/ +releases/14/ +releases/15/ +releases/16/ +updates/8/ +updates/9/ +updates/10/ +updates/11/ +updates/12/ +updates/13/ +updates/14/ +updates/15/ +updates/16/ +updates/testing/8/ +updates/testing/9/ +updates/testing/10/ +updates/testing/11/ +updates/testing/12/ +updates/testing/13/ +updates/testing/14/ +updates/testing/15/ +updates/testing/16/ diff --git a/roles/s3-mirror/files/s3-mirror.init b/roles/s3-mirror/files/s3-mirror.init new file mode 100644 index 0000000000..59dbb39c12 --- /dev/null +++ b/roles/s3-mirror/files/s3-mirror.init @@ -0,0 +1,77 @@ +#!/bin/sh +# +# s3-mirror - sync content to S3 +# +# chkconfig: - 99 99 +# description: sync content to S3 + +# http://fedoraproject.org/wiki/FCNewInit/Initscripts +### BEGIN INIT INFO +# Provides: +# Required-Start: $network $named $remote_fs +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: +# Default-Stop: +# Short-Description: +# Description: +### END INIT INFO + +# Source function library. +. /etc/rc.d/init.d/functions + +exec="/usr/local/bin/s3sync" +prog=${exec##*/} + +lockfile=/var/lock/subsys/$prog +pidfile=/var/run/s3-mirror/pid + +start() { + echo -n $"Starting $prog: " + mkdir -p /var/run/s3-mirror + chown -R s3-mirror:s3-mirror /var/run/s3-mirror + daemon --user s3-mirror --pidfile $pidfile $exec & + retval=$? + echo + [ $retval -eq 0 ] && touch $lockfile + return $retval +} + +stop() { + echo -n $"Stopping $prog: " + killproc -p $pidfile + retval=$? + echo + [ $retval -eq 0 ] && rm -f $lockfile + return $retval +} + +restart() { + stop + start +} + +case "$1" in + start|stop|restart) + $1 + ;; + force-reload) + restart + ;; + status) + status $prog + ;; + try-restart|condrestart) + if status $prog >/dev/null ; then + restart + fi + ;; + reload) + action $"Service ${0##*/} does not support the reload action: " /bin/false + exit 3 + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|try-restart|force-reload}" + exit 2 +esac diff --git a/roles/s3-mirror/files/s3-mirror.logrotate b/roles/s3-mirror/files/s3-mirror.logrotate new file mode 100644 index 0000000000..12e66aa61e --- /dev/null +++ b/roles/s3-mirror/files/s3-mirror.logrotate @@ -0,0 +1,15 @@ +/var/log/s3-mirror/*log { + daily + rotate 7 + missingok + ifempty + compress + compresscmd /usr/bin/bzip2 + uncompresscmd /usr/bin/bunzip2 + compressext .bz2 + dateext + copytruncate + postrotate + /bin/kill -HUP `cat /var/run/s3-mirror/pid 2>/dev/null` 2>/dev/null || true + endscript +} diff --git a/roles/s3-mirror/files/s3sync b/roles/s3-mirror/files/s3sync new file mode 100755 index 0000000000..6d0c552195 --- /dev/null +++ b/roles/s3-mirror/files/s3sync @@ -0,0 +1,84 @@ +#!/bin/bash + +pidfile=/var/run/s3-mirror/pid +logfile="/var/log/s3-mirror/s3sync.log" + +trap sighup_handler HUP +trap "rm -f ${pidfile}" QUIT EXIT INT TERM + +function sighup_handler() +{ + exec 1>>${logfile} 2>&1 +} + + +function newer() +{ + if [ "$1" -nt "$2" ] ; then + return 0 + fi + return 1 +} + +function repeat() +{ + while :; do + $1 + /bin/sleep $((${RANDOM} % 300)) + done +} + +s3cmd=/usr/bin/s3cmd + +S3CMD_ARGS="sync \ + --verbose \ + --preserve \ + --recursive \ + --check-md5 \ + --delay-updates \ + --guess-mime-type --mime-type application/octet-stream \ + --reduced-redundancy \ + --acl-public \ + --exclude-from /usr/local/etc/s3-mirror-excludes.txt \ + " + +content="epel" +targets="s3-mirror-us-east-1 s3-mirror-us-west-1 s3-mirror-us-west-2 s3-mirror-eu-west-1 s3-mirror-ap-northeast-1" + +function parallel_sync_full() +{ + report=0 + for c in ${content}; do + if $(newer /pub/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist) ; then + echo "==============================================================" + echo -n "Starting at " + date + time $s3cmd $S3CMD_ARGS \ + --cache-file /var/lib/s3-mirror/${c}.cache \ + --delete-removed \ + --delete-after \ + --add-destination s3://s3-mirror-us-west-1.fedoraproject.org/pub/${c}/ \ + --add-destination s3://s3-mirror-us-west-2.fedoraproject.org/pub/${c}/ \ + --add-destination s3://s3-mirror-eu-west-1.fedoraproject.org/pub/${c}/ \ + --add-destination s3://s3-mirror-ap-northeast-1.fedoraproject.org/pub/${c}/ \ + /pub/${c}/ s3://s3-mirror-us-east-1.fedoraproject.org/pub/${c}/ + report=1 + cp -a /pub/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist + fi + done + + if [ ${report} -ne 0 ] ; then + for target in ${targets} ; do + report_mirror -c /etc/mirrormanager-client/report_mirror_${target}.conf --exclude-from /usr/local/etc/s3-mirror-excludes.txt + done + echo -n "Ending at " + date + fi +} + + +# send stdout/stderr to logfile +sighup_handler +mkdir -p $(dirname ${pidfile}) +echo "$$" > ${pidfile} +repeat parallel_sync_full diff --git a/roles/s3-mirror/files/s3sync-logs b/roles/s3-mirror/files/s3sync-logs new file mode 100755 index 0000000000..03ccf19c9f --- /dev/null +++ b/roles/s3-mirror/files/s3sync-logs @@ -0,0 +1,33 @@ +#!/bin/sh + +s3cmd=/usr/bin/s3cmd + +S3CMD_ARGS="sync \ + --recursive \ + --delete-removed \ + --verbose \ + " +targets="s3-mirror-us-east-1 s3-mirror-us-west-1 s3-mirror-us-west-2 s3-mirror-eu-west-1 s3-mirror-ap-northeast-1" +for t in ${targets}; do + mkdir -p /var/log/s3-mirror-logs/bucket/${t} +done + +echo "==============================================================" +echo -n "Starting at " +date + +time $s3cmd $S3CMD_ARGS s3://s3-mirror-logs.fedoraproject.org/s3-mirror-us-east-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-us-east-1/ & +time $s3cmd $S3CMD_ARGS s3://aws-logs-us-west-1.fedoraproject.org/s3-mirror-us-west-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-us-west-1/ & +time $s3cmd $S3CMD_ARGS s3://aws-logs-us-west-2.fedoraproject.org/s3-mirror-us-west-2/ /var/log/s3-mirror-logs/bucket/s3-mirror-us-west-2/ & +time $s3cmd $S3CMD_ARGS s3://aws-logs-eu-west-1.fedoraproject.org/s3-mirror-eu-west-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-eu-west-1/ & +time $s3cmd $S3CMD_ARGS s3://aws-logs-ap-northeast-1.fedoraproject.org/s3-mirror-s3-northeast-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-ap-northeast-1/ & +wait + +# concatenate them for processing +rm /var/log/s3-mirror-logs/combined.log +find /var/log/s3-mirror-logs/bucket/ -type f -mtime -2 | sort -n | while read f; do + cat $f >> /var/log/s3-mirror-logs/combined.log +done + +echo -n "Ending at " +date diff --git a/roles/s3-mirror/files/symmetric_diff b/roles/s3-mirror/files/symmetric_diff new file mode 100755 index 0000000000..e7fb62c390 --- /dev/null +++ b/roles/s3-mirror/files/symmetric_diff @@ -0,0 +1,35 @@ +#!/usr/bin/python +# Copyright 2013 by Matt Domsch +# Licensed under the MIT/X11 license +# usage: symmetric_diff +# given two text files a and b, returns the symmetric difference of the lines changed between them. +# The order in which lines appear in each file is irrelevant. + +import sys +import optparse + +def read_file(filename): + f = open(filename, 'r') + lines = f.readlines() + f.close() + return set(lines) + +def diff(a, b): + file_a = read_file(a) + file_b = read_file(b) + sdiff = file_a.symmetric_difference(file_b) + sdiff = list(sdiff) + sdiff.sort() + return sdiff + +def main(): + parser = optparse.OptionParser(usage=sys.argv[0] + " [options]") + (options, args) = parser.parse_args() + if len(args) < 2: + parser.print_help() + sdiff = diff(args[0], args[1]) + for l in sdiff: + sys.stdout.write(l) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/roles/s3-mirror/tasks/main.yml b/roles/s3-mirror/tasks/main.yml new file mode 100644 index 0000000000..5c32f4a120 --- /dev/null +++ b/roles/s3-mirror/tasks/main.yml @@ -0,0 +1,41 @@ +- name: Create s3-mirror group + group: gid=442 name=s3-mirror + +- name: Create s3-mirror user + user: group=s3-mirror groups=mirrors,mirrors2,apache name=s3-mirror comment="s3-mirror user" home=/home/s3-mirror shell=/bin/bash + +- name: s3cfg file + template: src=s3cfg dest=/home/s3-mirror/.s3cfg owner=s3-mirror group=s3-mirror mode=0600 + +- name: Install packages + yum: pkg={{ item }} state=installed + with_items: + - mirrormanager-client + - s3cmd + +- name: Create needed directories + file: path={{item}} owner=s3-mirror group=s3-mirror state=directory mode=0755 + with_items: + - /var/log/s3-mirror + - /var/lib/s3-mirror + - /var/run/s3-mirror + - /var/log/s3-mirror-logs + - /var/log/s3-mirror-logs/bucket + +- name: s3sync and symmetric_diff scripts + copy: src={{item}} dest=/usr/local/bin/{{item}} owner=s3-mirror group=s3-mirror mode=0755 + with_items: + - s3sync + - symmetric_diff + +- name: s3-mirror-excludes.txt + copy: src=s3-excludes.txt dest=/usr/local/etc/s3-mirror-excludes.txt owner=s3-mirror group=s3-mirror mode=0644 + +- name: s3-mirror logrotate + copy: src=s3-mirror.logrotate dest=/etc/logrotate.d/s3-mirror owner=s3-mirror group=s3-mirror mode=0644 + +- name: s3-mirror init.d + copy: src=s3-mirror.init dest=/etc/init.d/s3-mirror owner=s3-mirror group=s3-mirror mode=0755 + +- name: s3sync-logs + copy: src=s3sync-logs dest=/usr/local/bin/s3sync-logs owner=s3-mirror group=s3-mirror mode=0755 diff --git a/roles/s3-mirror/templates/s3cfg b/roles/s3-mirror/templates/s3cfg new file mode 100644 index 0000000000..0f16bec68d --- /dev/null +++ b/roles/s3-mirror/templates/s3cfg @@ -0,0 +1,35 @@ +[default] +access_key = {{ s3_mirror_aws_access_key }} +secret_key = {{ s3_mirror_aws_secret_key }} +acl_public = False +bucket_location = US +cloudfront_host = cloudfront.amazonaws.com +cloudfront_resource = /2008-06-30/distribution +default_mime_type = binary/octet-stream +delete_removed = False +dry_run = False +encoding = UTF-8 +encrypt = False +force = False +get_continue = False +gpg_command = /usr/bin/gpg +gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s +gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s +gpg_passphrase = +guess_mime_type = True +host_base = s3.amazonaws.com +host_bucket = %(bucket)s.s3.amazonaws.com +human_readable_sizes = False +list_md5 = False +preserve_attrs = True +progress_meter = True +proxy_host = +proxy_port = 0 +recursive = False +recv_chunk = 4096 +send_chunk = 4096 +simpledb_host = sdb.amazonaws.com +skip_existing = False +urlencoding_mode = normal +use_https = False +verbosity = WARNING