most of s3-mirror role

Signed-off-by: Ricky Elrod <codeblock@fedoraproject.org>

roles/s3-mirror/files/s3-excludes.txt

@@ -0,0 +1,46 @@
+.snapshot/
+source/
+SRPMS/
+debug/
+beta/
+ppc/
+ppc64/
+repoview/
+Fedora/
+Live/
+isolinux/
+images/
+EFI/
+drpms/
+core/
+extras/
+LiveOS/
+development/
+releases/test/
+releases/8/
+releases/9/
+releases/10/
+releases/11/
+releases/12/
+releases/13/
+releases/14/
+releases/15/
+releases/16/
+updates/8/
+updates/9/
+updates/10/
+updates/11/
+updates/12/
+updates/13/
+updates/14/
+updates/15/
+updates/16/
+updates/testing/8/
+updates/testing/9/
+updates/testing/10/
+updates/testing/11/
+updates/testing/12/
+updates/testing/13/
+updates/testing/14/
+updates/testing/15/
+updates/testing/16/

roles/s3-mirror/files/s3-mirror.init

@@ -0,0 +1,77 @@
+#!/bin/sh
+#
+# s3-mirror - sync content to S3
+#
+# chkconfig:   - 99 99
+# description: sync content to S3
+
+# http://fedoraproject.org/wiki/FCNewInit/Initscripts
+### BEGIN INIT INFO
+# Provides: 
+# Required-Start: $network $named $remote_fs
+# Required-Stop: 
+# Should-Start: 
+# Should-Stop: 
+# Default-Start: 
+# Default-Stop: 
+# Short-Description: 
+# Description: 
+### END INIT INFO
+
+# Source function library.
+. /etc/rc.d/init.d/functions
+
+exec="/usr/local/bin/s3sync"
+prog=${exec##*/}
+
+lockfile=/var/lock/subsys/$prog
+pidfile=/var/run/s3-mirror/pid
+
+start() {
+    echo -n $"Starting $prog: "
+    mkdir -p /var/run/s3-mirror
+    chown -R s3-mirror:s3-mirror /var/run/s3-mirror
+    daemon --user s3-mirror --pidfile $pidfile $exec &
+    retval=$?
+    echo
+    [ $retval -eq 0 ] && touch $lockfile
+    return $retval
+}
+
+stop() {
+    echo -n $"Stopping $prog: "
+    killproc -p $pidfile
+    retval=$?
+    echo
+    [ $retval -eq 0 ] && rm -f $lockfile
+    return $retval
+}
+
+restart() {
+    stop
+    start
+}
+
+case "$1" in
+    start|stop|restart)
+        $1
+        ;;
+    force-reload)
+        restart
+        ;;
+    status)
+        status $prog
+        ;;
+    try-restart|condrestart)
+        if status $prog >/dev/null ; then
+            restart
+        fi
+	;;
+    reload)
+        action $"Service ${0##*/} does not support the reload action: " /bin/false
+        exit 3
+        ;;
+    *)
+        echo $"Usage: $0 {start|stop|status|restart|try-restart|force-reload}"
+        exit 2
+esac

roles/s3-mirror/files/s3-mirror.logrotate

@@ -0,0 +1,15 @@
+/var/log/s3-mirror/*log {
+    daily
+    rotate 7
+    missingok
+    ifempty
+    compress
+    compresscmd /usr/bin/bzip2
+    uncompresscmd /usr/bin/bunzip2
+    compressext .bz2
+    dateext
+    copytruncate
+    postrotate
+        /bin/kill -HUP `cat /var/run/s3-mirror/pid 2>/dev/null` 2>/dev/null || true
+    endscript
+}

roles/s3-mirror/files/s3sync

@@ -0,0 +1,84 @@
+#!/bin/bash
+
+pidfile=/var/run/s3-mirror/pid
+logfile="/var/log/s3-mirror/s3sync.log"
+
+trap sighup_handler HUP
+trap "rm -f ${pidfile}" QUIT EXIT INT TERM
+
+function sighup_handler()
+{
+    exec 1>>${logfile} 2>&1
+}
+
+
+function newer()
+{
+    if [ "$1" -nt "$2" ] ; then
+	return 0
+    fi
+    return 1
+}
+
+function repeat()
+{
+    while :; do
+	$1
+        /bin/sleep $((${RANDOM} % 300))
+    done
+}
+
+s3cmd=/usr/bin/s3cmd
+
+S3CMD_ARGS="sync \
+  --verbose \
+  --preserve \
+  --recursive \
+  --check-md5 \
+  --delay-updates \
+  --guess-mime-type --mime-type application/octet-stream \
+  --reduced-redundancy \
+  --acl-public \
+  --exclude-from /usr/local/etc/s3-mirror-excludes.txt \
+  "
+
+content="epel"
+targets="s3-mirror-us-east-1 s3-mirror-us-west-1 s3-mirror-us-west-2 s3-mirror-eu-west-1 s3-mirror-ap-northeast-1"
+
+function parallel_sync_full()
+{
+    report=0
+    for c in ${content}; do
+	if $(newer /pub/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist) ; then
+	    echo "=============================================================="
+	    echo -n "Starting at "
+	    date
+            time $s3cmd $S3CMD_ARGS  \
+		       --cache-file /var/lib/s3-mirror/${c}.cache \
+                       --delete-removed \
+                       --delete-after \
+                       --add-destination s3://s3-mirror-us-west-1.fedoraproject.org/pub/${c}/ \
+                       --add-destination s3://s3-mirror-us-west-2.fedoraproject.org/pub/${c}/ \
+                       --add-destination s3://s3-mirror-eu-west-1.fedoraproject.org/pub/${c}/ \
+                       --add-destination s3://s3-mirror-ap-northeast-1.fedoraproject.org/pub/${c}/ \
+                              /pub/${c}/ s3://s3-mirror-us-east-1.fedoraproject.org/pub/${c}/
+	    report=1
+	    cp -a /pub/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist
+	fi
+    done
+
+    if [ ${report} -ne 0 ] ; then
+	for target in ${targets} ; do
+           report_mirror -c /etc/mirrormanager-client/report_mirror_${target}.conf --exclude-from /usr/local/etc/s3-mirror-excludes.txt
+	done
+        echo -n "Ending at "
+        date
+    fi
+}
+
+
+# send stdout/stderr to logfile
+sighup_handler
+mkdir -p $(dirname ${pidfile})
+echo "$$" > ${pidfile}
+repeat parallel_sync_full

roles/s3-mirror/files/s3sync-logs

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+s3cmd=/usr/bin/s3cmd
+
+S3CMD_ARGS="sync \
+  --recursive \
+  --delete-removed \
+  --verbose \
+  "
+targets="s3-mirror-us-east-1 s3-mirror-us-west-1 s3-mirror-us-west-2 s3-mirror-eu-west-1 s3-mirror-ap-northeast-1"
+for t in ${targets}; do
+    mkdir -p /var/log/s3-mirror-logs/bucket/${t}
+done
+
+echo "=============================================================="
+echo -n "Starting at "
+date
+
+time $s3cmd $S3CMD_ARGS s3://s3-mirror-logs.fedoraproject.org/s3-mirror-us-east-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-us-east-1/ &
+time $s3cmd $S3CMD_ARGS s3://aws-logs-us-west-1.fedoraproject.org/s3-mirror-us-west-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-us-west-1/ &
+time $s3cmd $S3CMD_ARGS s3://aws-logs-us-west-2.fedoraproject.org/s3-mirror-us-west-2/ /var/log/s3-mirror-logs/bucket/s3-mirror-us-west-2/ &
+time $s3cmd $S3CMD_ARGS s3://aws-logs-eu-west-1.fedoraproject.org/s3-mirror-eu-west-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-eu-west-1/ &
+time $s3cmd $S3CMD_ARGS s3://aws-logs-ap-northeast-1.fedoraproject.org/s3-mirror-s3-northeast-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-ap-northeast-1/ &
+wait
+
+# concatenate them for processing
+rm /var/log/s3-mirror-logs/combined.log
+find /var/log/s3-mirror-logs/bucket/ -type f -mtime -2 | sort -n | while read f; do
+    cat $f >> /var/log/s3-mirror-logs/combined.log
+done
+
+echo -n "Ending at "
+date

roles/s3-mirror/files/symmetric_diff

@@ -0,0 +1,35 @@
+#!/usr/bin/python
+# Copyright 2013 by Matt Domsch
+# Licensed under the MIT/X11 license
+# usage: symmetric_diff <file a> <file b>
+#   given two text files a and b, returns the symmetric difference of the lines changed between them.
+#   The order in which lines appear in each file is irrelevant.
+
+import sys
+import optparse
+
+def read_file(filename):
+    f = open(filename, 'r')
+    lines = f.readlines()
+    f.close()
+    return set(lines)
+
+def diff(a, b):
+    file_a = read_file(a)
+    file_b = read_file(b)
+    sdiff = file_a.symmetric_difference(file_b)
+    sdiff = list(sdiff)
+    sdiff.sort()
+    return sdiff
+
+def main():
+    parser = optparse.OptionParser(usage=sys.argv[0] + " [options]")
+    (options, args) = parser.parse_args()
+    if len(args) < 2:
+        parser.print_help()
+    sdiff = diff(args[0], args[1])
+    for l in sdiff:
+        sys.stdout.write(l)
+
+if __name__ == "__main__":
+    sys.exit(main())

roles/s3-mirror/tasks/main.yml

@@ -0,0 +1,41 @@
+- name: Create s3-mirror group
+  group: gid=442 name=s3-mirror
+
+- name: Create s3-mirror user
+  user: group=s3-mirror groups=mirrors,mirrors2,apache name=s3-mirror comment="s3-mirror user" home=/home/s3-mirror shell=/bin/bash
+
+- name: s3cfg file
+  template: src=s3cfg dest=/home/s3-mirror/.s3cfg owner=s3-mirror group=s3-mirror mode=0600
+
+- name: Install packages
+  yum: pkg={{ item }} state=installed
+  with_items:
+  - mirrormanager-client
+  - s3cmd
+
+- name: Create needed directories
+  file: path={{item}} owner=s3-mirror group=s3-mirror state=directory mode=0755
+  with_items:
+  - /var/log/s3-mirror
+  - /var/lib/s3-mirror
+  - /var/run/s3-mirror
+  - /var/log/s3-mirror-logs
+  - /var/log/s3-mirror-logs/bucket
+
+- name: s3sync and symmetric_diff scripts
+  copy: src={{item}} dest=/usr/local/bin/{{item}} owner=s3-mirror group=s3-mirror mode=0755
+  with_items:
+  - s3sync
+  - symmetric_diff
+
+- name: s3-mirror-excludes.txt
+  copy: src=s3-excludes.txt dest=/usr/local/etc/s3-mirror-excludes.txt owner=s3-mirror group=s3-mirror mode=0644
+
+- name: s3-mirror logrotate
+  copy: src=s3-mirror.logrotate dest=/etc/logrotate.d/s3-mirror owner=s3-mirror group=s3-mirror mode=0644
+
+- name: s3-mirror init.d
+  copy: src=s3-mirror.init dest=/etc/init.d/s3-mirror owner=s3-mirror group=s3-mirror mode=0755
+
+- name: s3sync-logs
+  copy: src=s3sync-logs dest=/usr/local/bin/s3sync-logs owner=s3-mirror group=s3-mirror mode=0755

roles/s3-mirror/templates/s3cfg

@@ -0,0 +1,35 @@
+[default]
+access_key = {{ s3_mirror_aws_access_key }}
+secret_key = {{ s3_mirror_aws_secret_key }}
+acl_public = False
+bucket_location = US
+cloudfront_host = cloudfront.amazonaws.com
+cloudfront_resource = /2008-06-30/distribution
+default_mime_type = binary/octet-stream
+delete_removed = False
+dry_run = False
+encoding = UTF-8
+encrypt = False
+force = False
+get_continue = False
+gpg_command = /usr/bin/gpg
+gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_passphrase =
+guess_mime_type = True
+host_base = s3.amazonaws.com
+host_bucket = %(bucket)s.s3.amazonaws.com
+human_readable_sizes = False
+list_md5 = False
+preserve_attrs = True
+progress_meter = True
+proxy_host =
+proxy_port = 0
+recursive = False
+recv_chunk = 4096
+send_chunk = 4096
+simpledb_host = sdb.amazonaws.com
+skip_existing = False
+urlencoding_mode = normal
+use_https = False
+verbosity = WARNING