From 7984e52034fc593a41399c47b2d6a66d40aceab3 Mon Sep 17 00:00:00 2001 From: Rick Elrod Date: Tue, 9 Oct 2018 17:17:47 +0000 Subject: [PATCH] attempt at cron-ify-ing new s3 sync Signed-off-by: Rick Elrod --- roles/s3-mirror/files/config | 3 + roles/s3-mirror/files/s3-excludes.txt | 46 ----------- roles/s3-mirror/files/s3-mirror.logrotate | 12 --- roles/s3-mirror/files/s3.sh | 93 +++++++++++++++++++++++ roles/s3-mirror/files/s3sync | 84 -------------------- roles/s3-mirror/files/symmetric_diff | 35 --------- roles/s3-mirror/tasks/main.yml | 71 ++++++++--------- roles/s3-mirror/templates/credentials | 3 + roles/s3-mirror/templates/s3cfg | 35 --------- 9 files changed, 135 insertions(+), 247 deletions(-) create mode 100644 roles/s3-mirror/files/config delete mode 100644 roles/s3-mirror/files/s3-excludes.txt delete mode 100644 roles/s3-mirror/files/s3-mirror.logrotate create mode 100644 roles/s3-mirror/files/s3.sh delete mode 100755 roles/s3-mirror/files/s3sync delete mode 100755 roles/s3-mirror/files/symmetric_diff create mode 100644 roles/s3-mirror/templates/credentials delete mode 100644 roles/s3-mirror/templates/s3cfg diff --git a/roles/s3-mirror/files/config b/roles/s3-mirror/files/config new file mode 100644 index 0000000000..812ac13053 --- /dev/null +++ b/roles/s3-mirror/files/config @@ -0,0 +1,3 @@ +[default] +region = us-west-2 +output = text diff --git a/roles/s3-mirror/files/s3-excludes.txt b/roles/s3-mirror/files/s3-excludes.txt deleted file mode 100644 index ab82e6f188..0000000000 --- a/roles/s3-mirror/files/s3-excludes.txt +++ /dev/null @@ -1,46 +0,0 @@ -.snapshot/ -source/ -SRPMS/ -debug/ -beta/ -ppc/ -ppc64/ -repoview/ -Fedora/ -Live/ -isolinux/ -images/ -EFI/ -drpms/ -core/ -extras/ -LiveOS/ -development/ -releases/test/ -releases/8/ -releases/9/ -releases/10/ -releases/11/ -releases/12/ -releases/13/ -releases/14/ -releases/15/ -releases/16/ -updates/8/ -updates/9/ -updates/10/ -updates/11/ -updates/12/ -updates/13/ -updates/14/ -updates/15/ -updates/16/ -updates/testing/8/ -updates/testing/9/ -updates/testing/10/ -updates/testing/11/ -updates/testing/12/ -updates/testing/13/ -updates/testing/14/ -updates/testing/15/ -updates/testing/16/ diff --git a/roles/s3-mirror/files/s3-mirror.logrotate b/roles/s3-mirror/files/s3-mirror.logrotate deleted file mode 100644 index 754902880b..0000000000 --- a/roles/s3-mirror/files/s3-mirror.logrotate +++ /dev/null @@ -1,12 +0,0 @@ -/var/log/s3-mirror/*log { - daily - rotate 7 - missingok - ifempty - compress - compresscmd /usr/bin/bzip2 - uncompresscmd /usr/bin/bunzip2 - compressext .bz2 - dateext - copytruncate -} diff --git a/roles/s3-mirror/files/s3.sh b/roles/s3-mirror/files/s3.sh new file mode 100644 index 0000000000..c1dd2c47d5 --- /dev/null +++ b/roles/s3-mirror/files/s3.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# (c) 2018 Red Hat, Inc. +# LGPL +# Author: Rick Elrod + +CMD="aws s3 sync \ + --delete \ + --exclude */.snapshot/* \ + --exclude */source/* \ + --exclude */SRPMS/* \ + --exclude */debug/* \ + --exclude */beta/* \ + --exclude */ppc/* \ + --exclude */ppc64/* \ + --exclude */repoview/* \ + --exclude */Fedora/* \ + --exclude */Live/* \ + --exclude */isolinux/* \ + --exclude */images/* \ + --exclude */EFI/* \ + --exclude */core/* \ + --exclude */extras/* \ + --exclude */LiveOS/* \ + --exclude */development/* \ + --exclude */releases/test/* \ + --exclude */releases/8/* \ + --exclude */releases/9/* \ + --exclude */releases/10/* \ + --exclude */releases/11/* \ + --exclude */releases/12/* \ + --exclude */releases/13/* \ + --exclude */releases/14/* \ + --exclude */releases/15/* \ + --exclude */releases/16/* \ + --exclude */releases/17/* \ + --exclude */releases/18/* \ + --exclude */releases/19/* \ + --exclude */releases/20/* \ + --exclude */releases/21/* \ + --exclude */releases/22/* \ + --exclude */releases/23/* \ + --exclude */releases/24/* \ + --exclude */releases/25/* \ + --exclude */releases/26/* \ + --exclude */updates/8/* \ + --exclude */updates/9/* \ + --exclude */updates/10/* \ + --exclude */updates/11/* \ + --exclude */updates/12/* \ + --exclude */updates/13/* \ + --exclude */updates/14/* \ + --exclude */updates/15/* \ + --exclude */updates/16/* \ + --exclude */updates/17/* \ + --exclude */updates/18/* \ + --exclude */updates/19/* \ + --exclude */updates/20/* \ + --exclude */updates/21/* \ + --exclude */updates/22/* \ + --exclude */updates/23/* \ + --exclude */updates/24/* \ + --exclude */updates/25/* \ + --exclude */updates/26/* \ + --exclude */updates/testing/8/* \ + --exclude */updates/testing/9/* \ + --exclude */updates/testing/10/* \ + --exclude */updates/testing/11/* \ + --exclude */updates/testing/12/* \ + --exclude */updates/testing/13/* \ + --exclude */updates/testing/14/* \ + --exclude */updates/testing/15/* \ + --exclude */updates/testing/16/* \ + --exclude */updates/testing/17/* \ + --exclude */updates/testing/18/* \ + --exclude */updates/testing/19/* \ + --exclude */updates/testing/20/* \ + --exclude */updates/testing/21/* \ + --exclude */updates/testing/22/* \ + --exclude */updates/testing/23/* \ + --exclude */updates/testing/24/* \ + --exclude */updates/testing/25/* \ + --exclude */updates/testing/26/* \ + --no-follow-symlinks \ + " + #--dryrun \ + +# Sync EPEL +echo $CMD /pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/ +$CMD /pub/epel/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/epel/ + +# Sync Fedora +echo $CMD /pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/ +$CMD /pub/fedora/ s3://s3-mirror-us-west-1-02.fedoraproject.org/pub/fedora/ diff --git a/roles/s3-mirror/files/s3sync b/roles/s3-mirror/files/s3sync deleted file mode 100755 index f8d04137ca..0000000000 --- a/roles/s3-mirror/files/s3sync +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -logfile="/var/log/s3-mirror/s3sync.log" -tree="/srv/pub/" -s3cmd=/usr/bin/s3cmd -curdate=`date +%s` - -if [ ! -d ${tree} ]; then - echo "Specified root of the directory tree (${tree}) does not exist. Exiting." | tee ${logfile} - exit 1 -fi - -exec 1>>${logfile} 2>&1 - -function newer() -{ - if [ "$1" -nt "$2" ] ; then - return 0 - fi - return 1 -} - -S3CMD_ARGS="sync \ - --verbose \ - --preserve \ - --recursive \ - --check-md5 \ - --delay-updates \ - --guess-mime-type --mime-type application/octet-stream \ - --reduced-redundancy \ - --acl-public \ - --exclude-from /usr/local/etc/s3-mirror-excludes.txt \ - " - -content="epel" -targets="s3-mirror-us-east-1 s3-mirror-us-west-1 s3-mirror-us-west-2 s3-mirror-eu-west-1 s3-mirror-ap-northeast-1" -report=0 - -if [ -e /var/run/s3-mirror/lastrun ]; then - . /var/run/s3-mirror/lastrun -else - # 48 hours -> 172800 seconds - let lastrun=curdate-172800 -fi - -let delta=curdate-lastrun - -# hardcoded to epel only '-e' -/usr/local/bin/last-sync -d ${delta} -e - -if [ "$?" -ne "0" ]; then - # no changes since the last sync - # abort - exit 0 -fi - -for c in ${content}; do - if $(newer ${tree}/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist) ; then - echo "==============================================================" - echo -n "Starting at " - date - time $s3cmd $S3CMD_ARGS \ - --cache-file /var/lib/s3-mirror/${c}.cache \ - --delete-removed \ - --delete-after \ - --add-destination s3://s3-mirror-us-west-1.fedoraproject.org/pub/${c}/ \ - --add-destination s3://s3-mirror-us-west-2.fedoraproject.org/pub/${c}/ \ - --add-destination s3://s3-mirror-eu-west-1.fedoraproject.org/pub/${c}/ \ - --add-destination s3://s3-mirror-ap-northeast-1.fedoraproject.org/pub/${c}/ \ - ${tree}/${c}/ s3://s3-mirror-us-east-1.fedoraproject.org/pub/${c}/ - report=1 - cp -a ${tree}/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist - echo "lastrun=${curdate}" > /var/run/s3-mirror/lastrun - fi -done - - -if [ ${report} -ne 0 ] ; then - for target in ${targets} ; do - report_mirror -c /etc/mirrormanager-client/report_mirror_${target}.conf --exclude-from /usr/local/etc/s3-mirror-excludes.txt - done - echo -n "Ending at " - date -fi diff --git a/roles/s3-mirror/files/symmetric_diff b/roles/s3-mirror/files/symmetric_diff deleted file mode 100755 index e7fb62c390..0000000000 --- a/roles/s3-mirror/files/symmetric_diff +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/python -# Copyright 2013 by Matt Domsch -# Licensed under the MIT/X11 license -# usage: symmetric_diff -# given two text files a and b, returns the symmetric difference of the lines changed between them. -# The order in which lines appear in each file is irrelevant. - -import sys -import optparse - -def read_file(filename): - f = open(filename, 'r') - lines = f.readlines() - f.close() - return set(lines) - -def diff(a, b): - file_a = read_file(a) - file_b = read_file(b) - sdiff = file_a.symmetric_difference(file_b) - sdiff = list(sdiff) - sdiff.sort() - return sdiff - -def main(): - parser = optparse.OptionParser(usage=sys.argv[0] + " [options]") - (options, args) = parser.parse_args() - if len(args) < 2: - parser.print_help() - sdiff = diff(args[0], args[1]) - for l in sdiff: - sys.stdout.write(l) - -if __name__ == "__main__": - sys.exit(main()) diff --git a/roles/s3-mirror/tasks/main.yml b/roles/s3-mirror/tasks/main.yml index ad96dc61b4..a41b0d6bae 100644 --- a/roles/s3-mirror/tasks/main.yml +++ b/roles/s3-mirror/tasks/main.yml @@ -1,26 +1,12 @@ - name: Create s3-mirror group group: gid=442 name=s3-mirror + tags: + - s3-mirror - name: Create s3-mirror user user: group=s3-mirror name=s3-mirror comment="s3-mirror user" home=/home/s3-mirror shell=/bin/bash - -- name: s3cfg file - template: src=s3cfg dest=/home/s3-mirror/.s3cfg owner=s3-mirror group=s3-mirror mode=0600 - -- name: Install packages - package: name={{ item }} state=present - with_items: - - s3cmd - - mirrormanager2-client - -- name: Create report_mirror config files - template: src=report_mirror.conf dest=/etc/mirrormanager-client/report_mirror_{{ item }}.conf owner=s3-mirror group=s3-mirror mode=0400 - with_items: - - s3-mirror-us-east-1 - - s3-mirror-us-west-1 - - s3-mirror-us-west-2 - - s3-mirror-eu-west-1 - - s3-mirror-ap-northeast-1 + tags: + - s3-mirror - name: Create needed directories file: path={{item}} owner=s3-mirror group=s3-mirror state=directory mode=0755 @@ -28,29 +14,44 @@ - /var/log/s3-mirror - /var/lib/s3-mirror - /var/run/s3-mirror - - /var/log/s3-mirror-logs - - /var/log/s3-mirror-logs/bucket + - /home/s3-mirror/.aws + #- /var/log/s3-mirror-logs + #- /var/log/s3-mirror-logs/bucket + tags: + - s3-mirror -- name: s3sync and symmetric_diff scripts +- name: aws credentials file + template: src=credentials dest=/home/s3-mirror/.aws/credentials owner=s3-mirror group=s3-mirror mode=0600 + tags: + - s3-mirror + +- name: aws config file + copy: src=config dest=/home/s3-mirror/.aws/config owner=s3-mirror group=s3-mirror mode=0600 + tags: + - s3-mirror + +- name: Install packages + package: name={{ item }} state=present + with_items: + - awscli + # - mirrormanager2-client + tags: + - s3-mirror + +- name: s3 script copy: src={{item}} dest=/usr/local/bin/{{item}} owner=s3-mirror group=s3-mirror mode=0755 with_items: - - s3sync - - symmetric_diff - -- name: s3-mirror-excludes.txt - copy: src=s3-excludes.txt dest=/usr/local/etc/s3-mirror-excludes.txt owner=s3-mirror group=s3-mirror mode=0644 - -- name: s3-mirror logrotate - copy: src=s3-mirror.logrotate dest=/etc/logrotate.d/s3-mirror owner=root group=root mode=0644 - -- name: s3sync-logs - copy: src=s3sync-logs dest=/usr/local/bin/s3sync-logs owner=s3-mirror group=s3-mirror mode=0755 + - s3.sh + tags: + - s3-mirror - name: s3sync cron - cron: name="s3sync" minute="3,18,33,48" user="s3-mirror" - job='/usr/local/bin/lock-wrapper s3sync /usr/local/bin/s3sync 2>&1 | /usr/local/bin/nag-once s3sync 1d 2>&1' - cron_file=s3sync + cron: name="s3sync" minute="0" hour="0,11" user="s3-mirror" + job='/usr/local/bin/lock-wrapper s3sync /usr/local/bin/s3.sh 2>&1 | /usr/local/bin/nag-once s3.sh 1d 2>&1' + cron_file=s3.sh when: env != 'staging' + tags: + - s3-mirror #- name: s3sync-logs cron # cron: name="s3sync-logs" hour="0" user="root" diff --git a/roles/s3-mirror/templates/credentials b/roles/s3-mirror/templates/credentials new file mode 100644 index 0000000000..e1e72c6b0c --- /dev/null +++ b/roles/s3-mirror/templates/credentials @@ -0,0 +1,3 @@ +[default] +aws_access_key_id = {{ s3_mirror_aws_access_key }} +aws_secret_access_key = {{ s3_mirror_aws_secret_key }} diff --git a/roles/s3-mirror/templates/s3cfg b/roles/s3-mirror/templates/s3cfg deleted file mode 100644 index 0f16bec68d..0000000000 --- a/roles/s3-mirror/templates/s3cfg +++ /dev/null @@ -1,35 +0,0 @@ -[default] -access_key = {{ s3_mirror_aws_access_key }} -secret_key = {{ s3_mirror_aws_secret_key }} -acl_public = False -bucket_location = US -cloudfront_host = cloudfront.amazonaws.com -cloudfront_resource = /2008-06-30/distribution -default_mime_type = binary/octet-stream -delete_removed = False -dry_run = False -encoding = UTF-8 -encrypt = False -force = False -get_continue = False -gpg_command = /usr/bin/gpg -gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s -gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s -gpg_passphrase = -guess_mime_type = True -host_base = s3.amazonaws.com -host_bucket = %(bucket)s.s3.amazonaws.com -human_readable_sizes = False -list_md5 = False -preserve_attrs = True -progress_meter = True -proxy_host = -proxy_port = 0 -recursive = False -recv_chunk = 4096 -send_chunk = 4096 -simpledb_host = sdb.amazonaws.com -skip_existing = False -urlencoding_mode = normal -use_https = False -verbosity = WARNING