Move syncHttpLogs.sh into web-data-analysis role

Signed-off-by: Nils Philippsen <nils@redhat.com>

roles/web-data-analysis/files/syncHttpLogs.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+RSYNC_FLAGS='-avSHP --no-motd --timeout=1200 --contimeout=1200'
+DEBUG=1
+
+function syncHttpLogs {
+
+    # in case we missed a run or two.. try to catch up the last 3 days.
+    for d in 1 2 3; do
+        HOST=$1
+	# some machines store stuff in old format. some new.
+        if [ "$2" = "old" ]; then
+            YESTERDAY=$(/bin/date -d "-$d days" +%Y-%m-%d)
+        else
+            YESTERDAY=$(/bin/date -d "-$d days" +%Y%m%d)
+        fi
+        YEAR=$(/bin/date -d "-$d days" +%Y)
+        MONTH=$(/bin/date -d "-$d days" +%m)
+        DAY=$(/bin/date -d "-$d days" +%d)
+        /bin/mkdir -p /var/log/hosts/$HOST/$YEAR/$MONTH/$DAY/http
+        cd /var/log/hosts/$HOST/$YEAR/$MONTH/$DAY/http/
+
+	RSYNC_OUTPUT=$(/usr/bin/rsync $RSYNC_FLAGS --list-only $HOST::log/httpd/*$YESTERDAY* | grep xz$ | awk '{ print $5 }' )
+        for f in ${RSYNC_OUTPUT}; do
+            DEST=$(echo $f | /bin/sed s/-$YESTERDAY//)
+	        if [[ ${DEBUG} -eq 1 ]]; then
+		        echo "${HOST}: Getting ${RSYNC_OUTPUT} and saving to ${DEST}"
+	        fi
+            for i in 2 1 0; do
+                timeout 2h /usr/bin/rsync $RSYNC_FLAGS $HOST::log/httpd/$f ./$DEST &> /dev/null && break
+	            if [[ $? -ne 0 ]]; then
+		            echo "rsync from $HOST for file $f failed, will repeat $i times"
+	            fi
+            done
+        done
+    done
+}
+
+syncHttpLogs proxy01.iad2.fedoraproject.org
+syncHttpLogs proxy02.vpn.fedoraproject.org
+syncHttpLogs proxy03.vpn.fedoraproject.org
+syncHttpLogs proxy04.vpn.fedoraproject.org
+syncHttpLogs proxy05.vpn.fedoraproject.org
+syncHttpLogs proxy06.vpn.fedoraproject.org
+# syncHttpLogs proxy08.vpn.fedoraproject.org
+syncHttpLogs proxy09.vpn.fedoraproject.org # proxy09 is acting up
+syncHttpLogs proxy10.iad2.fedoraproject.org
+syncHttpLogs proxy11.vpn.fedoraproject.org
+syncHttpLogs proxy12.vpn.fedoraproject.org
+syncHttpLogs proxy13.vpn.fedoraproject.org
+syncHttpLogs proxy14.vpn.fedoraproject.org
+syncHttpLogs proxy30.vpn.fedoraproject.org
+syncHttpLogs proxy31.vpn.fedoraproject.org
+syncHttpLogs proxy32.vpn.fedoraproject.org
+syncHttpLogs proxy33.vpn.fedoraproject.org
+syncHttpLogs proxy34.vpn.fedoraproject.org
+syncHttpLogs proxy35.vpn.fedoraproject.org
+syncHttpLogs proxy36.vpn.fedoraproject.org
+syncHttpLogs proxy37.vpn.fedoraproject.org
+syncHttpLogs proxy38.vpn.fedoraproject.org
+syncHttpLogs proxy39.vpn.fedoraproject.org
+syncHttpLogs proxy40.vpn.fedoraproject.org
+syncHttpLogs proxy101.iad2.fedoraproject.org
+syncHttpLogs proxy110.iad2.fedoraproject.org
+# syncHttpLogs proxy01.stg.iad2.fedoraproject.org
+syncHttpLogs datagrepper01.iad2.fedoraproject.org
+# syncHttpLogs datagrepper02.iad2.fedoraproject.org
+# syncHttpLogs datagrepper01.stg.iad2.fedoraproject.org
+# syncHttpLogs badges-web01.iad2.fedoraproject.org
+# syncHttpLogs badges-web02.iad2.fedoraproject.org
+# syncHttpLogs badges-web01.stg.iad2.fedoraproject.org
+# syncHttpLogs packages03.iad2.fedoraproject.org
+# syncHttpLogs packages04.iad2.fedoraproject.org
+# syncHttpLogs packages03.stg.iad2.fedoraproject.org
+syncHttpLogs blockerbugs01.iad2.fedoraproject.org
+# syncHttpLogs blockerbugs02.iad2.fedoraproject.org
+# syncHttpLogs blockerbugs01.stg.iad2.fedoraproject.org
+syncHttpLogs value01.iad2.fedoraproject.org
+syncHttpLogs people02.vpn.fedoraproject.org
+syncHttpLogs noc01.iad2.fedoraproject.org
+syncHttpLogs dl01.iad2.fedoraproject.org
+syncHttpLogs dl02.iad2.fedoraproject.org
+syncHttpLogs dl03.iad2.fedoraproject.org
+syncHttpLogs dl04.iad2.fedoraproject.org
+syncHttpLogs dl05.iad2.fedoraproject.org
+syncHttpLogs download-rdu01.vpn.fedoraproject.org
+syncHttpLogs download-ib01.vpn.fedoraproject.org
+syncHttpLogs download-cc-rdu01.vpn.fedoraproject.org
+syncHttpLogs sundries01.iad2.fedoraproject.org
+# syncHttpLogs sundries02.iad2.fedoraproject.org
+# syncHttpLogs sundries01.stg.iad2.fedoraproject.org
+## eof

roles/web-data-analysis/tasks/main.yml

@@ -150,3 +150,10 @@
   tags:
     - web-data
     - cron
+
+- name: install a sync httpd logs cron script only on log01
+  copy: src=syncHttpLogs.sh dest=/etc/cron.daily/syncHttpLogs.sh mode=0755
+  when: inventory_hostname.startswith('log01')
+  tags:
+    - web-data
+    - config