From 7a013fe51192d30f15e8de98d724c8546dea5905 Mon Sep 17 00:00:00 2001
From: Adam Saleh <asaleh@redhat.com>
Date: Tue, 27 Jul 2021 08:49:22 +0200
Subject: [PATCH] Send tracing messages to the bus in syncHttpLogs

In the course, fix a typo which reduces stdout spam.

Signed-off-by: Adam Saleh <asaleh@redhat.com>
Signed-off-by: Nils Philippsen <nils@redhat.com>
---
 roles/web-data-analysis/files/syncHttpLogs.sh | 68 ++++++++++++++++---
 1 file changed, 57 insertions(+), 11 deletions(-)

diff --git a/roles/web-data-analysis/files/syncHttpLogs.sh b/roles/web-data-analysis/files/syncHttpLogs.sh
index a34bba4cb5..5e70a6f3ec 100644
--- a/roles/web-data-analysis/files/syncHttpLogs.sh
+++ b/roles/web-data-analysis/files/syncHttpLogs.sh
@@ -3,12 +3,49 @@
 RSYNC_FLAGS='-avSHP --no-motd --timeout=1200 --contimeout=1200'
 DEBUG=1
 
-function syncHttpLogs {
+RUN_ID="$(uuidgen -r)"
+LOGHOST="$(hostname)"
+MSGTOPIC_PREFIX="logging.stats"
 
+function send_bus_msg {
+    local topic="${MSGTOPIC_PREFIX}.$1"
+    shift
+    local sent_at="$(TZ=UTC date -Iseconds)"
+    local id="$(uuidgen -r)"
+    local body_piece
+    local body="{"
+    local sep=""
+
+    for body_piece; do
+        local key_type key type value
+        key_type="${body_piece%%=*}"
+        key="${key_type%%:*}"
+        type="${key_type#${key}}"
+        type="${type#:}"
+        value="${body_piece#*=}"
+
+        if [ "$type" != "int" ]; then
+            # quote strings
+            value="${value//\\/\\\\}"
+            value="${value//\"/\\\"}"
+            value="\"${value}\""
+        fi
+        body="${body}${sep}\"${key}\": ${value}"
+        sep=", "
+    done
+    body="${body}}"
+
+    fedora-messaging publish - << EOF >/dev/null
+{"body": ${body}, "headers": {"fedora_messaging_schema": "base.message", "fedora_messaging_severity": 20, "sent-at": "${sent_at}"}, "id": "${id}", "queue": "queue", "topic": "${topic}"}
+EOF
+}
+
+function syncHttpLogs {
+    HOST=$1
+    send_bus_msg sync.host.start host="$LOGHOST" run_id="$RUN_ID" synced_host="$HOST"
     # in case we missed a run or two.. try to catch up the last 3 days.
     for d in 1 2 3; do
-        HOST=$1
-	# some machines store stuff in old format. some new.
+    # some machines store stuff in old format. some new.
         if [ "$2" = "old" ]; then
             YESTERDAY=$(/bin/date -d "-$d days" +%Y-%m-%d)
         else
@@ -17,25 +54,32 @@ function syncHttpLogs {
         YEAR=$(/bin/date -d "-$d days" +%Y)
         MONTH=$(/bin/date -d "-$d days" +%m)
         DAY=$(/bin/date -d "-$d days" +%d)
+        send_bus_msg sync.host.logdate.start host="$LOGHOST" run_id="$RUN_ID" synced_host="$HOST" log_date="${YEAR}-${MONTH}-${DAY}"
         /bin/mkdir -p /var/log/hosts/$HOST/$YEAR/$MONTH/$DAY/http
         cd /var/log/hosts/$HOST/$YEAR/$MONTH/$DAY/http/
-
-	RSYNC_OUTPUT=$(/usr/bin/rsync $RSYNC_FLAGS --list-only $HOST::log/httpd/*$YESTERDAY* | grep xz$ | awk '{ print $5 }' )
+        RSYNC_OUTPUT=$(/usr/bin/rsync $RSYNC_FLAGS --list-only $HOST::log/httpd/*$YESTERDAY* | grep xz$ | awk '{ print $5 }' )
         for f in ${RSYNC_OUTPUT}; do
             DEST=$(echo $f | /bin/sed s/-$YESTERDAY//)
-	        if [[ ${DEBUG} -eq 1 ]]; then
-		        echo "${HOST}: Getting ${RSYNC_OUTPUT} and saving to ${DEST}"
-	        fi
+            if [[ ${DEBUG} -eq 1 ]]; then
+                echo "${HOST}: Getting ${f} and saving to ${DEST}"
+            fi
             for i in 2 1 0; do
                 timeout 2h /usr/bin/rsync $RSYNC_FLAGS $HOST::log/httpd/$f ./$DEST &> /dev/null && break
-	            if [[ $? -ne 0 ]]; then
-		            echo "rsync from $HOST for file $f failed, will repeat $i times"
-	            fi
+                if [[ $? -ne 0 ]]; then
+                    send_bus_msg sync.host.logdate.fail.retry host="$LOGHOST" run_id="$RUN_ID" synced_host="$HOST" log_date="${YEAR}-${MONTH}-${DAY}" failure="Error code: $?"
+                    echo "rsync from $HOST for file $f failed, will repeat $i times"
+                else
+                    send_bus_msg sync.host.logdate.fail.final host="$LOGHOST" run_id="$RUN_ID" synced_host="$HOST" log_date="${YEAR}-${MONTH}-${DAY}" failure="Error code: $?"
+                fi
             done
         done
+        send_bus_msg sync.host.logdate.finish host="$LOGHOST" run_id="$RUN_ID" synced_host="$HOST" log_date="${YEAR}-${MONTH}-${DAY}"
     done
+    send_bus_msg sync.host.finish host="$LOGHOST" run_id="$RUN_ID" synced_host="$HOST"
 }
 
+send_bus_msg sync.start host="$LOGHOST" run_id="$RUN_ID"
+
 syncHttpLogs proxy01.iad2.fedoraproject.org
 syncHttpLogs proxy02.vpn.fedoraproject.org
 syncHttpLogs proxy03.vpn.fedoraproject.org
@@ -89,4 +133,6 @@ syncHttpLogs download-cc-rdu01.vpn.fedoraproject.org
 syncHttpLogs sundries01.iad2.fedoraproject.org
 # syncHttpLogs sundries02.iad2.fedoraproject.org
 # syncHttpLogs sundries01.stg.iad2.fedoraproject.org
+
+send_bus_msg sync.finish host="$LOGHOST" run_id="$RUN_ID"
 ## eof