Infrastructure/ansible
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

Send tracing messages to the bus in syncHttpLogs

Browse source 
In the course, fix a typo which reduces stdout spam.

Signed-off-by: Adam Saleh <asaleh@redhat.com>
Signed-off-by: Nils Philippsen <nils@redhat.com>
This commit is contained in:
Adam Saleh 2021-07-27 08:49:22 +02:00 • committed by Nils Philippsen
parent e7faec028d
commit 7a013fe511
1 changed files with 57 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

68
roles/web-data-analysis/files/syncHttpLogs.sh
View file

@ -3,12 +3,49 @@
RSYNC_FLAGS='-avSHP --no-motd --timeout=1200 --contimeout=1200'
DEBUG=1
function syncHttpLogs {
RUN_ID="$(uuidgen -r)"
LOGHOST="$(hostname)"
MSGTOPIC_PREFIX="logging.stats"
function send_bus_msg {
local topic="${MSGTOPIC_PREFIX}.$1"
shift
local sent_at="$(TZ=UTC date -Iseconds)"
local id="$(uuidgen -r)"
local body_piece
local body="{"
local sep=""
for body_piece; do
local key_type key type value
key_type="${body_piece%%=*}"
key="${key_type%%:*}"
type="${key_type#${key}}"
type="${type#:}"
value="${body_piece#*=}"
if [ "$type" != "int" ]; then
# quote strings
value="${value//\\/\\\\}"
value="${value//\"/\\\"}"
value="\"${value}\""
fi
body="${body}${sep}\"${key}\": ${value}"
sep=", "
done
body="${body}}"
fedora-messaging publish - << EOF >/dev/null
{"body": ${body}, "headers": {"fedora_messaging_schema": "base.message", "fedora_messaging_severity": 20, "sent-at": "${sent_at}"}, "id": "${id}", "queue": "queue", "topic": "${topic}"}
EOF
}
function syncHttpLogs {
HOST=$1
send_bus_msg sync.host.start host="$LOGHOST" run_id="$RUN_ID" synced_host="$HOST"
# in case we missed a run or two.. try to catch up the last 3 days.
for d in 1 2 3; do
HOST=$1
# some machines store stuff in old format. some new.
# some machines store stuff in old format. some new.
if [ "$2" = "old" ]; then
YESTERDAY=$(/bin/date -d "-$d days" +%Y-%m-%d)
else
@ -17,25 +54,32 @@ function syncHttpLogs {
YEAR=$(/bin/date -d "-$d days" +%Y)
MONTH=$(/bin/date -d "-$d days" +%m)
DAY=$(/bin/date -d "-$d days" +%d)
send_bus_msg sync.host.logdate.start host="$LOGHOST" run_id="$RUN_ID" synced_host="$HOST" log_date="${YEAR}-${MONTH}-${DAY}"
/bin/mkdir -p /var/log/hosts/$HOST/$YEAR/$MONTH/$DAY/http
cd /var/log/hosts/$HOST/$YEAR/$MONTH/$DAY/http/
RSYNC_OUTPUT=$(/usr/bin/rsync $RSYNC_FLAGS --list-only $HOST::log/httpd/*$YESTERDAY* | grep xz$ | awk '{ print $5 }' )
RSYNC_OUTPUT=$(/usr/bin/rsync $RSYNC_FLAGS --list-only $HOST::log/httpd/*$YESTERDAY* | grep xz$ | awk '{ print $5 }' )
for f in ${RSYNC_OUTPUT}; do
DEST=$(echo $f | /bin/sed s/-$YESTERDAY//)
if [[ ${DEBUG} -eq 1 ]]; then
echo "${HOST}: Getting ${RSYNC_OUTPUT} and saving to ${DEST}"
fi
if [[ ${DEBUG} -eq 1 ]]; then
echo "${HOST}: Getting ${f} and saving to ${DEST}"
fi
for i in 2 1 0; do
timeout 2h /usr/bin/rsync $RSYNC_FLAGS $HOST::log/httpd/$f ./$DEST &> /dev/null && break
if [[ $? -ne 0 ]]; then
echo "rsync from $HOST for file $f failed, will repeat $i times"
fi
if [[ $? -ne 0 ]]; then
send_bus_msg sync.host.logdate.fail.retry host="$LOGHOST" run_id="$RUN_ID" synced_host="$HOST" log_date="${YEAR}-${MONTH}-${DAY}" failure="Error code: $?"
echo "rsync from $HOST for file $f failed, will repeat $i times"
else
send_bus_msg sync.host.logdate.fail.final host="$LOGHOST" run_id="$RUN_ID" synced_host="$HOST" log_date="${YEAR}-${MONTH}-${DAY}" failure="Error code: $?"
fi
done
done
send_bus_msg sync.host.logdate.finish host="$LOGHOST" run_id="$RUN_ID" synced_host="$HOST" log_date="${YEAR}-${MONTH}-${DAY}"
done
send_bus_msg sync.host.finish host="$LOGHOST" run_id="$RUN_ID" synced_host="$HOST"
}
send_bus_msg sync.start host="$LOGHOST" run_id="$RUN_ID"
syncHttpLogs proxy01.iad2.fedoraproject.org
syncHttpLogs proxy02.vpn.fedoraproject.org
syncHttpLogs proxy03.vpn.fedoraproject.org
@ -89,4 +133,6 @@ syncHttpLogs download-cc-rdu01.vpn.fedoraproject.org
syncHttpLogs sundries01.iad2.fedoraproject.org
# syncHttpLogs sundries02.iad2.fedoraproject.org
# syncHttpLogs sundries01.stg.iad2.fedoraproject.org
send_bus_msg sync.finish host="$LOGHOST" run_id="$RUN_ID"
## eof