From cb61463c2645cbd6be5472505eb1c20022ef6f57 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Tue, 7 Sep 2021 15:09:55 +0200 Subject: [PATCH] Allow presets for message bodies This lets users of simple_message_to_bus predefine items which should be present in all message bodies this way: export MSGBODY_PRESET="key1=value1 key2=value2" This doesn't work with spaces in either keys or values, any quotation will be used verbatim. Signed-off-by: Nils Philippsen --- .../files/simple_message_to_bus | 4 +++- .../web-data-analysis/files/combineHttpLogs.sh | 18 +++++++++--------- .../files/condense-mirrorlogs.sh | 10 +++++----- .../web-data-analysis/files/countme-update.sh | 10 +++++----- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/roles/fedora-messaging-utils/files/simple_message_to_bus b/roles/fedora-messaging-utils/files/simple_message_to_bus index 2e8a17f157..dcddc8beea 100644 --- a/roles/fedora-messaging-utils/files/simple_message_to_bus +++ b/roles/fedora-messaging-utils/files/simple_message_to_bus @@ -11,7 +11,9 @@ id="$(uuidgen -r)" body="{" sep="" -for body_piece; do +# To preset items in the message body, do `export MSGBODY_PRESET="key1=value1 key2=value2"` before +# calling simple_message_to_bus. This doesn't work with spaces in the keys or values, sorry. +for body_piece in $MSGBODY_PRESET "$@"; do key_type="${body_piece%%=*}" key="${key_type%%:*}" type="${key_type#${key}}" diff --git a/roles/web-data-analysis/files/combineHttpLogs.sh b/roles/web-data-analysis/files/combineHttpLogs.sh index 73d1f5d8c9..d2b4e47884 100644 --- a/roles/web-data-analysis/files/combineHttpLogs.sh +++ b/roles/web-data-analysis/files/combineHttpLogs.sh @@ -20,8 +20,8 @@ # not, see . export MSGTOPIC_PREFIX=logging.stats -RUN_ID="$(uuidgen -r)" -simple_message_to_bus combinehttplogs.start run_id="$RUN_ID" +export MSGBODY_PRESET="loghost=$(hostname) run_id=$(uuidgen -r)" +simple_message_to_bus combinehttplogs.start # Some constants / standard paths LOGDIR=/var/log/hosts @@ -83,9 +83,9 @@ FILES=$( ls -1 ${PROXYLOG}/*access.log.xz | awk '{x=split($0,a,"/"); print a[x]} for FILE in ${FILES}; do TEMP=$(echo ${FILE} | sed 's/\.xz$//') - simple_message_to_bus combinehttplogs.logmerge.proxy.start run_id="$RUN_ID" log="$PROXYLOG" file="$FILE" target="$TARGET" temp="$TEMP" + simple_message_to_bus combinehttplogs.logmerge.proxy.start log="$PROXYLOG" file="$FILE" target="$TARGET" temp="$TEMP" perl ${LOGMERGE} ${PROXYLOG}/${FILE} > ${TARGET}/${TEMP} - simple_message_to_bus combinehttplogs.logmerge.proxy.finish run_id="$RUN_ID" log="$PROXYLOG" file="$FILE" target="$TARGET" temp="$TEMP" result="$?" + simple_message_to_bus combinehttplogs.logmerge.proxy.finish log="$PROXYLOG" file="$FILE" target="$TARGET" temp="$TEMP" result="$?" done ## @@ -94,9 +94,9 @@ FILES=$( ls -1 ${DL_LOG}/dl*access.log.xz | awk '{x=split($0,a,"/"); print a[x]} for FILE in ${FILES}; do TEMP=$(echo ${FILE} | sed 's/\.xz$//') - simple_message_to_bus combinehttplogs.logmerge.download.start run_id="$RUN_ID" proxylog="$DL_LOG" file="$FILE" target="$TARGET" temp="$TEMP" + simple_message_to_bus combinehttplogs.logmerge.download.start proxylog="$DL_LOG" file="$FILE" target="$TARGET" temp="$TEMP" perl ${LOGMERGE} ${DL_LOG}/${FILE} > ${TARGET}/${TEMP} - simple_message_to_bus combinehttplogs.logmerge.download.finish run_id="$RUN_ID" proxylog="$DL_LOG" file="$FILE" target="$TARGET" temp="$TEMP" result="$?" + simple_message_to_bus combinehttplogs.logmerge.download.finish proxylog="$DL_LOG" file="$FILE" target="$TARGET" temp="$TEMP" result="$?" done ## @@ -107,9 +107,9 @@ FILES=$( ls -1 ${PEOPLE}/fedora*access.log.xz | awk '{x=split($0,a,"/"); print a for FILE in ${FILES}; do TEMP=$(echo ${FILE} | sed 's/\.xz$//') - simple_message_to_bus combinehttplogs.logmerge.people.start run_id="$RUN_ID" proxylog="$PEOPLE" file="$FILE" target="$TARGET" temp="$TEMP" + simple_message_to_bus combinehttplogs.logmerge.people.start proxylog="$PEOPLE" file="$FILE" target="$TARGET" temp="$TEMP" perl ${LOGMERGE} ${PEOPLE}/${FILE} > ${TARGET}/${TEMP} - simple_message_to_bus combinehttplogs.logmerge.people.finish run_id="$RUN_ID" proxylog="$PEOPLE" file="$FILE" target="$TARGET" temp="$TEMP" result="$?" + simple_message_to_bus combinehttplogs.logmerge.people.finish proxylog="$PEOPLE" file="$FILE" target="$TARGET" temp="$TEMP" result="$?" done # Now we link up the files into latest directory @@ -126,4 +126,4 @@ if [[ "$UPDATE_LATEST" && -d ${NFSDIR}/latest ]]; then done popd &> /dev/null fi -simple_message_to_bus combinehttplogs.finish run_id="$RUN_ID" +simple_message_to_bus combinehttplogs.finish diff --git a/roles/web-data-analysis/files/condense-mirrorlogs.sh b/roles/web-data-analysis/files/condense-mirrorlogs.sh index 7d5f7dd862..40c382386f 100644 --- a/roles/web-data-analysis/files/condense-mirrorlogs.sh +++ b/roles/web-data-analysis/files/condense-mirrorlogs.sh @@ -27,8 +27,8 @@ # We have dropped this down to 3 days on 2019-10-01 export MSGTOPIC_PREFIX=logging.stats -RUN_ID="$(uuidgen -r)" -simple_message_to_bus condense-mirrorlogs.start run_id="$RUN_ID" +export MSGBODY_PRESET="loghost=$(hostname) run_id=$(uuidgen -r)" +simple_message_to_bus condense-mirrorlogs.start let NUMDAYS=3 let OLDDAYS=$(( $NUMDAYS+1 )) @@ -75,9 +75,9 @@ if [[ ! -f ${LOGFILE} ]]; then elif [[ -f ${WORKFILE} ]]; then echo "The workfile for ${YEAR}/${MONTH}/${DAY} already existed." else - simple_message_to_bus condense-mirrorlogs.mirrorlist.start run_id="$RUN_ID" workfile="$WORKFILE" logfile="$LOGFILE" + simple_message_to_bus condense-mirrorlogs.mirrorlist.start workfile="$WORKFILE" logfile="$LOGFILE" ${LBIN}/mirrorlist.py -o ${WORKFILE} ${LOGFILE}; - simple_message_to_bus condense-mirrorlogs.mirrorlist.finish run_id="$RUN_ID" workfile="$WORKFILE" logfile="$LOGFILE" result="$?" + simple_message_to_bus condense-mirrorlogs.mirrorlist.finish workfile="$WORKFILE" logfile="$LOGFILE" result="$?" fi # So the data isn't strictly across month boundries due to the end of @@ -112,4 +112,4 @@ gnuplot ${LSHARE}/${PROJECT}-data.gp # cleanup the temp data rm -rf ${TEMPDIR} -simple_message_to_bus condense-mirrorlogs.finish run_id="$RUN_ID" +simple_message_to_bus condense-mirrorlogs.finish diff --git a/roles/web-data-analysis/files/countme-update.sh b/roles/web-data-analysis/files/countme-update.sh index 9f3281d96c..5ccb0fc830 100644 --- a/roles/web-data-analysis/files/countme-update.sh +++ b/roles/web-data-analysis/files/countme-update.sh @@ -1,8 +1,8 @@ #!/bin/bash export MSGTOPIC_PREFIX=logging.stats -RUN_ID="$(uuidgen -r)" -simple_message_to_bus countme-update.start run_id="$RUN_ID" +export MSGBODY_PRESET="loghost=$(hostname) run_id=$(uuidgen -r)" +simple_message_to_bus countme-update.start # Where do we keep our local/internal data? LOCAL_DATA_DIR=/var/lib/countme @@ -38,10 +38,10 @@ _run() { if [ "$DRYRUN" ]; then return 0 else - simple_message_to_bus countme-update.command.start run_id="$RUN_ID" command="$@" + simple_message_to_bus countme-update.command.start command="$@" "$@" RESULT=$? - simple_message_to_bus countme-update.command.finish run_id="$RUN_ID" command="$@" result="$?" + simple_message_to_bus countme-update.command.finish command="$@" result="$?" return $RESULT fi } @@ -121,5 +121,5 @@ _run $_GIT diff --quiet || _run $_GIT commit -a -m "$(date -u +%F) update" _run atomic_copy $TOTALS_DB $PUBLIC_TOTALS_DB _run atomic_copy $TOTALS_CSV $PUBLIC_TOTALS_CSV -simple_message_to_bus countme-update.finish run_id="$RUN_ID" +simple_message_to_bus countme-update.finish