Infrastructure/ansible
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

Allow presets for message bodies

Browse source 
This lets users of simple_message_to_bus predefine items which should be
present in all message bodies this way:

export MSGBODY_PRESET="key1=value1 key2=value2"

This doesn't work with spaces in either keys or values, any quotation
will be used verbatim.

Signed-off-by: Nils Philippsen <nils@redhat.com>
This commit is contained in:
Nils Philippsen 2021-09-07 15:09:55 +02:00
parent a3203d29d9
commit cb61463c26
4 changed files with 22 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

18
roles/web-data-analysis/files/combineHttpLogs.sh
View file

@ -20,8 +20,8 @@
# not, see <http://www.gnu.org/licenses/>.
export MSGTOPIC_PREFIX=logging.stats
RUN_ID="$(uuidgen -r)"
simple_message_to_bus combinehttplogs.start run_id="$RUN_ID"
export MSGBODY_PRESET="loghost=$(hostname) run_id=$(uuidgen -r)"
simple_message_to_bus combinehttplogs.start
# Some constants / standard paths
LOGDIR=/var/log/hosts
@ -83,9 +83,9 @@ FILES=$( ls -1 ${PROXYLOG}/*access.log.xz | awk '{x=split($0,a,"/"); print a[x]}
for FILE in ${FILES}; do
TEMP=$(echo ${FILE} | sed 's/\.xz$//')
simple_message_to_bus combinehttplogs.logmerge.proxy.start run_id="$RUN_ID" log="$PROXYLOG" file="$FILE" target="$TARGET" temp="$TEMP"
simple_message_to_bus combinehttplogs.logmerge.proxy.start log="$PROXYLOG" file="$FILE" target="$TARGET" temp="$TEMP"
perl ${LOGMERGE} ${PROXYLOG}/${FILE} > ${TARGET}/${TEMP}
simple_message_to_bus combinehttplogs.logmerge.proxy.finish run_id="$RUN_ID" log="$PROXYLOG" file="$FILE" target="$TARGET" temp="$TEMP" result="$?"
simple_message_to_bus combinehttplogs.logmerge.proxy.finish log="$PROXYLOG" file="$FILE" target="$TARGET" temp="$TEMP" result="$?"
done
##
@ -94,9 +94,9 @@ FILES=$( ls -1 ${DL_LOG}/dl*access.log.xz | awk '{x=split($0,a,"/"); print a[x]}
for FILE in ${FILES}; do
TEMP=$(echo ${FILE} | sed 's/\.xz$//')
simple_message_to_bus combinehttplogs.logmerge.download.start run_id="$RUN_ID" proxylog="$DL_LOG" file="$FILE" target="$TARGET" temp="$TEMP"
simple_message_to_bus combinehttplogs.logmerge.download.start proxylog="$DL_LOG" file="$FILE" target="$TARGET" temp="$TEMP"
perl ${LOGMERGE} ${DL_LOG}/${FILE} > ${TARGET}/${TEMP}
simple_message_to_bus combinehttplogs.logmerge.download.finish run_id="$RUN_ID" proxylog="$DL_LOG" file="$FILE" target="$TARGET" temp="$TEMP" result="$?"
simple_message_to_bus combinehttplogs.logmerge.download.finish proxylog="$DL_LOG" file="$FILE" target="$TARGET" temp="$TEMP" result="$?"
done
##
@ -107,9 +107,9 @@ FILES=$( ls -1 ${PEOPLE}/fedora*access.log.xz | awk '{x=split($0,a,"/"); print a
for FILE in ${FILES}; do
TEMP=$(echo ${FILE} | sed 's/\.xz$//')
simple_message_to_bus combinehttplogs.logmerge.people.start run_id="$RUN_ID" proxylog="$PEOPLE" file="$FILE" target="$TARGET" temp="$TEMP"
simple_message_to_bus combinehttplogs.logmerge.people.start proxylog="$PEOPLE" file="$FILE" target="$TARGET" temp="$TEMP"
perl ${LOGMERGE} ${PEOPLE}/${FILE} > ${TARGET}/${TEMP}
simple_message_to_bus combinehttplogs.logmerge.people.finish run_id="$RUN_ID" proxylog="$PEOPLE" file="$FILE" target="$TARGET" temp="$TEMP" result="$?"
simple_message_to_bus combinehttplogs.logmerge.people.finish proxylog="$PEOPLE" file="$FILE" target="$TARGET" temp="$TEMP" result="$?"
done
# Now we link up the files into latest directory
@ -126,4 +126,4 @@ if [[ "$UPDATE_LATEST" && -d ${NFSDIR}/latest ]]; then
done
popd &> /dev/null
fi
simple_message_to_bus combinehttplogs.finish run_id="$RUN_ID"
simple_message_to_bus combinehttplogs.finish

10
roles/web-data-analysis/files/condense-mirrorlogs.sh
View file

@ -27,8 +27,8 @@
# We have dropped this down to 3 days on 2019-10-01
export MSGTOPIC_PREFIX=logging.stats
RUN_ID="$(uuidgen -r)"
simple_message_to_bus condense-mirrorlogs.start run_id="$RUN_ID"
export MSGBODY_PRESET="loghost=$(hostname) run_id=$(uuidgen -r)"
simple_message_to_bus condense-mirrorlogs.start
let NUMDAYS=3
let OLDDAYS=$(( $NUMDAYS+1 ))
@ -75,9 +75,9 @@ if [[ ! -f ${LOGFILE} ]]; then
elif [[ -f ${WORKFILE} ]]; then
echo "The workfile for ${YEAR}/${MONTH}/${DAY} already existed."
else
simple_message_to_bus condense-mirrorlogs.mirrorlist.start run_id="$RUN_ID" workfile="$WORKFILE" logfile="$LOGFILE"
simple_message_to_bus condense-mirrorlogs.mirrorlist.start workfile="$WORKFILE" logfile="$LOGFILE"
${LBIN}/mirrorlist.py -o ${WORKFILE} ${LOGFILE};
simple_message_to_bus condense-mirrorlogs.mirrorlist.finish run_id="$RUN_ID" workfile="$WORKFILE" logfile="$LOGFILE" result="$?"
simple_message_to_bus condense-mirrorlogs.mirrorlist.finish workfile="$WORKFILE" logfile="$LOGFILE" result="$?"
fi
# So the data isn't strictly across month boundries due to the end of
@ -112,4 +112,4 @@ gnuplot ${LSHARE}/${PROJECT}-data.gp
# cleanup the temp data
rm -rf ${TEMPDIR}
simple_message_to_bus condense-mirrorlogs.finish run_id="$RUN_ID"
simple_message_to_bus condense-mirrorlogs.finish

10
roles/web-data-analysis/files/countme-update.sh
View file

@ -1,8 +1,8 @@
#!/bin/bash
export MSGTOPIC_PREFIX=logging.stats
RUN_ID="$(uuidgen -r)"
simple_message_to_bus countme-update.start run_id="$RUN_ID"
export MSGBODY_PRESET="loghost=$(hostname) run_id=$(uuidgen -r)"
simple_message_to_bus countme-update.start
# Where do we keep our local/internal data?
LOCAL_DATA_DIR=/var/lib/countme
@ -38,10 +38,10 @@ _run() {
if [ "$DRYRUN" ]; then
return 0
else
simple_message_to_bus countme-update.command.start run_id="$RUN_ID" command="$@"
simple_message_to_bus countme-update.command.start command="$@"
"$@"
RESULT=$?
simple_message_to_bus countme-update.command.finish run_id="$RUN_ID" command="$@" result="$?"
simple_message_to_bus countme-update.command.finish command="$@" result="$?"
return $RESULT
fi
}
@ -121,5 +121,5 @@ _run $_GIT diff --quiet || _run $_GIT commit -a -m "$(date -u +%F) update"
_run atomic_copy $TOTALS_DB $PUBLIC_TOTALS_DB
_run atomic_copy $TOTALS_CSV $PUBLIC_TOTALS_CSV
simple_message_to_bus countme-update.finish run_id="$RUN_ID"
simple_message_to_bus countme-update.finish