Add more message-based tracing to log01 scripts

playbooks/groups/logserver.yml

@@ -23,6 +23,7 @@
   - sudo
   - awstats
   - web-data-analysis
+  - fedora-messaging-utils
   - role: keytab/service
     owner_user: apache
     owner_group: apache

roles/awstats/files/combineHttpLogs.sh

@@ -19,6 +19,9 @@
 # along with Fedora Project Infrastructure Ansible Repository.  If
 # not, see <http://www.gnu.org/licenses/>.
 
+RUN_ID="$(uuidgen -r)"
+simple_message_to_bus combinehttplogs.start run_id="$RUN_ID"
+
 # Some constants / standard paths
 LOGDIR=/var/log/hosts
 NFSDIR=/mnt/fedora_stats/combined-http
@@ -79,7 +82,9 @@ FILES=$( ls -1 ${PROXYLOG}/*access.log.xz | awk '{x=split($0,a,"/"); print a[x]}
 
 for FILE in ${FILES}; do
     TEMP=$(echo ${FILE} | sed 's/\.xz$//')
+    simple_message_to_bus combinehttplogs.logmerge.proxy.start run_id="$RUN_ID" log="$PROXYLOG" file="$FILE" target="$TARGET" temp="$TEMP"
     perl ${LOGMERGE} ${PROXYLOG}/${FILE} > ${TARGET}/${TEMP}
+    simple_message_to_bus combinehttplogs.logmerge.proxy.finish run_id="$RUN_ID" log="$PROXYLOG" file="$FILE" target="$TARGET" temp="$TEMP" result="$?"
 done
 
 ##
@@ -88,7 +93,9 @@ FILES=$( ls -1 ${DL_LOG}/dl*access.log.xz | awk '{x=split($0,a,"/"); print a[x]}
 
 for FILE in ${FILES}; do
     TEMP=$(echo ${FILE} | sed 's/\.xz$//')
+    simple_message_to_bus combinehttplogs.logmerge.download.start run_id="$RUN_ID" proxylog="$DL_LOG" file="$FILE" target="$TARGET" temp="$TEMP"
     perl ${LOGMERGE} ${DL_LOG}/${FILE} > ${TARGET}/${TEMP}
+    simple_message_to_bus combinehttplogs.logmerge.download.finish run_id="$RUN_ID" proxylog="$DL_LOG" file="$FILE" target="$TARGET" temp="$TEMP"  result="$?"
 done
 
 ##
@@ -99,7 +106,9 @@ FILES=$( ls -1 ${PEOPLE}/fedora*access.log.xz | awk '{x=split($0,a,"/"); print a
 
 for FILE in ${FILES}; do
     TEMP=$(echo ${FILE} | sed 's/\.xz$//')
+    simple_message_to_bus combinehttplogs.logmerge.people.start run_id="$RUN_ID" proxylog="$PEOPLE" file="$FILE" target="$TARGET" temp="$TEMP"
     perl ${LOGMERGE} ${PEOPLE}/${FILE} > ${TARGET}/${TEMP}
+    simple_message_to_bus combinehttplogs.logmerge.people.finish run_id="$RUN_ID" proxylog="$PEOPLE" file="$FILE" target="$TARGET" temp="$TEMP" result="$?"
 done
 
 # Now we link up the files into latest directory
@@ -116,3 +125,4 @@ if [[ "$UPDATE_LATEST" && -d ${NFSDIR}/latest ]]; then
     done
     popd &> /dev/null
 fi
+simple_message_to_bus combinehttplogs.finish run_id="$RUN_ID"

roles/fedora-messaging-utils/files/simple_message_to_bus

@@ -0,0 +1,30 @@
+#!/bin/sh
+MSGTOPIC_PREFIX="${MSGTOPIC_PREFIX:-logging.stats}"
+topic="${MSGTOPIC_PREFIX}.$1"
+shift
+sent_at="$(TZ=UTC date -Iseconds)"
+id="$(uuidgen -r)"
+body="{"
+sep=""
+
+for body_piece; do
+    key_type="${body_piece%%=*}"
+    key="${key_type%%:*}"
+    type="${key_type#${key}}"
+    type="${type#:}"
+    value="${body_piece#*=}"
+
+    if [ "$type" != "int" ]; then
+        # quote strings
+        value="${value//\\/\\\\}"
+        value="${value//\"/\\\"}"
+        value="\"${value}\""
+    fi
+    body="${body}${sep}\"${key}\": ${value}"
+    sep=", "
+done
+body="${body}}"
+
+fedora-messaging publish - << EOF >/dev/null
+{"body": ${body}, "headers": {"fedora_messaging_schema": "base.message", "fedora_messaging_severity": 20, "sent-at": "${sent_at}"}, "id": "${id}", "queue": "queue", "topic": "${topic}"}
+EOF

roles/fedora-messaging-utils/tasks/main.yml

@@ -0,0 +1,2 @@
+- name: install messaging script for our improvized tracing
+  copy: src=simple_message_to_bus dest=/usr/local/bin/ mode=0755

roles/web-data-analysis/files/condense-mirrorlogs.sh

@@ -25,6 +25,8 @@
 # we are 5 days behind.
 
 # We have dropped this down to 3 days on 2019-10-01
+RUN_ID="$(uuidgen -r)"
+simple_message_to_bus condense-mirrorlogs.start run_id="$RUN_ID"
 
 let NUMDAYS=3
 let OLDDAYS=$(( $NUMDAYS+1 ))
@@ -71,7 +73,9 @@ if [[ ! -f ${LOGFILE} ]]; then
 elif [[ -f ${WORKFILE} ]]; then 
     echo "The workfile for ${YEAR}/${MONTH}/${DAY} already existed."
 else
+    simple_message_to_bus condense-mirrorlogs.mirrorlist.start run_id="$RUN_ID" workfile="$WORKFILE" logfile="$LOGFILE"
     ${LBIN}/mirrorlist.py -o ${WORKFILE} ${LOGFILE}; 
+    simple_message_to_bus condense-mirrorlogs.mirrorlist.finish run_id="$RUN_ID" workfile="$WORKFILE" logfile="$LOGFILE" result="$?"
 fi
 
 # So the data isn't strictly across month boundries due to the end of
@@ -106,3 +110,4 @@ gnuplot ${LSHARE}/${PROJECT}-data.gp
 
 # cleanup the temp data
 rm -rf ${TEMPDIR}
+simple_message_to_bus condense-mirrorlogs.finish run_id="$RUN_ID"

roles/web-data-analysis/files/countme-update.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+RUN_ID="$(uuidgen -r)"
+simple_message_to_bus countme-update.start run_id="$RUN_ID"
+
 # Where do we keep our local/internal data?
 LOCAL_DATA_DIR=/var/lib/countme
 RAW_DB=$LOCAL_DATA_DIR/raw.db
@@ -31,7 +34,15 @@ die() { echo "${0##*/}: error: $*" >&2; exit 2; }
 # _run [COMMAND...]: Run a command, honoring $VERBOSE and $DRYRUN
 _run() {
     if [ "$VERBOSE" -o "$DRYRUN" ]; then echo "$@"; fi
-    if [ "$DRYRUN" ]; then return 0; else "$@"; fi
+    if [ "$DRYRUN" ]; then 
+      return 0
+    else
+      simple_message_to_bus countme-update.command.start run_id="$RUN_ID" command="$@"
+      "$@"
+      RESULT=$?
+      simple_message_to_bus countme-update.command.finish run_id="$RUN_ID" command="$@" result="$?"
+      return $RESULT
+    fi
 }
 
 # CLI help text
@@ -108,3 +119,6 @@ _run $_GIT diff --quiet || _run $_GIT commit -a -m "$(date -u +%F) update"
 # Copy new data into place
 _run atomic_copy $TOTALS_DB $PUBLIC_TOTALS_DB
 _run atomic_copy $TOTALS_CSV $PUBLIC_TOTALS_CSV
+
+simple_message_to_bus countme-update.finish run_id="$RUN_ID"
+

roles/web-data-analysis/tasks/main.yml

@@ -140,6 +140,12 @@
   tags:
     - web-data
 
+- name: install messaging script for our improvized tracing
+  copy: src=simple_message_to_bus dest=/usr/local/bin/ mode=0755
+  tags:
+    - web-data
+    - cron
+
 - name: install countme script to parse new logs & update totals
   copy: src=countme-update.sh dest=/usr/local/bin/ mode=0755
   tags: