diff --git a/playbooks/groups/logserver.yml b/playbooks/groups/logserver.yml index 76ded5ec10..a1a9f06534 100644 --- a/playbooks/groups/logserver.yml +++ b/playbooks/groups/logserver.yml @@ -23,6 +23,7 @@ - sudo - awstats - web-data-analysis + - fedora-messaging-utils - role: keytab/service owner_user: apache owner_group: apache diff --git a/roles/awstats/files/combineHttpLogs.sh b/roles/awstats/files/combineHttpLogs.sh index fcafc2a0d3..ee03b81e98 100644 --- a/roles/awstats/files/combineHttpLogs.sh +++ b/roles/awstats/files/combineHttpLogs.sh @@ -19,6 +19,9 @@ # along with Fedora Project Infrastructure Ansible Repository. If # not, see . +RUN_ID="$(uuidgen -r)" +simple_message_to_bus combinehttplogs.start run_id="$RUN_ID" + # Some constants / standard paths LOGDIR=/var/log/hosts NFSDIR=/mnt/fedora_stats/combined-http @@ -79,7 +82,9 @@ FILES=$( ls -1 ${PROXYLOG}/*access.log.xz | awk '{x=split($0,a,"/"); print a[x]} for FILE in ${FILES}; do TEMP=$(echo ${FILE} | sed 's/\.xz$//') + simple_message_to_bus combinehttplogs.logmerge.proxy.start run_id="$RUN_ID" log="$PROXYLOG" file="$FILE" target="$TARGET" temp="$TEMP" perl ${LOGMERGE} ${PROXYLOG}/${FILE} > ${TARGET}/${TEMP} + simple_message_to_bus combinehttplogs.logmerge.proxy.finish run_id="$RUN_ID" log="$PROXYLOG" file="$FILE" target="$TARGET" temp="$TEMP" result="$?" done ## @@ -88,7 +93,9 @@ FILES=$( ls -1 ${DL_LOG}/dl*access.log.xz | awk '{x=split($0,a,"/"); print a[x]} for FILE in ${FILES}; do TEMP=$(echo ${FILE} | sed 's/\.xz$//') + simple_message_to_bus combinehttplogs.logmerge.download.start run_id="$RUN_ID" proxylog="$DL_LOG" file="$FILE" target="$TARGET" temp="$TEMP" perl ${LOGMERGE} ${DL_LOG}/${FILE} > ${TARGET}/${TEMP} + simple_message_to_bus combinehttplogs.logmerge.download.finish run_id="$RUN_ID" proxylog="$DL_LOG" file="$FILE" target="$TARGET" temp="$TEMP" result="$?" done ## @@ -99,7 +106,9 @@ FILES=$( ls -1 ${PEOPLE}/fedora*access.log.xz | awk '{x=split($0,a,"/"); print a for FILE in ${FILES}; do TEMP=$(echo ${FILE} | sed 's/\.xz$//') + simple_message_to_bus combinehttplogs.logmerge.people.start run_id="$RUN_ID" proxylog="$PEOPLE" file="$FILE" target="$TARGET" temp="$TEMP" perl ${LOGMERGE} ${PEOPLE}/${FILE} > ${TARGET}/${TEMP} + simple_message_to_bus combinehttplogs.logmerge.people.finish run_id="$RUN_ID" proxylog="$PEOPLE" file="$FILE" target="$TARGET" temp="$TEMP" result="$?" done # Now we link up the files into latest directory @@ -116,3 +125,4 @@ if [[ "$UPDATE_LATEST" && -d ${NFSDIR}/latest ]]; then done popd &> /dev/null fi +simple_message_to_bus combinehttplogs.finish run_id="$RUN_ID" diff --git a/roles/fedora-messaging-utils/files/simple_message_to_bus b/roles/fedora-messaging-utils/files/simple_message_to_bus new file mode 100644 index 0000000000..7091968fc0 --- /dev/null +++ b/roles/fedora-messaging-utils/files/simple_message_to_bus @@ -0,0 +1,30 @@ +#!/bin/sh +MSGTOPIC_PREFIX="${MSGTOPIC_PREFIX:-logging.stats}" +topic="${MSGTOPIC_PREFIX}.$1" +shift +sent_at="$(TZ=UTC date -Iseconds)" +id="$(uuidgen -r)" +body="{" +sep="" + +for body_piece; do + key_type="${body_piece%%=*}" + key="${key_type%%:*}" + type="${key_type#${key}}" + type="${type#:}" + value="${body_piece#*=}" + + if [ "$type" != "int" ]; then + # quote strings + value="${value//\\/\\\\}" + value="${value//\"/\\\"}" + value="\"${value}\"" + fi + body="${body}${sep}\"${key}\": ${value}" + sep=", " +done +body="${body}}" + +fedora-messaging publish - << EOF >/dev/null +{"body": ${body}, "headers": {"fedora_messaging_schema": "base.message", "fedora_messaging_severity": 20, "sent-at": "${sent_at}"}, "id": "${id}", "queue": "queue", "topic": "${topic}"} +EOF diff --git a/roles/fedora-messaging-utils/tasks/main.yml b/roles/fedora-messaging-utils/tasks/main.yml new file mode 100644 index 0000000000..74ff7401b6 --- /dev/null +++ b/roles/fedora-messaging-utils/tasks/main.yml @@ -0,0 +1,2 @@ +- name: install messaging script for our improvized tracing + copy: src=simple_message_to_bus dest=/usr/local/bin/ mode=0755 diff --git a/roles/web-data-analysis/files/condense-mirrorlogs.sh b/roles/web-data-analysis/files/condense-mirrorlogs.sh index 1228ae47cb..ab78fcc469 100644 --- a/roles/web-data-analysis/files/condense-mirrorlogs.sh +++ b/roles/web-data-analysis/files/condense-mirrorlogs.sh @@ -25,6 +25,8 @@ # we are 5 days behind. # We have dropped this down to 3 days on 2019-10-01 +RUN_ID="$(uuidgen -r)" +simple_message_to_bus condense-mirrorlogs.start run_id="$RUN_ID" let NUMDAYS=3 let OLDDAYS=$(( $NUMDAYS+1 )) @@ -71,7 +73,9 @@ if [[ ! -f ${LOGFILE} ]]; then elif [[ -f ${WORKFILE} ]]; then echo "The workfile for ${YEAR}/${MONTH}/${DAY} already existed." else + simple_message_to_bus condense-mirrorlogs.mirrorlist.start run_id="$RUN_ID" workfile="$WORKFILE" logfile="$LOGFILE" ${LBIN}/mirrorlist.py -o ${WORKFILE} ${LOGFILE}; + simple_message_to_bus condense-mirrorlogs.mirrorlist.finish run_id="$RUN_ID" workfile="$WORKFILE" logfile="$LOGFILE" result="$?" fi # So the data isn't strictly across month boundries due to the end of @@ -106,3 +110,4 @@ gnuplot ${LSHARE}/${PROJECT}-data.gp # cleanup the temp data rm -rf ${TEMPDIR} +simple_message_to_bus condense-mirrorlogs.finish run_id="$RUN_ID" diff --git a/roles/web-data-analysis/files/countme-update.sh b/roles/web-data-analysis/files/countme-update.sh index a9d2c083e5..43238ffcc2 100644 --- a/roles/web-data-analysis/files/countme-update.sh +++ b/roles/web-data-analysis/files/countme-update.sh @@ -1,5 +1,8 @@ #!/bin/bash +RUN_ID="$(uuidgen -r)" +simple_message_to_bus countme-update.start run_id="$RUN_ID" + # Where do we keep our local/internal data? LOCAL_DATA_DIR=/var/lib/countme RAW_DB=$LOCAL_DATA_DIR/raw.db @@ -31,7 +34,15 @@ die() { echo "${0##*/}: error: $*" >&2; exit 2; } # _run [COMMAND...]: Run a command, honoring $VERBOSE and $DRYRUN _run() { if [ "$VERBOSE" -o "$DRYRUN" ]; then echo "$@"; fi - if [ "$DRYRUN" ]; then return 0; else "$@"; fi + if [ "$DRYRUN" ]; then + return 0 + else + simple_message_to_bus countme-update.command.start run_id="$RUN_ID" command="$@" + "$@" + RESULT=$? + simple_message_to_bus countme-update.command.finish run_id="$RUN_ID" command="$@" result="$?" + return $RESULT + fi } # CLI help text @@ -108,3 +119,6 @@ _run $_GIT diff --quiet || _run $_GIT commit -a -m "$(date -u +%F) update" # Copy new data into place _run atomic_copy $TOTALS_DB $PUBLIC_TOTALS_DB _run atomic_copy $TOTALS_CSV $PUBLIC_TOTALS_CSV + +simple_message_to_bus countme-update.finish run_id="$RUN_ID" + diff --git a/roles/web-data-analysis/tasks/main.yml b/roles/web-data-analysis/tasks/main.yml index b4b662e409..b7dc036f92 100644 --- a/roles/web-data-analysis/tasks/main.yml +++ b/roles/web-data-analysis/tasks/main.yml @@ -140,6 +140,12 @@ tags: - web-data +- name: install messaging script for our improvized tracing + copy: src=simple_message_to_bus dest=/usr/local/bin/ mode=0755 + tags: + - web-data + - cron + - name: install countme script to parse new logs & update totals copy: src=countme-update.sh dest=/usr/local/bin/ mode=0755 tags: