Add more message-based tracing to log01 scripts
This commit is contained in:
parent
805599baeb
commit
db936062b3
7 changed files with 69 additions and 1 deletions
|
@ -23,6 +23,7 @@
|
||||||
- sudo
|
- sudo
|
||||||
- awstats
|
- awstats
|
||||||
- web-data-analysis
|
- web-data-analysis
|
||||||
|
- fedora-messaging-utils
|
||||||
- role: keytab/service
|
- role: keytab/service
|
||||||
owner_user: apache
|
owner_user: apache
|
||||||
owner_group: apache
|
owner_group: apache
|
||||||
|
|
|
@ -19,6 +19,9 @@
|
||||||
# along with Fedora Project Infrastructure Ansible Repository. If
|
# along with Fedora Project Infrastructure Ansible Repository. If
|
||||||
# not, see <http://www.gnu.org/licenses/>.
|
# not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
RUN_ID="$(uuidgen -r)"
|
||||||
|
simple_message_to_bus combinehttplogs.start run_id="$RUN_ID"
|
||||||
|
|
||||||
# Some constants / standard paths
|
# Some constants / standard paths
|
||||||
LOGDIR=/var/log/hosts
|
LOGDIR=/var/log/hosts
|
||||||
NFSDIR=/mnt/fedora_stats/combined-http
|
NFSDIR=/mnt/fedora_stats/combined-http
|
||||||
|
@ -79,7 +82,9 @@ FILES=$( ls -1 ${PROXYLOG}/*access.log.xz | awk '{x=split($0,a,"/"); print a[x]}
|
||||||
|
|
||||||
for FILE in ${FILES}; do
|
for FILE in ${FILES}; do
|
||||||
TEMP=$(echo ${FILE} | sed 's/\.xz$//')
|
TEMP=$(echo ${FILE} | sed 's/\.xz$//')
|
||||||
|
simple_message_to_bus combinehttplogs.logmerge.proxy.start run_id="$RUN_ID" log="$PROXYLOG" file="$FILE" target="$TARGET" temp="$TEMP"
|
||||||
perl ${LOGMERGE} ${PROXYLOG}/${FILE} > ${TARGET}/${TEMP}
|
perl ${LOGMERGE} ${PROXYLOG}/${FILE} > ${TARGET}/${TEMP}
|
||||||
|
simple_message_to_bus combinehttplogs.logmerge.proxy.finish run_id="$RUN_ID" log="$PROXYLOG" file="$FILE" target="$TARGET" temp="$TEMP" result="$?"
|
||||||
done
|
done
|
||||||
|
|
||||||
##
|
##
|
||||||
|
@ -88,7 +93,9 @@ FILES=$( ls -1 ${DL_LOG}/dl*access.log.xz | awk '{x=split($0,a,"/"); print a[x]}
|
||||||
|
|
||||||
for FILE in ${FILES}; do
|
for FILE in ${FILES}; do
|
||||||
TEMP=$(echo ${FILE} | sed 's/\.xz$//')
|
TEMP=$(echo ${FILE} | sed 's/\.xz$//')
|
||||||
|
simple_message_to_bus combinehttplogs.logmerge.download.start run_id="$RUN_ID" proxylog="$DL_LOG" file="$FILE" target="$TARGET" temp="$TEMP"
|
||||||
perl ${LOGMERGE} ${DL_LOG}/${FILE} > ${TARGET}/${TEMP}
|
perl ${LOGMERGE} ${DL_LOG}/${FILE} > ${TARGET}/${TEMP}
|
||||||
|
simple_message_to_bus combinehttplogs.logmerge.download.finish run_id="$RUN_ID" proxylog="$DL_LOG" file="$FILE" target="$TARGET" temp="$TEMP" result="$?"
|
||||||
done
|
done
|
||||||
|
|
||||||
##
|
##
|
||||||
|
@ -99,7 +106,9 @@ FILES=$( ls -1 ${PEOPLE}/fedora*access.log.xz | awk '{x=split($0,a,"/"); print a
|
||||||
|
|
||||||
for FILE in ${FILES}; do
|
for FILE in ${FILES}; do
|
||||||
TEMP=$(echo ${FILE} | sed 's/\.xz$//')
|
TEMP=$(echo ${FILE} | sed 's/\.xz$//')
|
||||||
|
simple_message_to_bus combinehttplogs.logmerge.people.start run_id="$RUN_ID" proxylog="$PEOPLE" file="$FILE" target="$TARGET" temp="$TEMP"
|
||||||
perl ${LOGMERGE} ${PEOPLE}/${FILE} > ${TARGET}/${TEMP}
|
perl ${LOGMERGE} ${PEOPLE}/${FILE} > ${TARGET}/${TEMP}
|
||||||
|
simple_message_to_bus combinehttplogs.logmerge.people.finish run_id="$RUN_ID" proxylog="$PEOPLE" file="$FILE" target="$TARGET" temp="$TEMP" result="$?"
|
||||||
done
|
done
|
||||||
|
|
||||||
# Now we link up the files into latest directory
|
# Now we link up the files into latest directory
|
||||||
|
@ -116,3 +125,4 @@ if [[ "$UPDATE_LATEST" && -d ${NFSDIR}/latest ]]; then
|
||||||
done
|
done
|
||||||
popd &> /dev/null
|
popd &> /dev/null
|
||||||
fi
|
fi
|
||||||
|
simple_message_to_bus combinehttplogs.finish run_id="$RUN_ID"
|
||||||
|
|
30
roles/fedora-messaging-utils/files/simple_message_to_bus
Normal file
30
roles/fedora-messaging-utils/files/simple_message_to_bus
Normal file
|
@ -0,0 +1,30 @@
|
||||||
|
#!/bin/sh
|
||||||
|
MSGTOPIC_PREFIX="${MSGTOPIC_PREFIX:-logging.stats}"
|
||||||
|
topic="${MSGTOPIC_PREFIX}.$1"
|
||||||
|
shift
|
||||||
|
sent_at="$(TZ=UTC date -Iseconds)"
|
||||||
|
id="$(uuidgen -r)"
|
||||||
|
body="{"
|
||||||
|
sep=""
|
||||||
|
|
||||||
|
for body_piece; do
|
||||||
|
key_type="${body_piece%%=*}"
|
||||||
|
key="${key_type%%:*}"
|
||||||
|
type="${key_type#${key}}"
|
||||||
|
type="${type#:}"
|
||||||
|
value="${body_piece#*=}"
|
||||||
|
|
||||||
|
if [ "$type" != "int" ]; then
|
||||||
|
# quote strings
|
||||||
|
value="${value//\\/\\\\}"
|
||||||
|
value="${value//\"/\\\"}"
|
||||||
|
value="\"${value}\""
|
||||||
|
fi
|
||||||
|
body="${body}${sep}\"${key}\": ${value}"
|
||||||
|
sep=", "
|
||||||
|
done
|
||||||
|
body="${body}}"
|
||||||
|
|
||||||
|
fedora-messaging publish - << EOF >/dev/null
|
||||||
|
{"body": ${body}, "headers": {"fedora_messaging_schema": "base.message", "fedora_messaging_severity": 20, "sent-at": "${sent_at}"}, "id": "${id}", "queue": "queue", "topic": "${topic}"}
|
||||||
|
EOF
|
2
roles/fedora-messaging-utils/tasks/main.yml
Normal file
2
roles/fedora-messaging-utils/tasks/main.yml
Normal file
|
@ -0,0 +1,2 @@
|
||||||
|
- name: install messaging script for our improvized tracing
|
||||||
|
copy: src=simple_message_to_bus dest=/usr/local/bin/ mode=0755
|
|
@ -25,6 +25,8 @@
|
||||||
# we are 5 days behind.
|
# we are 5 days behind.
|
||||||
|
|
||||||
# We have dropped this down to 3 days on 2019-10-01
|
# We have dropped this down to 3 days on 2019-10-01
|
||||||
|
RUN_ID="$(uuidgen -r)"
|
||||||
|
simple_message_to_bus condense-mirrorlogs.start run_id="$RUN_ID"
|
||||||
|
|
||||||
let NUMDAYS=3
|
let NUMDAYS=3
|
||||||
let OLDDAYS=$(( $NUMDAYS+1 ))
|
let OLDDAYS=$(( $NUMDAYS+1 ))
|
||||||
|
@ -71,7 +73,9 @@ if [[ ! -f ${LOGFILE} ]]; then
|
||||||
elif [[ -f ${WORKFILE} ]]; then
|
elif [[ -f ${WORKFILE} ]]; then
|
||||||
echo "The workfile for ${YEAR}/${MONTH}/${DAY} already existed."
|
echo "The workfile for ${YEAR}/${MONTH}/${DAY} already existed."
|
||||||
else
|
else
|
||||||
|
simple_message_to_bus condense-mirrorlogs.mirrorlist.start run_id="$RUN_ID" workfile="$WORKFILE" logfile="$LOGFILE"
|
||||||
${LBIN}/mirrorlist.py -o ${WORKFILE} ${LOGFILE};
|
${LBIN}/mirrorlist.py -o ${WORKFILE} ${LOGFILE};
|
||||||
|
simple_message_to_bus condense-mirrorlogs.mirrorlist.finish run_id="$RUN_ID" workfile="$WORKFILE" logfile="$LOGFILE" result="$?"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# So the data isn't strictly across month boundries due to the end of
|
# So the data isn't strictly across month boundries due to the end of
|
||||||
|
@ -106,3 +110,4 @@ gnuplot ${LSHARE}/${PROJECT}-data.gp
|
||||||
|
|
||||||
# cleanup the temp data
|
# cleanup the temp data
|
||||||
rm -rf ${TEMPDIR}
|
rm -rf ${TEMPDIR}
|
||||||
|
simple_message_to_bus condense-mirrorlogs.finish run_id="$RUN_ID"
|
||||||
|
|
|
@ -1,5 +1,8 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
|
RUN_ID="$(uuidgen -r)"
|
||||||
|
simple_message_to_bus countme-update.start run_id="$RUN_ID"
|
||||||
|
|
||||||
# Where do we keep our local/internal data?
|
# Where do we keep our local/internal data?
|
||||||
LOCAL_DATA_DIR=/var/lib/countme
|
LOCAL_DATA_DIR=/var/lib/countme
|
||||||
RAW_DB=$LOCAL_DATA_DIR/raw.db
|
RAW_DB=$LOCAL_DATA_DIR/raw.db
|
||||||
|
@ -31,7 +34,15 @@ die() { echo "${0##*/}: error: $*" >&2; exit 2; }
|
||||||
# _run [COMMAND...]: Run a command, honoring $VERBOSE and $DRYRUN
|
# _run [COMMAND...]: Run a command, honoring $VERBOSE and $DRYRUN
|
||||||
_run() {
|
_run() {
|
||||||
if [ "$VERBOSE" -o "$DRYRUN" ]; then echo "$@"; fi
|
if [ "$VERBOSE" -o "$DRYRUN" ]; then echo "$@"; fi
|
||||||
if [ "$DRYRUN" ]; then return 0; else "$@"; fi
|
if [ "$DRYRUN" ]; then
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
simple_message_to_bus countme-update.command.start run_id="$RUN_ID" command="$@"
|
||||||
|
"$@"
|
||||||
|
RESULT=$?
|
||||||
|
simple_message_to_bus countme-update.command.finish run_id="$RUN_ID" command="$@" result="$?"
|
||||||
|
return $RESULT
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# CLI help text
|
# CLI help text
|
||||||
|
@ -108,3 +119,6 @@ _run $_GIT diff --quiet || _run $_GIT commit -a -m "$(date -u +%F) update"
|
||||||
# Copy new data into place
|
# Copy new data into place
|
||||||
_run atomic_copy $TOTALS_DB $PUBLIC_TOTALS_DB
|
_run atomic_copy $TOTALS_DB $PUBLIC_TOTALS_DB
|
||||||
_run atomic_copy $TOTALS_CSV $PUBLIC_TOTALS_CSV
|
_run atomic_copy $TOTALS_CSV $PUBLIC_TOTALS_CSV
|
||||||
|
|
||||||
|
simple_message_to_bus countme-update.finish run_id="$RUN_ID"
|
||||||
|
|
||||||
|
|
|
@ -140,6 +140,12 @@
|
||||||
tags:
|
tags:
|
||||||
- web-data
|
- web-data
|
||||||
|
|
||||||
|
- name: install messaging script for our improvized tracing
|
||||||
|
copy: src=simple_message_to_bus dest=/usr/local/bin/ mode=0755
|
||||||
|
tags:
|
||||||
|
- web-data
|
||||||
|
- cron
|
||||||
|
|
||||||
- name: install countme script to parse new logs & update totals
|
- name: install countme script to parse new logs & update totals
|
||||||
copy: src=countme-update.sh dest=/usr/local/bin/ mode=0755
|
copy: src=countme-update.sh dest=/usr/local/bin/ mode=0755
|
||||||
tags:
|
tags:
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue