diff --git a/roles/web-data-analysis/files/countme-centos-update.sh b/roles/web-data-analysis/files/countme-centos-update.sh new file mode 100644 index 0000000000..0c66fa8e18 --- /dev/null +++ b/roles/web-data-analysis/files/countme-centos-update.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +export MSGTOPIC_PREFIX=logging.stats +export MSGBODY_PRESET="loghost=$(hostname) run_id=$(uuidgen -r)" +simple_message_to_bus countme-centos-update.start + +# Where do we keep our local/internal data? +LOCAL_DATA_DIR=/var/lib/countme +RAW_DB=$LOCAL_DATA_DIR/raw-centos.db +TOTALS_DB=$LOCAL_DATA_DIR/totals-centos.db +TOTALS_CSV=$LOCAL_DATA_DIR/totals-centos.csv +LOGFMT="%Y/%m/%d/mirrors.centos.org-access.log" + +# Where do we put the public-facing data? +PUBLIC_DATA_DIR=/var/www/html/csv-reports/countme +PUBLIC_TOTALS_DB=$PUBLIC_DATA_DIR/totals-centos.db +PUBLIC_TOTALS_CSV=$PUBLIC_DATA_DIR/totals-centos.csv + +# Names of the update commands. They are from the python3-mirrors-countme rpm +UPDATE_RAWDB=countme-update-rawdb.sh +UPDATE_TOTALS=countme-update-totals.sh + +# How we're gonna call git - our local repo dir is LOCAL_DATA_DIR +_GIT="git -C $LOCAL_DATA_DIR" + +# Copy with atomic overwrite +atomic_copy() { + local src="$1" dst="$2" + cp -f ${src} ${dst}.part + mv -f ${dst}.part ${dst} +} + +# die [MESSAGE]: prints "$PROG: error: $MESSAGE" on stderr and exits +die() { echo "${0##*/}: error: $*" >&2; exit 2; } + +# _run [COMMAND...]: Run a command, honoring $VERBOSE and $DRYRUN +_run() { + if [ "$VERBOSE" -o "$DRYRUN" ]; then echo "$@"; fi + if [ "$DRYRUN" ]; then + return 0 + else + simple_message_to_bus countme-centos-update.command.start command="$@" + "$@" + RESULT=$? + simple_message_to_bus countme-centos-update.command.finish command="$@" result="$?" + return $RESULT + fi +} + +# CLI help text +HELP_USAGE="usage: countme-centos-updates.sh [OPTION]..." +HELP_OPTIONS=" +Options: + -h, --help Show this message and exit + -v, --verbose Show more info about what's happening + -n, --dryrun Don't run anything, just show commands + -p, --progress Show progress meters while running +" + +# Turn on progress by default if stderr is a tty +if [ -z "$PROGRESS" -a -t 2 ]; then PROGRESS=1; fi + +# Parse CLI options with getopt(1) +_GETOPT_TMP=$(getopt \ + --name countme-centos-update \ + --options hvnp \ + --longoptions help,verbose,dryrun,progress,checkoutdir: \ + -- "$@") +eval set -- "$_GETOPT_TMP" +unset _GETOPT_TMP +while [ $# -gt 0 ]; do + arg=$1; shift + case $arg in + '-h'|'--help') echo "$HELP_USAGE"; echo "$HELP_OPTIONS"; exit 0 ;; + '-v'|'--verbose') VERBOSE=1 ;; + '-n'|'--dryrun') DRYRUN=1 ;; + '-p'|'--progress') PROGRESS=1 ;; + # Hidden option for testing / manual use + '--checkoutdir') COUNTME_CHECKOUT=$1; shift ;; + '--') break ;; + esac +done + +# Tweak path if needed +if [ -d "$COUNTME_CHECKOUT" ]; then + cd $COUNTME_CHECKOUT + PATH="$COUNTME_CHECKOUT:$COUNTME_CHECKOUT/scripts:$PATH" +fi + +# Hardcoding /usr/local/bin here is hacky; should be pulled from pip, but +# parsing pip output is nontrivial, and my father always told me: +# "Son, life's too damn short write a RFC2822 parser in bash." +PATH="$PATH:/usr/local/bin" + +# Check for required commands +command -v $UPDATE_RAWDB >/dev/null || die "can't find '$UPDATE_RAWDB'" +command -v $UPDATE_TOTALS >/dev/null || die "can't find '$UPDATE_TOTALS'" +command -v git >/dev/null || die "can't find 'git'" + +# Apply other CLI options +if [ "$PROGRESS" ]; then + UPDATE_RAWDB="$UPDATE_RAWDB --progress" + UPDATE_TOTALS="$UPDATE_TOTALS --progress" +fi + +# Exit immediately on errors +set -e + +# Run the updates +_run $UPDATE_RAWDB --rawdb $RAW_DB --logfmt $LOGFMT +_run $UPDATE_TOTALS --rawdb $RAW_DB --totals-db $TOTALS_DB --totals-csv $TOTALS_CSV + +# If needed: init local git repo and mark file(s) to be tracked therein +if [ ! -d $LOCAL_DATA_DIR/.git ]; then + _run $_GIT init + _run $_GIT add -N $(realpath $TOTALS_CSV --relative-to $LOCAL_DATA_DIR) +fi +# If any tracked files were updated, commit changes +_run $_GIT diff --quiet || _run $_GIT commit -a -m "$(date -u +%F) update" + +# Copy new data into place +_run atomic_copy $TOTALS_DB $PUBLIC_TOTALS_DB +_run atomic_copy $TOTALS_CSV $PUBLIC_TOTALS_CSV + +simple_message_to_bus countme-centos-update.finish + diff --git a/roles/web-data-analysis/files/countme-update.cron b/roles/web-data-analysis/files/countme-update.cron index 0b25e767a5..0ebf6cf6fa 100644 --- a/roles/web-data-analysis/files/countme-update.cron +++ b/roles/web-data-analysis/files/countme-update.cron @@ -1,2 +1,3 @@ MAILTO=root@fedoraproject.org,mattdm@fedoraproject.org,nils@redhat.com,asaleh@redhat.com 0 09 * * * countme /usr/local/bin/countme-update.sh > /dev/null +0 09 * * * countme /usr/local/bin/countme-centos-update.sh > /dev/null diff --git a/roles/web-data-analysis/tasks/main.yml b/roles/web-data-analysis/tasks/main.yml index 4aca665c9e..f32e1c427d 100644 --- a/roles/web-data-analysis/tasks/main.yml +++ b/roles/web-data-analysis/tasks/main.yml @@ -160,6 +160,11 @@ tags: - web-data +- name: install CentOS countme script to parse new logs & update totals + copy: src=countme-centos-update.sh dest=/usr/local/bin/ mode=0755 + tags: + - web-data + - name: install cron file to run countme-update.sh daily copy: src=countme-update.cron dest=/etc/cron.d/ mode=0644 tags: