Add a countme for CentOS Stream 9
This commit is contained in:
Stephen Smoogen
parent
b0f46d9ce7
commit
bfb1320bc9
3 changed files with 132 additions and 0 deletions
126
roles/web-data-analysis/files/countme-centos-update.sh
Normal file
126
roles/web-data-analysis/files/countme-centos-update.sh
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
#!/bin/bash
|
||||
|
||||
export MSGTOPIC_PREFIX=logging.stats
|
||||
export MSGBODY_PRESET="loghost=$(hostname) run_id=$(uuidgen -r)"
|
||||
simple_message_to_bus countme-centos-update.start
|
||||
|
||||
# Where do we keep our local/internal data?
|
||||
LOCAL_DATA_DIR=/var/lib/countme
|
||||
RAW_DB=$LOCAL_DATA_DIR/raw-centos.db
|
||||
TOTALS_DB=$LOCAL_DATA_DIR/totals-centos.db
|
||||
TOTALS_CSV=$LOCAL_DATA_DIR/totals-centos.csv
|
||||
LOGFMT="%Y/%m/%d/mirrors.centos.org-access.log"
|
||||
|
||||
# Where do we put the public-facing data?
|
||||
PUBLIC_DATA_DIR=/var/www/html/csv-reports/countme
|
||||
PUBLIC_TOTALS_DB=$PUBLIC_DATA_DIR/totals-centos.db
|
||||
PUBLIC_TOTALS_CSV=$PUBLIC_DATA_DIR/totals-centos.csv
|
||||
|
||||
# Names of the update commands. They are from the python3-mirrors-countme rpm
|
||||
UPDATE_RAWDB=countme-update-rawdb.sh
|
||||
UPDATE_TOTALS=countme-update-totals.sh
|
||||
|
||||
# How we're gonna call git - our local repo dir is LOCAL_DATA_DIR
|
||||
_GIT="git -C $LOCAL_DATA_DIR"
|
||||
|
||||
# Copy with atomic overwrite
|
||||
atomic_copy() {
|
||||
local src="$1" dst="$2"
|
||||
cp -f ${src} ${dst}.part
|
||||
mv -f ${dst}.part ${dst}
|
||||
}
|
||||
|
||||
# die [MESSAGE]: prints "$PROG: error: $MESSAGE" on stderr and exits
|
||||
die() { echo "${0##*/}: error: $*" >&2; exit 2; }
|
||||
|
||||
# _run [COMMAND...]: Run a command, honoring $VERBOSE and $DRYRUN
|
||||
_run() {
|
||||
if [ "$VERBOSE" -o "$DRYRUN" ]; then echo "$@"; fi
|
||||
if [ "$DRYRUN" ]; then
|
||||
return 0
|
||||
else
|
||||
simple_message_to_bus countme-centos-update.command.start command="$@"
|
||||
"$@"
|
||||
RESULT=$?
|
||||
simple_message_to_bus countme-centos-update.command.finish command="$@" result="$?"
|
||||
return $RESULT
|
||||
fi
|
||||
}
|
||||
|
||||
# CLI help text
|
||||
HELP_USAGE="usage: countme-centos-updates.sh [OPTION]..."
|
||||
HELP_OPTIONS="
|
||||
Options:
|
||||
-h, --help Show this message and exit
|
||||
-v, --verbose Show more info about what's happening
|
||||
-n, --dryrun Don't run anything, just show commands
|
||||
-p, --progress Show progress meters while running
|
||||
"
|
||||
|
||||
# Turn on progress by default if stderr is a tty
|
||||
if [ -z "$PROGRESS" -a -t 2 ]; then PROGRESS=1; fi
|
||||
|
||||
# Parse CLI options with getopt(1)
|
||||
_GETOPT_TMP=$(getopt \
|
||||
--name countme-centos-update \
|
||||
--options hvnp \
|
||||
--longoptions help,verbose,dryrun,progress,checkoutdir: \
|
||||
-- "$@")
|
||||
eval set -- "$_GETOPT_TMP"
|
||||
unset _GETOPT_TMP
|
||||
while [ $# -gt 0 ]; do
|
||||
arg=$1; shift
|
||||
case $arg in
|
||||
'-h'|'--help') echo "$HELP_USAGE"; echo "$HELP_OPTIONS"; exit 0 ;;
|
||||
'-v'|'--verbose') VERBOSE=1 ;;
|
||||
'-n'|'--dryrun') DRYRUN=1 ;;
|
||||
'-p'|'--progress') PROGRESS=1 ;;
|
||||
# Hidden option for testing / manual use
|
||||
'--checkoutdir') COUNTME_CHECKOUT=$1; shift ;;
|
||||
'--') break ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Tweak path if needed
|
||||
if [ -d "$COUNTME_CHECKOUT" ]; then
|
||||
cd $COUNTME_CHECKOUT
|
||||
PATH="$COUNTME_CHECKOUT:$COUNTME_CHECKOUT/scripts:$PATH"
|
||||
fi
|
||||
|
||||
# Hardcoding /usr/local/bin here is hacky; should be pulled from pip, but
|
||||
# parsing pip output is nontrivial, and my father always told me:
|
||||
# "Son, life's too damn short write a RFC2822 parser in bash."
|
||||
PATH="$PATH:/usr/local/bin"
|
||||
|
||||
# Check for required commands
|
||||
command -v $UPDATE_RAWDB >/dev/null || die "can't find '$UPDATE_RAWDB'"
|
||||
command -v $UPDATE_TOTALS >/dev/null || die "can't find '$UPDATE_TOTALS'"
|
||||
command -v git >/dev/null || die "can't find 'git'"
|
||||
|
||||
# Apply other CLI options
|
||||
if [ "$PROGRESS" ]; then
|
||||
UPDATE_RAWDB="$UPDATE_RAWDB --progress"
|
||||
UPDATE_TOTALS="$UPDATE_TOTALS --progress"
|
||||
fi
|
||||
|
||||
# Exit immediately on errors
|
||||
set -e
|
||||
|
||||
# Run the updates
|
||||
_run $UPDATE_RAWDB --rawdb $RAW_DB --logfmt $LOGFMT
|
||||
_run $UPDATE_TOTALS --rawdb $RAW_DB --totals-db $TOTALS_DB --totals-csv $TOTALS_CSV
|
||||
|
||||
# If needed: init local git repo and mark file(s) to be tracked therein
|
||||
if [ ! -d $LOCAL_DATA_DIR/.git ]; then
|
||||
_run $_GIT init
|
||||
_run $_GIT add -N $(realpath $TOTALS_CSV --relative-to $LOCAL_DATA_DIR)
|
||||
fi
|
||||
# If any tracked files were updated, commit changes
|
||||
_run $_GIT diff --quiet || _run $_GIT commit -a -m "$(date -u +%F) update"
|
||||
|
||||
# Copy new data into place
|
||||
_run atomic_copy $TOTALS_DB $PUBLIC_TOTALS_DB
|
||||
_run atomic_copy $TOTALS_CSV $PUBLIC_TOTALS_CSV
|
||||
|
||||
simple_message_to_bus countme-centos-update.finish
|
||||
|
||||
|
|
@ -1,2 +1,3 @@
|
|||
MAILTO=root@fedoraproject.org,mattdm@fedoraproject.org,nils@redhat.com,asaleh@redhat.com
|
||||
0 09 * * * countme /usr/local/bin/countme-update.sh > /dev/null
|
||||
0 09 * * * countme /usr/local/bin/countme-centos-update.sh > /dev/null
|
||||
|
|
|
|||
|
|
@ -160,6 +160,11 @@
|
|||
tags:
|
||||
- web-data
|
||||
|
||||
- name: install CentOS countme script to parse new logs & update totals
|
||||
copy: src=countme-centos-update.sh dest=/usr/local/bin/ mode=0755
|
||||
tags:
|
||||
- web-data
|
||||
|
||||
- name: install cron file to run countme-update.sh daily
|
||||
copy: src=countme-update.cron dest=/etc/cron.d/ mode=0644
|
||||
tags:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue