Infrastructure/ansible
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

Trim the RAWDB, and split/compress the pushed DB.

Browse source 
Signed-off-by: James Antill <jantill@redhat.com>
This commit is contained in:
James Antill 2024-04-03 13:12:08 -04:00 • committed by james
parent 08937963c4
commit 21acd2412e
2 changed files with 26 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

13
roles/web-data-analysis/files/countme-centos-update.sh
View file

@ -24,6 +24,10 @@ PUBLIC_TOTALS_CSV=$PUBLIC_DATA_DIR/totals-centos.csv
# They should be in PATH somewhere... # They should be in PATH somewhere...
UPDATE_RAWDB=countme-update-rawdb.sh UPDATE_RAWDB=countme-update-rawdb.sh
UPDATE_TOTALS=countme-update-totals.sh UPDATE_TOTALS=countme-update-totals.sh
UPDATE_TRIM=countme-trim-raw
# Names of the post update commands. They should be in PATH somewhere...
PUSH_SPLIT=split-totals-db.sh
PUSH_REZIP=rezip
# ------------------------------ NOTE ------------------------------ # ------------------------------ NOTE ------------------------------
# Everything below this line should try to be identical between the # Everything below this line should try to be identical between the
@ -107,6 +111,7 @@ PATH="$PATH:/usr/local/bin"
command -v $UPDATE_RAWDB >/dev/null || die "can't find '$UPDATE_RAWDB'" command -v $UPDATE_RAWDB >/dev/null || die "can't find '$UPDATE_RAWDB'"
command -v $UPDATE_TOTALS >/dev/null || die "can't find '$UPDATE_TOTALS'" command -v $UPDATE_TOTALS >/dev/null || die "can't find '$UPDATE_TOTALS'"
command -v git >/dev/null || die "can't find 'git'" command -v git >/dev/null || die "can't find 'git'"
command -v $UPDATE_TRIM >/dev/null || die "can't find '$UPDATE_TRIM'"
# Apply other CLI options # Apply other CLI options
if [ "$PROGRESS" ]; then if [ "$PROGRESS" ]; then
@ -121,6 +126,8 @@ set -e
_run $UPDATE_RAWDB --rawdb $RAW_DB --logfmt $LOGFMT _run $UPDATE_RAWDB --rawdb $RAW_DB --logfmt $LOGFMT
_run $UPDATE_TOTALS --rawdb $RAW_DB --totals-db $TOTALS_DB --totals-csv $TOTALS_CSV _run $UPDATE_TOTALS --rawdb $RAW_DB --totals-db $TOTALS_DB --totals-csv $TOTALS_CSV
_run $UPDATE_TRIM --rw $RAW_DB 2
# If needed: init local git repo and mark file(s) to be tracked therein # If needed: init local git repo and mark file(s) to be tracked therein
if [ ! -d $LOCAL_DATA_DIR/.git ]; then if [ ! -d $LOCAL_DATA_DIR/.git ]; then
_run $_GIT init _run $_GIT init
@ -133,5 +140,11 @@ _run $_GIT diff --quiet || _run $_GIT commit -a -m "$(date -u +%F) update"
_run atomic_copy $TOTALS_DB $PUBLIC_TOTALS_DB _run atomic_copy $TOTALS_DB $PUBLIC_TOTALS_DB
_run atomic_copy $TOTALS_CSV $PUBLIC_TOTALS_CSV _run atomic_copy $TOTALS_CSV $PUBLIC_TOTALS_CSV
command -v $PUSH_SPLIT >/dev/null || die "can't find '$PUSH_SPLIT'"
command -v $PUSH_REZIP >/dev/null || die "can't find '$PUSH_REZIP'"
_run $PUSH_SPLIT $PUBLIC_TOTALS_DB
_run $PUSH_REZIP $PUBLIC_DATA_DIR/*.db
simple_message_to_bus $CMD_NAME.finish simple_message_to_bus $CMD_NAME.finish

13
roles/web-data-analysis/files/countme-update.sh
View file

@ -23,6 +23,10 @@ PUBLIC_TOTALS_CSV=$PUBLIC_DATA_DIR/totals.csv
# They should be in PATH somewhere... # They should be in PATH somewhere...
UPDATE_RAWDB=countme-update-rawdb.sh UPDATE_RAWDB=countme-update-rawdb.sh
UPDATE_TOTALS=countme-update-totals.sh UPDATE_TOTALS=countme-update-totals.sh
UPDATE_TRIM=countme-trim-raw
# Names of the post update commands. They should be in PATH somewhere...
PUSH_SPLIT=split-totals-db.sh
PUSH_REZIP=rezip
# ------------------------------ NOTE ------------------------------ # ------------------------------ NOTE ------------------------------
# Everything below this line should try to be identical between the # Everything below this line should try to be identical between the
@ -106,6 +110,7 @@ PATH="$PATH:/usr/local/bin"
command -v $UPDATE_RAWDB >/dev/null || die "can't find '$UPDATE_RAWDB'" command -v $UPDATE_RAWDB >/dev/null || die "can't find '$UPDATE_RAWDB'"
command -v $UPDATE_TOTALS >/dev/null || die "can't find '$UPDATE_TOTALS'" command -v $UPDATE_TOTALS >/dev/null || die "can't find '$UPDATE_TOTALS'"
command -v git >/dev/null || die "can't find 'git'" command -v git >/dev/null || die "can't find 'git'"
command -v $UPDATE_TRIM >/dev/null || die "can't find '$UPDATE_TRIM'"
# Apply other CLI options # Apply other CLI options
if [ "$PROGRESS" ]; then if [ "$PROGRESS" ]; then
@ -120,6 +125,8 @@ set -e
_run $UPDATE_RAWDB --rawdb $RAW_DB _run $UPDATE_RAWDB --rawdb $RAW_DB
_run $UPDATE_TOTALS --rawdb $RAW_DB --totals-db $TOTALS_DB --totals-csv $TOTALS_CSV _run $UPDATE_TOTALS --rawdb $RAW_DB --totals-db $TOTALS_DB --totals-csv $TOTALS_CSV
_run $UPDATE_TRIM --rw $RAW_DB 2
# If needed: init local git repo and mark file(s) to be tracked therein # If needed: init local git repo and mark file(s) to be tracked therein
if [ ! -d $LOCAL_DATA_DIR/.git ]; then if [ ! -d $LOCAL_DATA_DIR/.git ]; then
_run $_GIT init _run $_GIT init
@ -132,5 +139,11 @@ _run $_GIT diff --quiet || _run $_GIT commit -a -m "$(date -u +%F) update"
_run atomic_copy $TOTALS_DB $PUBLIC_TOTALS_DB _run atomic_copy $TOTALS_DB $PUBLIC_TOTALS_DB
_run atomic_copy $TOTALS_CSV $PUBLIC_TOTALS_CSV _run atomic_copy $TOTALS_CSV $PUBLIC_TOTALS_CSV
command -v $PUSH_SPLIT >/dev/null || die "can't find '$PUSH_SPLIT'"
command -v $PUSH_REZIP >/dev/null || die "can't find '$PUSH_REZIP'"
_run $PUSH_SPLIT $PUBLIC_TOTALS_DB
_run $PUSH_REZIP $PUBLIC_DATA_DIR/*.db
simple_message_to_bus $CMD_NAME.finish simple_message_to_bus $CMD_NAME.finish