Infrastructure/ansible
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

and we now are much faster on getting the data because I took a walk

Browse source
This commit is contained in:
Stephen Smoogen 2016-05-04 19:12:46 +00:00
parent 5b50279f89
commit 7eb6de4243
1 changed files with 6 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

10
roles/web-data-analysis/files/condense-mirrorlogs.sh
View file

@ -33,7 +33,8 @@ MONTH=$(/bin/date -d "-${NUMDAYS} days" +%m)
DAY=$(/bin/date -d "-${NUMDAYS} days" +%d) DAY=$(/bin/date -d "-${NUMDAYS} days" +%d)
# And we have have to deal with year/month/day boundaries for our later grep. # And we have have to deal with year/month/day boundaries for our later grep.
OLDGREP=$(/bin/date -d "-${OLDDAYS} days" +%Y-%m-%d) OLDDATE=$(/bin/date -d "-${OLDDAYS} days" +%Y-%m-%d)
OLDYEAR=$(/bin/date -d "-${OLDDAYS} days" +%Y)
NFSDIR=/mnt/fedora_stats/combined-http NFSDIR=/mnt/fedora_stats/combined-http
TARGET=${NFSDIR}/latest TARGET=${NFSDIR}/latest
@ -65,8 +66,9 @@ sort -o ${WORKDIR}/${YEAR}/out-${MONTH} -S 4G -u -m ${WORKDIR}/${YEAR}/${MONTH}/
sort -o ${WORKDIR}/out-${YEAR} -S 4G -u -m ${WORKDIR}/${YEAR}/out-* sort -o ${WORKDIR}/out-${YEAR} -S 4G -u -m ${WORKDIR}/${YEAR}/out-*
# Because the logs stop at 04:00 we can only get 24 hours from 6 days before. # Because the logs stop at 04:00 we can only get 24 hours from 6 days before.
egrep "${OLDGREP}" ${WORKDIR}/out-${YEAR} > ${TEMPDIR}/watched-day egrep "${OLDDATE}" ${WORKDIR}/out-${OLDYEAR} > ${TEMPDIR}/watched-day
awk -f /usr/local/share/web-data-analysis/mirror-data.awk ${TEMPDIR}/watched-day >> ${WEBDIR}/mirrordata-${OLDYEAR}.csv
awk -f /usr/local/share/web-data-analysis/mirror-data.awk ${TEMPDIR}/watched-day >> ${WEBDIR}/mirrordata-all.csv
awk -f /usr/local/share/web-data-analysis/mirror-data.awk ${TEMPDIR}/watched-day >> ${WEBDIR}/mirrordata-${YEAR}.csv
sort -o ${WEBDIR}/mirrordata-all.csv -u ${WEBDIR}/mirrordata-*.csv
gnuplot /usr/local/share/web-data-analysis/mirror-data.gp gnuplot /usr/local/share/web-data-analysis/mirror-data.gp