we have a temporary fix for the hotspot data
This commit is contained in:
Stephen Smoogen
parent
e1601ca869
commit
568d06cd84
5 changed files with 304 additions and 0 deletions
88
roles/web-data-analysis/files/condense-hotspot.sh
Normal file
88
roles/web-data-analysis/files/condense-hotspot.sh
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
#!/bin/bash
|
||||
|
||||
# This file is part of Fedora Project Infrastructure Ansible
|
||||
# Repository.
|
||||
#
|
||||
# Fedora Project Infrastructure Ansible Repository is free software:
|
||||
# you can redistribute it and/or modify it under the terms of the GNU
|
||||
# General Public License as published by the Free Software Foundation,
|
||||
# either version 3 of the License, or (at your option) any later
|
||||
# version.
|
||||
#
|
||||
# Fedora Project Infrastructure Ansible Repository is distributed in
|
||||
# the hope that it will be useful, but WITHOUT ANY WARRANTY; without
|
||||
# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
|
||||
# PARTICULAR PURPOSE. See the GNU General Public License for more
|
||||
# details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with Fedora Project Infrastructure Ansible Repository. If
|
||||
# not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# There is a multiday delay involved in processing the logs. It
|
||||
# may take up to 4 days to get the logs to the main-server. It may
|
||||
# take a day to combine all the logs onto combined-httpd. So we assume
|
||||
# we are 5 days behind.
|
||||
|
||||
let NUMDAYS=5
|
||||
let OLDDAYS=$(( $NUMDAYS+1 ))
|
||||
|
||||
PROJECT=hotspot
|
||||
WEBLOG=fedoraproject.org
|
||||
|
||||
# This is the year/month/day for a N days ago.
|
||||
YEAR=$(/bin/date -d "-${NUMDAYS} days" +%Y)
|
||||
MONTH=$(/bin/date -d "-${NUMDAYS} days" +%m)
|
||||
DAY=$(/bin/date -d "-${NUMDAYS} days" +%d)
|
||||
|
||||
# And we have have to deal with year/month/day boundaries for our later grep.
|
||||
OLDDATE=$(/bin/date -d "-${OLDDAYS} days" +%Y-%m-%d)
|
||||
OLDYEAR=$(/bin/date -d "-${OLDDAYS} days" +%Y)
|
||||
|
||||
NFSDIR=/mnt/fedora_stats/combined-http
|
||||
TARGET=${NFSDIR}/${YEAR}/${MONTH}/${DAY}
|
||||
|
||||
LOGFILE=${TARGET}/${WEBLOG}-access.log
|
||||
|
||||
WORKDIR=/mnt/fedora_stats/data/${PROJECT}
|
||||
WORKFILE=${WORKDIR}/${YEAR}/${MONTH}/raw-${DAY}
|
||||
|
||||
WEBDIR=/var/www/html/csv-reports/${PROJECT}
|
||||
|
||||
TEMPDIR=$( mktemp -d /tmp/web-data-analysis.XXXXXXXXX )
|
||||
|
||||
LBIN=/usr/local/bin/
|
||||
LSHARE=/usr/local/share/web-data-analysis
|
||||
|
||||
mkdir -p ${WORKDIR}/${YEAR}/${MONTH}
|
||||
if [[ ! -f ${WORKDIR}/${YEAR}/out-${YEAR}-${MONTH} ]]; then
|
||||
touch ${WORKDIR}/${YEAR}/out-${YEAR}-${MONTH}
|
||||
fi
|
||||
|
||||
if [[ ! -f ${WORKDIR}/out-${YEAR} ]]; then
|
||||
touch ${WORKDIR}/out-${YEAR}
|
||||
fi
|
||||
|
||||
if [[ ! -f ${LOGFILE} ]]; then
|
||||
echo "No logfile found for ${YEAR}/${MONTH}/${DAY}. Please fix."
|
||||
else
|
||||
awk -f ${LSHARE}/${PROJECT}.awk ${LOGFILE} > ${WORKFILE}
|
||||
fi
|
||||
|
||||
# So the data isn't strictly across month boundries due to the end of
|
||||
# the logfiles being at 04:00 versus 23:59. Also log files might get
|
||||
# stuck and you end up with days or weeks of data in a single
|
||||
# file. Because the data is pretty small we can get away with adding up the data every day.
|
||||
|
||||
find ${WORKEDIR} -type f | grep raw- | xargs cat | sort -u | awk 'BEGIN{x=0; y=0}; {if (x != $1){ print x,y; x=$1; y=$2} else {y=y+$2}}' > ${WORKEDIR}/worked-all
|
||||
|
||||
|
||||
awk -f ${LSHARE}/${PROJECT}-data.awk ${WORKEDIR}/worked-all | sort -u > ${WEBDIR}/${PROJECT}data-all.csv
|
||||
|
||||
# Make the seven day moving average file
|
||||
/usr/local/bin/hotspot-moving_avg.py > ${WEBDIR}/${PROJECT}data-all-7day-ma.csv
|
||||
|
||||
gnuplot ${LSHARE}/${PROJECT}.gp
|
||||
|
||||
# cleanup the temp data
|
||||
rm -rf ${TEMPDIR}
|
||||
40
roles/web-data-analysis/files/hotspot-data.awk
Normal file
40
roles/web-data-analysis/files/hotspot-data.awk
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
BEGIN{
|
||||
date=strftime("%F",0);
|
||||
count=1;
|
||||
sum=0;
|
||||
most=0;
|
||||
least=0
|
||||
}
|
||||
|
||||
{
|
||||
newdate=strftime("%F",$1); # convert this to a printable date
|
||||
if (date == strftime("%F",0)){ # we hit a min time and need to just print same stuff.
|
||||
print date ",AVG,LEAST,MAX"
|
||||
date=newdate;
|
||||
count=1; # start count to 0. we should have 288 per day but logs are stupid
|
||||
sum=$2; # start the sum
|
||||
most=$2; # what is going to be our most per day
|
||||
least=$2; # what is going to be our least per day
|
||||
} else {
|
||||
if (date != newdate){
|
||||
print date "," int(sum/count) "," least "," most;
|
||||
date=newdate;
|
||||
count=1; # start count to 0. we should have 288 per day but logs are stupid
|
||||
sum=$2; # start the sum
|
||||
most=$2; # what is going to be our most per day
|
||||
least=$2; # what is going to be our least per day
|
||||
} else {
|
||||
count=count+1;
|
||||
sum=sum+$2;
|
||||
if ($2 > most){
|
||||
most=$2;
|
||||
};
|
||||
if ($2 < least) {
|
||||
least=$2;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
END{
|
||||
print date "," int(sum/count) "," least "," most;
|
||||
}
|
||||
56
roles/web-data-analysis/files/hotspot-moving_avg.py
Normal file
56
roles/web-data-analysis/files/hotspot-moving_avg.py
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
# This file is part of Fedora Project Infrastructure Ansible
|
||||
# Repository.
|
||||
#
|
||||
# Fedora Project Infrastructure Ansible Repository is free software:
|
||||
# you can redistribute it and/or modify it under the terms of the GNU
|
||||
# General Public License as published by the Free Software Foundation,
|
||||
# either version 3 of the License, or (at your option) any later
|
||||
# version.
|
||||
#
|
||||
# Fedora Project Infrastructure Ansible Repository is distributed in
|
||||
# the hope that it will be useful, but WITHOUT ANY WARRANTY; without
|
||||
# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
|
||||
# PARTICULAR PURPOSE. See the GNU General Public License for more
|
||||
# details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with Fedora Project Infrastructure Ansible Repository. If
|
||||
# not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# This is a complete horrible hack to get something done. Patches are
|
||||
# really welcome.
|
||||
|
||||
import pandas
|
||||
#import matplotlib.pyplot as plt
|
||||
import math
|
||||
|
||||
# Moving Average
|
||||
|
||||
import pandas
|
||||
import matplotlib.pyplot as plt
|
||||
import math
|
||||
|
||||
rolling = 7
|
||||
|
||||
tree = {}
|
||||
|
||||
df = pandas.read_csv("hotspot-new.csv")
|
||||
|
||||
dates = df['1970-01-01']
|
||||
AVG = pandas.rolling_mean(df['AVG'],rolling)
|
||||
LEAST = pandas.rolling_mean(df['LEAST'],rolling)
|
||||
MAX = pandas.rolling_mean(df['MAX'],rolling)
|
||||
|
||||
|
||||
for i in xrange(0,len(dates)):
|
||||
if math.isnan(MAX[i]):
|
||||
csv_line = ",".join([dates[i],"0","0"])
|
||||
else:
|
||||
csv_line = ",".join([dates[i],
|
||||
str(int(AVG[i])),
|
||||
str(int(LEAST[i])),
|
||||
str(int(MAX[i])),
|
||||
])
|
||||
print csv_line
|
||||
95
roles/web-data-analysis/files/hotspot.awk
Normal file
95
roles/web-data-analysis/files/hotspot.awk
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
#
|
||||
# Take the apache log line
|
||||
# 83.163.161.147 - - [30/Sep/2012:13:54:19 +0000] "GET /static/hotspot.txt HTTP/1.1" 200 3 "-" "dnssec-trigger/0.11"
|
||||
# Convert to
|
||||
# 1349013000 1
|
||||
|
||||
function convertdate(str) {
|
||||
gsub(/\[/, "", str)
|
||||
gsub(/\]/, "", str)
|
||||
split(str,a,":");
|
||||
split(a[1],b,"/");
|
||||
temp="";
|
||||
switch (b[2]) {
|
||||
case "Jan":
|
||||
temp="01"
|
||||
break;
|
||||
case "Feb":
|
||||
temp="02"
|
||||
break;
|
||||
case "Mar":
|
||||
temp="03"
|
||||
break;
|
||||
case "Apr":
|
||||
temp="04"
|
||||
break;
|
||||
case "May":
|
||||
temp="05"
|
||||
break;
|
||||
case "Jun":
|
||||
temp="06"
|
||||
break;
|
||||
case "Jul":
|
||||
temp="07"
|
||||
break;
|
||||
case "Aug":
|
||||
temp="08"
|
||||
break;
|
||||
case "Sep":
|
||||
temp="09"
|
||||
break;
|
||||
case "Oct":
|
||||
temp="10"
|
||||
break;
|
||||
case "Nov":
|
||||
temp="11"
|
||||
break;
|
||||
case "Dec":
|
||||
temp="12"
|
||||
break;
|
||||
default:
|
||||
temp="00"
|
||||
break;
|
||||
}
|
||||
x=b[3]" "temp" "b[1]" "a[2]" "a[3] " "a[4]
|
||||
y=int(mktime(x)/300) # 300 seconds make 5 minutes (I NEED A GLOBAL VAR)
|
||||
return y
|
||||
}
|
||||
|
||||
|
||||
BEGIN{
|
||||
timestamp=0;
|
||||
num_ts = 0;
|
||||
ts_hotspots=0;
|
||||
total_hotsponts=0;
|
||||
}
|
||||
|
||||
#
|
||||
# We assume that every 300 seconds a system will log in at least 1
|
||||
# time because the Networkmanager addon does so.
|
||||
# Convert our date stamp to the nearest 5 minute block and add data to
|
||||
# it. If the log file goes backwards or jumps etc this will mean
|
||||
# multiple outputs for a timestamp. A later process will need to deal
|
||||
# with that. All this will do is output how many it saw at that block
|
||||
# in the log file.
|
||||
#
|
||||
|
||||
$7 ~/hotspot.txt/ && $6 ~/GET/ {
|
||||
date = convertdate($4)
|
||||
if (timestamp != date) {
|
||||
num_ts = num_ts +1;
|
||||
print (timestamp*300),ts_hotspots # GLOBAL VAR GOES HERE
|
||||
timestamp = date;
|
||||
ts_hotspots = 1;
|
||||
} else {
|
||||
ts_hotspots = ts_hotspots +1;
|
||||
total_hotspots = total_hotspots +1;
|
||||
}
|
||||
}
|
||||
|
||||
END {
|
||||
num_ts = num_ts +1;
|
||||
print int(timestamp*300),ts_hotspots # LOOK GLOBAL VAR AGAIN
|
||||
}
|
||||
|
||||
## END OF FILE
|
||||
25
roles/web-data-analysis/files/hotspot.gp
Normal file
25
roles/web-data-analysis/files/hotspot.gp
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
set grid
|
||||
set xdata time
|
||||
set format x "%Y-%m-%d"
|
||||
set timefmt "%Y-%m-%d"
|
||||
|
||||
set datafile separator ","
|
||||
set term png size 1600,1200
|
||||
|
||||
##
|
||||
set output "hotspot-all.png"
|
||||
set title "IPs grabbing hotspot per day"
|
||||
plot ["2014-12-01":"2017-12-31"] \
|
||||
'/var/www/html/csv-reports/hotspot/hotspotdata-all.csv' using 1:2 title 'Average every 5min' with lines lw 4, \
|
||||
'/var/www/html/csv-reports/hotspot/hotspotdata-all.csv' using 1:3 title 'Least 5min' with lines lw 4, \
|
||||
'/var/www/html/csv-reports/hotspot/hotspotdata-all.csv' using 1:4 title 'Max 5min' with lines lw 4
|
||||
unset output
|
||||
|
||||
##
|
||||
set output "hotspot-all-ma.png"
|
||||
set title "Moving Average of IPs grabbing hotspot"
|
||||
plot ["2014-12-01":"2017-12-31"] \
|
||||
'/var/www/html/csv-reports/hotspot/hotspotdatadata-all-7day-ma.csv' using 1:2 title 'Average every 5min' with lines lw 4, \
|
||||
'/var/www/html/csv-reports/hotspot/hotspotdatadata-all-7day-ma.csv' using 1:3 title 'Least 5min' with lines lw 4, \
|
||||
'/var/www/html/csv-reports/hotspot/hotspotdatadata-all-7day-ma.csv' using 1:4 title 'Max 5min' with lines lw 4
|
||||
unset output
|
||||
Loading…
Add table
Add a link
Reference in a new issue