add the scripts for data-analysis

This commit is contained in:
Stephen Smoogen 2016-05-04 17:00:23 +00:00
parent e55c8f37c0
commit 4f8471d08e
6 changed files with 1399 additions and 0 deletions

View file

@ -0,0 +1 @@
0 06 * * * root /usr/local/bin/condense-mirrorlogs.sh

View file

@ -0,0 +1,60 @@
#!/bin/bash
# This file is part of Fedora Project Infrastructure Ansible
# Repository.
#
# Fedora Project Infrastructure Ansible Repository is free software:
# you can redistribute it and/or modify it under the terms of the GNU
# General Public License as published by the Free Software Foundation,
# either version 3 of the License, or (at your option) any later
# version.
#
# Fedora Project Infrastructure Ansible Repository is distributed in
# the hope that it will be useful, but WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License
# along with Fedora Project Infrastructure Ansible Repository. If
# not, see <http://www.gnu.org/licenses/>.
# There is a multiday delay involved in processing the logs. It
# may take up to 4 days to get the logs to the main-server. It may
# take a day to combine all the logs onto combined-httpd. So we assume
# we are 5 days behind.
NUMDAYS=5
YEAR=$(/bin/date -d "-${NUMDAYS} days" +%Y)
MONTH=$(/bin/date -d "-${NUMDAYS} days" +%m)
DAY=$(/bin/date -d "-${NUMDAYS} days" +%d)
NFSDIR=/mnt/fedora_stats/combined-http
TARGET=${NFSDIR}/latest
LOGFILE=${TARGET}/mirrors.fedoraproject.org-access.log
WORKDIR=/mnt/fedora_stats/data/
WORKFILE=${WORKDIR}/${YEAR}/${MONTH}/out-${DAY}
WEBDIR=/var/www/html/csv-reports/
mkdir -p ${WORKDIR}/${YEAR}/${MONTH}
if [[ ! -f ${LOGFILE} ]]; then
echo "No logfile found for ${YEAR}/${MONTH}/${DAY}. Please fix."
elif [[ -f ${WORKFILE} ]]; then
echo "The workfile for ${YEAR}/${MONTH}/${DAY} already existed."
else
/usr/local/bin/mirrorlist.py -o ${WORKFILE} ${LOGFILE};
fi
# So the data isn't strictly across month boundries due to the end of the logfiles
# being at 04:00 versus 23:59. Also log files might get stuck and you end up with days
# or weeks of data in a single file. Doing a continual sort clears up that.
sort -o ${WORKDIR}/${YEAR}/out-${MONTH} -S 4G -u -m ${WORKDIR}/${YEAR}/${MONTH}/out-*
sort -o ${WORKDIR}/out-${YEAR} -S 4G -u -m ${WORKDIR}/${YEAR}/out-*
awk -f /usr/local/share/web-data-analysis/mirror-data.awk ${WORKDIR}/out-${YEAR} > ${WEBDIR}/mirrordata-${YEAR}.csv
sort -o ${WEBDIR}/mirrordata-all.csv -u ${WEBDIR}/mirrordata-*.csv
gnuplot /usr/local/share/web-data-analysis/mirror-data.gp

View file

@ -0,0 +1,196 @@
BEGIN{
olddate="1970-01-01"
epel=0;
fedora=0;
epel4=0;
epel5=0;
epel6=0;
epel7=0;
f03=0;
f04=0;
f05=0;
f06=0;
f07=0;
f08=0;
f09=0;
f10=0;
f11=0;
f12=0;
f13=0;
f14=0;
f15=0;
f16=0;
f17=0;
f18=0;
f19=0;
f20=0;
f21=0;
f22=0;
f23=0;
f24=0;
f25=0;
f26=0;
f27=0;
f28=0;
f29=0;
f30=0;
rawhide=0;
unknown_release = 0;
# arch
alpha=0;
arm64=0;
arm=0;
ia64=0;
mips=0;
ppc=0;
s390=0;
sparc=0;
tilegx=0;
x86_32=0;
x86_64=0;
# sub arch
ppc_e=0;
ppc_f=0;
x86_32_e=0;
x86_32_f=0;
x86_64_e = 0;
x86_64_f = 0;
unknown_arch = 0;
centos = 0;
rhel = 0;
print olddate ",02-epel4,03-epel5,04-epel6,05-epel7,06-f03,07-f04,08-f05,09-f06,10-f07,11-f08,12-f09,13-f10,14-f11,15-f12,16-f13,17-f14,18-f15,19-f16,20-f17,21-f18,22-f19,23-f20,24-f21,25-f22,26-f23,27-f24,28-rawhide,29-unk_rel,30-epel,31-fedora,32-alpha,33-arm,34-arm64,35-ia64,36-mips,37-ppc,38-s390,39-sparc,40-tilegx,41-x86_32,42-x86_64,43-x86_32_e,44-x86_32_f,45-x86_64_e,46-x86_64_f,47-ppc_e,48-ppc_f,49-unk_arc,50-centos,51-rhel";
}
{
if ($1 == olddate) {
if ($3 ~"epel4") { epel4=epel4+1; epel=epel+1}
else if ($3 ~"epel5") { epel5=epel5+1; epel=epel+1}
else if ($3 ~"epel6") { epel6=epel6+1; epel=epel+1}
else if ($3 ~"epel7") { epel7=epel7+1; epel=epel+1}
else if ($3 ~"f03") { f03=f03+1; fedora=fedora+1}
else if ($3 ~"f04") { f04=f04+1; fedora=fedora+1}
else if ($3 ~"f05") { f05=f05+1; fedora=fedora+1}
else if ($3 ~"f06") { f06=f06+1; fedora=fedora+1}
else if ($3 ~"f07") { f07=f07+1; fedora=fedora+1}
else if ($3 ~"f08") { f08=f08+1; fedora=fedora+1}
else if ($3 ~"f09") { f09=f09+1; fedora=fedora+1}
else if ($3 ~"f10") { f10=f10+1; fedora=fedora+1}
else if ($3 ~"f11") { f11=f11+1; fedora=fedora+1}
else if ($3 ~"f12") { f12=f12+1; fedora=fedora+1}
else if ($3 ~"f13") { f13=f13+1; fedora=fedora+1}
else if ($3 ~"f14") { f14=f14+1; fedora=fedora+1}
else if ($3 ~"f15") { f15=f15+1; fedora=fedora+1}
else if ($3 ~"f16") { f16=f16+1; fedora=fedora+1}
else if ($3 ~"f17") { f17=f17+1; fedora=fedora+1}
else if ($3 ~"f18") { f18=f18+1; fedora=fedora+1}
else if ($3 ~"f19") { f19=f19+1; fedora=fedora+1}
else if ($3 ~"f20") { f20=f20+1; fedora=fedora+1}
else if ($3 ~"f21") { f21=f21+1; fedora=fedora+1}
else if ($3 ~"f22") { f22=f22+1; fedora=fedora+1}
else if ($3 ~"f23") { f23=f23+1; fedora=fedora+1}
else if ($3 ~"f24") { f24=f24+1; fedora=fedora+1}
else if ($3 ~"f25") { f25=f25+1; fedora=fedora+1}
else if ($3 ~"f26") { f26=f26+1; fedora=fedora+1}
else if ($3 ~"f27") { f27=f27+1; fedora=fedora+1}
else if ($3 ~"f28") { f28=f28+1; fedora=fedora+1}
else if ($3 ~"f29") { f29=f29+1; fedora=fedora+1}
else if ($3 ~"f30") { f30=f30+1; fedora=fedora+1}
else if ($3 ~"rawhide") { rawhide=rawhide+1; fedora=fedora+1}
else if ($3 ~"centos") { centos=centos+1; epel=epel+1}
else if ($3 ~"rhel") { rhel=rhel+1; epel=epel+1}
else { unknown_release = unknown_release + 1 ; };
## ARCH
if ($4 ~ "arm") { arm = arm + 1}
else if ($4 ~ "aarch64") { arm64 = arm64 +1 }
else if ($4 ~"ia64") { ia64 = ia64 + 1}
else if ($4 ~"mips") { mips = mips + 1}
else if ($4 ~"s390") { s390 = s390 +1 }
else if ($4 ~"sparc") { sparc = sparc +1 }
else if ($4 ~"tilegx") { tilegx = tilegx +1 }
else if (($4 ~"i386") && ($3 ~"epel")) { x86_32 = x86_32 + 1; x86_32_e = x86_32_e + 1}
else if (($4 ~"i386") && ($3 ~/rhel/)) { x86_32 = x86_32 + 1; x86_32_e = x86_32_e + 1}
else if (($4 ~"i386") && ($3 ~/centos/)) { x86_32 = x86_32 + 1; x86_32_e = x86_32_e + 1}
else if (($4 ~"i386") && ($3 ~/f[0-2]/)) { x86_32 = x86_32 + 1; x86_32_f = x86_32_f + 1}
else if (($4 ~"i386") && ($3 ~"rawhide")) { x86_32 = x86_32 + 1; x86_32_f = x86_32_f + 1}
else if (($4 ~"x86_64") && ($3 ~"epel")) { x86_64 = x86_64 + 1; x86_64_e = x86_64_e + 1; }
else if (($4 ~"x86_64") && ($3 ~/rhel/)) { x86_64 = x86_64 + 1; x86_64_e = x86_64_e + 1; }
else if (($4 ~"x86_64") && ($3 ~/centos/)) { x86_64 = x86_64 + 1; x86_64_e = x86_64_e + 1; }
else if (($4 ~"x86_64") && ($3 ~/f[0-2]/)) { x86_64 = x86_64 + 1; x86_64_f = x86_64_f + 1; }
else if (($4 ~"x86_64") && ($3 ~"rawhide")) { x86_64 = x86_64 + 1; x86_64_f = x86_64_f + 1; }
else if (($4 ~"ppc") && ($3 ~"epel")) { ppc = ppc + 1 ; ppc_e = ppc_e + 1;}
else if (($4 ~"ppc") && ($3 ~/f[0-2]/)) { ppc = ppc + 1 ; ppc_f = ppc_f + 1;}
else if (($4 ~"ppc") && ($3 ~"rawhide")) { ppc = ppc + 1 ; ppc_f = ppc_f + 1;}
else if ($4 ~/i386/) {x86_32 = x86_32 +1 ; }
else if ($4 ~/x86_64/) {x86_64 = x86_64 +1; }
else if ($4 ~/ppc/) {ppc = ppc +1}
else if ($4 ~"mips") { mips = mips +1 }
else if ($4 ~"alpha") { alpha = alpha +1 }
else {unknown_arch = unknown_arch +1; };
} else {
print olddate "," epel4 "," epel5 "," epel6 "," epel7 "," f03 "," f04 "," f05 "," f06 "," f07 "," f08 "," f09 "," f10 "," f11 "," f12 "," f13 "," f14 "," f15 "," f16 "," f17 "," f18 "," f19 "," f20 "," f21 "," f22 "," f23 "," f24 "," rawhide "," unknown_release "," epel "," fedora "," alpha "," arm "," arm64 "," ia64 "," mips "," ppc "," s390 "," sparc "," tilegx "," x86_32 "," x86_64 "," x86_32_e "," x86_32_f "," x86_64_e "," x86_64_f "," ppc_e "," ppc_f "," unknown_arch "," centos "," rhel;
olddate=$1
epel=0;
fedora=0;
epel4=0;
epel5=0;
epel6=0;
epel7=0;
f03=0;
f04=0;
f05=0;
f06=0;
f07=0;
f08=0;
f09=0;
f10=0;
f11=0;
f12=0;
f13=0;
f14=0;
f15=0;
f16=0;
f17=0;
f18=0;
f19=0;
f20=0;
f21=0;
f22=0;
f23=0;
f24=0;
f25=0;
f26=0;
f27=0;
f28=0;
f29=0;
f30=0;
rawhide=0;
unknown_release = 0;
# arch
arm=0;
arm64=0;
ia64=0;
mips=0;
ppc=0;
s390=0;
sparc=0;
tilegx=0;
x86_32=0;
x86_64=0;
# sub arch
ppc_e=0;
ppc_f=0;
x86_32_e=0;
x86_32_f=0;
x86_64_e = 0;
x86_64_f = 0;
unknown_arch = 0;
centos = 0;
rhel = 0;
}
}
END {
print olddate "," epel4 "," epel5 "," epel6 "," epel7 "," f03 "," f04 "," f05 "," f06 "," f07 "," f08 "," f09 "," f10 "," f11 "," f12 "," f13 "," f14 "," f15 "," f16 "," f17 "," f18 "," f19 "," f20 "," f21 "," f22 "," f23 "," f24 "," rawhide "," unknown_release "," epel "," fedora "," alpha "," arm "," arm64 "," ia64 "," mips "," ppc "," s390 "," sparc "," tilegx "," x86_32 "," x86_64 "," x86_32_e "," x86_32_f "," x86_64_e "," x86_64_f "," ppc_e "," ppc_f "," unknown_arch "," centos, "," rhel;
}

View file

@ -0,0 +1,505 @@
set grid
set xdata time
set format x "%Y-%m-%d"
set timefmt "%Y-%m-%d"
set datafile separator ","
set term png size 1600,1200
##
set output "/var/www/html/csv-reports/fedora-all.png"
set title "Fedora+Epel Yum Unique IPs"
plot ["2007-05-17":"2016-01-20"] \
'data-all.csv' using 1:2 title 'epel4' with lines lw 3,\
'data-all.csv' using 1:3 title 'epel5' with lines lw 3,\
'data-all.csv' using 1:4 title 'epel6' with lines lw 3,\
'data-all.csv' using 1:5 title 'epel7' with lines lw 3,\
'data-all.csv' using 1:6 title 'fed03' with lines lw 3,\
'data-all.csv' using 1:7 title 'fed04' with lines lw 3,\
'data-all.csv' using 1:8 title 'fed05' with lines lw 3,\
'data-all.csv' using 1:9 title 'fed06' with lines lw 3,\
'data-all.csv' using 1:10 title 'fed07' with lines lw 3,\
'data-all.csv' using 1:11 title 'fed08' with lines lw 3,\
'data-all.csv' using 1:12 title 'fed09' with lines lw 3,\
'data-all.csv' using 1:13 title 'fed10' with lines lw 3,\
'data-all.csv' using 1:14 title 'fed11' with lines lw 3,\
'data-all.csv' using 1:15 title 'fed12' with lines lw 3,\
'data-all.csv' using 1:16 title 'fed13' with lines lw 3,\
'data-all.csv' using 1:17 title 'fed14' with lines lw 3,\
'data-all.csv' using 1:18 title 'fed15' with lines lw 3,\
'data-all.csv' using 1:19 title 'fed16' with lines lw 3,\
'data-all.csv' using 1:20 title 'fed17' with lines lw 3,\
'data-all.csv' using 1:21 title 'fed18' with lines lw 3,\
'data-all.csv' using 1:22 title 'fed19' with lines lw 3,\
'data-all.csv' using 1:23 title 'fed20' with lines lw 3,\
'data-all.csv' using 1:24 title 'fed21' with lines lw 3,\
'data-all.csv' using 1:25 title 'fed22' with lines lw 3,\
'data-all.csv' using 1:26 title 'fed23' with lines lw 3,\
'data-all.csv' using 1:27 title 'fed24' with lines lw 3,\
'data-all.csv' using 1:28 title 'rawhide' with lines lw 3,\
'data-all.csv' using 1:29 title 'unk_rel' with lines lw 3,\
'data-all.csv' using 1:30 title 'EPEL' with lines lw 3,\
'data-all.csv' using 1:50 title 'centos' with lines lw 3,\
'data-all.csv' using 1:51 title 'rhel' with lines lw 3,\
'data-all.csv' using 1:31 title 'Fedora' with lines lw 3
unset output
set output "/var/www/html/csv-reports/fedora-and-epel.png"
set title "Fedora Epel Totals Unique IPs"
plot ["2007-05-17":"2016-01-20"] \
'data-all.csv' using 1:30 title 'EPEL' with lines lw 3,\
'data-all.csv' using 1:31 title 'Fedora' with lines lw 3
unset output
set output "/var/www/html/csv-reports/fedora-daily.png"
set title "Fedora Daily Totals Unique IPs"
plot ["2007-05-17":"2016-01-20"] \
'data-all.csv' using 1:31 title 'Fedora' with lines lw 3
unset output
set output "/var/www/html/csv-reports/fedora-os-all.png"
set title "Fedora OS Yum Unique IPs"
plot ["2007-05-17":"2016-01-20"] \
'data-all.csv' using 1:6 title 'fed03' with lines lw 3,\
'data-all.csv' using 1:7 title 'fed04' with lines lw 3,\
'data-all.csv' using 1:8 title 'fed05' with lines lw 3,\
'data-all.csv' using 1:9 title 'fed06' with lines lw 3,\
'data-all.csv' using 1:10 title 'fed07' with lines lw 3,\
'data-all.csv' using 1:11 title 'fed08' with lines lw 3,\
'data-all.csv' using 1:12 title 'fed09' with lines lw 3,\
'data-all.csv' using 1:13 title 'fed10' with lines lw 3,\
'data-all.csv' using 1:14 title 'fed11' with lines lw 3,\
'data-all.csv' using 1:15 title 'fed12' with lines lw 3,\
'data-all.csv' using 1:16 title 'fed13' with lines lw 3,\
'data-all.csv' using 1:17 title 'fed14' with lines lw 3,\
'data-all.csv' using 1:18 title 'fed15' with lines lw 3,\
'data-all.csv' using 1:19 title 'fed16' with lines lw 3,\
'data-all.csv' using 1:20 title 'fed17' with lines lw 3,\
'data-all.csv' using 1:21 title 'fed18' with lines lw 3,\
'data-all.csv' using 1:22 title 'fed19' with lines lw 3,\
'data-all.csv' using 1:23 title 'fed20' with lines lw 3,\
'data-all.csv' using 1:24 title 'fed21' with lines lw 3,\
'data-all.csv' using 1:25 title 'fed22' with lines lw 3,\
'data-all.csv' using 1:26 title 'fed23' with lines lw 3,\
'data-all.csv' using 1:27 title 'fed24' with lines lw 3,\
'data-all.csv' using 1:28 title 'rawhide' with lines lw 3,\
'data-all.csv' using 1:29 title 'unk_rel' with lines lw 3
unset output
set output "/var/www/html/csv-reports/fedora-os-latest.png"
set title "Fedora Selected Versions Unique IPs"
plot ["2013-01-01":"2016-01-20"] \
'data-all.csv' using 1:31 title 'Fedora' with lines lw 3,\
'data-all.csv' using 1:21 title 'fed18' with lines lw 3,\
'data-all.csv' using 1:22 title 'fed19' with lines lw 3,\
'data-all.csv' using 1:23 title 'fed20' with lines lw 3,\
'data-all.csv' using 1:24 title 'fed21' with lines lw 3,\
'data-all.csv' using 1:25 title 'fed22' with lines lw 3,\
'data-all.csv' using 1:26 title 'fed23' with lines lw 3,\
'data-all.csv' using 1:27 title 'fed24' with lines lw 3,\
'data-all.csv' using 1:28 title 'rawhide' with lines lw 3
unset output
set output "/var/www/html/csv-reports/fedora-os-latest-stacked.png"
set title "Fedora Selected Versions Unique IPs"
plot ["2013-01-01":"2016-01-20"] \
'data-all.csv' using 1:31 title 'Fedora' with lines lw 3,\
'data-all.csv' using 1:($21+$22+$23+$24+$25+$26+$27+$28) title 'fed18' with filledcurves x1,\
'data-all.csv' using 1:($22+$23+$24+$25+$26+$27+$28) title 'fed19' with filledcurves x1,\
'data-all.csv' using 1:($23+$24+$25+$26+$27+$28) title 'fed20' with filledcurves x1,\
'data-all.csv' using 1:($24+$25+$26+$27+$28) title 'fed21' with filledcurves x1,\
'data-all.csv' using 1:($25+$26+$27+$28) title 'fed22' with filledcurves x1,\
'data-all.csv' using 1:($26+$27+$28) title 'fed23' with filledcurves x1,\
'data-all.csv' using 1:($27+$28) title 'fed24' with filledcurves x1,\
'data-all.csv' using 1:($28) title 'rawhide' with filledcurves x1
unset output
set output "/var/www/html/csv-reports/fedora-hardware-full.png"
set title "Fedora Hardware via Unique IPs"
plot ["2007-05-17":"2016-01-20"] \
'data-all.csv' using 1:32 title 'alpha' with lines lw 3,\
'data-all.csv' using 1:33 title 'ARM' with lines lw 3,\
'data-all.csv' using 1:34 title 'ARM64' with lines lw 3,\
'data-all.csv' using 1:35 title 'ia64' with lines lw 3,\
'data-all.csv' using 1:36 title 'mips' with lines lw 3,\
'data-all.csv' using 1:48 title 'ppc' with lines lw 3,\
'data-all.csv' using 1:38 title 's390' with lines lw 3,\
'data-all.csv' using 1:39 title 'sparc' with lines lw 3,\
'data-all.csv' using 1:40 title 'tilegx' with lines lw 3,\
'data-all.csv' using 1:44 title 'x86_32' with lines lw 3,\
'data-all.csv' using 1:46 title 'x86_64' with lines lw 3,\
'data-all.csv' using 1:49 title 'unknown' with lines lw 3
unset output
set output "/var/www/html/csv-reports/fedora-hardware-2nd.png"
set title "Fedora Secondary via Unique IPs"
plot ["2007-05-17":"2016-01-20"] \
'data-all.csv' using 1:32 title 'alpha' with lines lw 3,\
'data-all.csv' using 1:33 title 'ARM' with lines lw 3,\
'data-all.csv' using 1:34 title 'ARM64' with lines lw 3,\
'data-all.csv' using 1:35 title 'ia64' with lines lw 3,\
'data-all.csv' using 1:36 title 'mips' with lines lw 3,\
'data-all.csv' using 1:48 title 'ppc' with lines lw 3,\
'data-all.csv' using 1:38 title 's390' with lines lw 3,\
'data-all.csv' using 1:39 title 'sparc' with lines lw 3,\
'data-all.csv' using 1:40 title 'tilegx' with lines lw 3,\
'data-all.csv' using 1:49 title 'unknown' with lines lw 3
unset output
set output "/var/www/html/csv-reports/fedora-epel-stacked.png"
set title "Fedora Yum Unique IPs"
plot ["2007-05-17":"2016-01-20"] \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28+$29) title 'unknown_rel' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28) title 'rawhide' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27) title 'fed24' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26) title 'fed23' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25) title 'fed22' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24) title 'fed21' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23) title 'fed20' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22) title 'fed19' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21) title 'fed18' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20) title 'fed17' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19) title 'fed16' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18) title 'fed15' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17) title 'fed14' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16) title 'fed13' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15) title 'fed12' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14) title 'fed11' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13) title 'fed10' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12) title 'fed09' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11) title 'fed08' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10) title 'fed07' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9) title 'fed06' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8) title 'fed05' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6+$7) title 'fed04' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5+$6) title 'fed03' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4+$5) title 'epel7' w filledcurves x1, \
'data-all.csv' using 1:($2+$3+$4) title 'epel6' w filledcurves x1, \
'data-all.csv' using 1:($2+$3) title 'epel5' w filledcurves x1, \
'data-all.csv' using 1:($2) title 'epel4' w filledcurves x1
unset output
set output "/var/www/html/csv-reports/fedora-stacked.png"
set title "Fedora Epel Yum Unique IPs"
plot ["2007-05-17":"2016-01-20"] \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28+$29) title 'unknown_rel' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28) title 'rawhide' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27) title 'f24' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26) title 'f23' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25) title 'f22' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24) title 'fed21' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23) title 'fed20' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22) title 'fed19' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21) title 'fed18' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20) title 'fed17' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19) title 'fed16' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18) title 'fed15' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17) title 'fed14' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16) title 'fed13' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15) title 'fed12' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14) title 'fed11' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13) title 'fed10' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12) title 'fed09' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11) title 'fed08' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10) title 'fed07' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9) title 'fed06' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8) title 'fed05' w filledcurves x1, \
'data-all.csv' using 1:($6+$7) title 'fed04' w filledcurves x1, \
'data-all.csv' using 1:($6) title 'fed03' w filledcurves x1
unset output
set output "/var/www/html/csv-reports/fedora-select-stacked.png"
set title "Fedora Epel Yum Unique IPs"
plot ["2007-05-17":"2016-01-20"] \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28+$29) title 'unknown_rel' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28) title 'rawhide' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27) title 'fed18-24' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20) title 'fed12-17' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14) title 'fed08-11' w filledcurves x1, \
'data-all.csv' using 1:($6+$7+$8+$9+$10) title 'fed03-07' w filledcurves x1
unset output
# ##
# set output "/var/www/html/csv-reports/fedora-all-mov_avg.png"
# set title "Fedora+Epel Yum Unique IPs (7 day moving average)"
# plot ["2007-05-17":"2016-01-20"] \
# 'data-mov_avg.csv' using 1:2 title 'epel4' with lines lw 3,\
# 'data-mov_avg.csv' using 1:3 title 'epel5' with lines lw 3,\
# 'data-mov_avg.csv' using 1:4 title 'epel6' with lines lw 3,\
# 'data-mov_avg.csv' using 1:5 title 'epel7' with lines lw 3,\
# 'data-mov_avg.csv' using 1:6 title 'fed03' with lines lw 3,\
# 'data-mov_avg.csv' using 1:7 title 'fed04' with lines lw 3,\
# 'data-mov_avg.csv' using 1:8 title 'fed05' with lines lw 3,\
# 'data-mov_avg.csv' using 1:9 title 'fed06' with lines lw 3,\
# 'data-mov_avg.csv' using 1:10 title 'fed07' with lines lw 3,\
# 'data-mov_avg.csv' using 1:11 title 'fed08' with lines lw 3,\
# 'data-mov_avg.csv' using 1:12 title 'fed09' with lines lw 3,\
# 'data-mov_avg.csv' using 1:13 title 'fed10' with lines lw 3,\
# 'data-mov_avg.csv' using 1:14 title 'fed11' with lines lw 3,\
# 'data-mov_avg.csv' using 1:15 title 'fed12' with lines lw 3,\
# 'data-mov_avg.csv' using 1:16 title 'fed13' with lines lw 3,\
# 'data-mov_avg.csv' using 1:17 title 'fed14' with lines lw 3,\
# 'data-mov_avg.csv' using 1:18 title 'fed15' with lines lw 3,\
# 'data-mov_avg.csv' using 1:19 title 'fed16' with lines lw 3,\
# 'data-mov_avg.csv' using 1:20 title 'fed17' with lines lw 3,\
# 'data-mov_avg.csv' using 1:21 title 'fed18' with lines lw 3,\
# 'data-mov_avg.csv' using 1:22 title 'fed19' with lines lw 3,\
# 'data-mov_avg.csv' using 1:23 title 'fed20' with lines lw 3,\
# 'data-mov_avg.csv' using 1:24 title 'fed21' with lines lw 3,\
# 'data-mov_avg.csv' using 1:25 title 'fed22' with lines lw 3,\
# 'data-mov_avg.csv' using 1:26 title 'fed23' with lines lw 3,\
# 'data-mov_avg.csv' using 1:27 title 'fed24' with lines lw 3,\
# 'data-mov_avg.csv' using 1:28 title 'rawhide' with lines lw 3,\
# 'data-mov_avg.csv' using 1:29 title 'unk_rel' with lines lw 3,\
# 'data-mov_avg.csv' using 1:30 title 'EPEL' with lines lw 3,\
# 'data-mov_avg.csv' using 1:50 title 'centos' with lines lw 3,\
# 'data-mov_avg.csv' using 1:51 title 'rhel' with lines lw 3,\
# 'data-mov_avg.csv' using 1:31 title 'Fedora' with lines lw 3
# unset output
# set output "/var/www/html/csv-reports/fedora-and-epel-mov_avg.png"
# set title "Fedora Epel Totals Unique IPs (7 days moving average)"
# plot ["2007-05-17":"2016-01-20"] \
# 'data-mov_avg.csv' using 1:30 title 'EPEL' with lines lw 3,\
# 'data-mov_avg.csv' using 1:31 title 'Fedora' with lines lw 3
# unset output
# set output "/var/www/html/csv-reports/fedora-daily-mov_avg.png"
# set title "Fedora Daily Totals Unique IPs"
# plot ["2007-05-17":"2016-01-20"] \
# 'data-mov_avg.csv' using 1:31 title 'Fedora' with lines lw 3
# unset output
# set output "/var/www/html/csv-reports/fedora-os-all-mov_avg.png"
# set title "Fedora OS Yum Unique IPs (7 days moving average)"
# plot ["2007-05-17":"2016-01-20"] \
# 'data-mov_avg.csv' using 1:6 title 'fed03' with lines lw 3,\
# 'data-mov_avg.csv' using 1:7 title 'fed04' with lines lw 3,\
# 'data-mov_avg.csv' using 1:8 title 'fed05' with lines lw 3,\
# 'data-mov_avg.csv' using 1:9 title 'fed06' with lines lw 3,\
# 'data-mov_avg.csv' using 1:10 title 'fed07' with lines lw 3,\
# 'data-mov_avg.csv' using 1:11 title 'fed08' with lines lw 3,\
# 'data-mov_avg.csv' using 1:12 title 'fed09' with lines lw 3,\
# 'data-mov_avg.csv' using 1:13 title 'fed10' with lines lw 3,\
# 'data-mov_avg.csv' using 1:14 title 'fed11' with lines lw 3,\
# 'data-mov_avg.csv' using 1:15 title 'fed12' with lines lw 3,\
# 'data-mov_avg.csv' using 1:16 title 'fed13' with lines lw 3,\
# 'data-mov_avg.csv' using 1:17 title 'fed14' with lines lw 3,\
# 'data-mov_avg.csv' using 1:18 title 'fed15' with lines lw 3,\
# 'data-mov_avg.csv' using 1:19 title 'fed16' with lines lw 3,\
# 'data-mov_avg.csv' using 1:20 title 'fed17' with lines lw 3,\
# 'data-mov_avg.csv' using 1:21 title 'fed18' with lines lw 3,\
# 'data-mov_avg.csv' using 1:22 title 'fed19' with lines lw 3,\
# 'data-mov_avg.csv' using 1:23 title 'fed20' with lines lw 3,\
# 'data-mov_avg.csv' using 1:24 title 'fed21' with lines lw 3,\
# 'data-mov_avg.csv' using 1:25 title 'fed22' with lines lw 3,\
# 'data-mov_avg.csv' using 1:26 title 'fed23' with lines lw 3,\
# 'data-mov_avg.csv' using 1:27 title 'fed24' with lines lw 3,\
# 'data-mov_avg.csv' using 1:28 title 'rawhide' with lines lw 3,\
# 'data-mov_avg.csv' using 1:29 title 'unk_rel' with lines lw 3
# unset output
# set output "/var/www/html/csv-reports/fedora-os-latest-mov_avg.png"
# set title "Fedora Selected Versions Unique IPs (7 days moving average)"
# plot ["2014-01-01":"2016-01-20"] \
# 'data-mov_avg.csv' using 1:31 title 'Fedora' with lines lw 3,\
# 'data-mov_avg.csv' using 1:21 title 'fed18' with lines lw 3,\
# 'data-mov_avg.csv' using 1:22 title 'fed19' with lines lw 3,\
# 'data-mov_avg.csv' using 1:23 title 'fed20' with lines lw 3,\
# 'data-mov_avg.csv' using 1:24 title 'fed21' with lines lw 3,\
# 'data-mov_avg.csv' using 1:25 title 'fed22' with lines lw 3,\
# 'data-mov_avg.csv' using 1:26 title 'fed23' with lines lw 3,\
# 'data-mov_avg.csv' using 1:27 title 'fed24' with lines lw 3,\
# 'data-mov_avg.csv' using 1:28 title 'rawhide' with lines lw 3
# unset output
# set output "/var/www/html/csv-reports/fedora-os-latest-stacked-mov_avg.png"
# set title "Fedora Selected Versions Unique IPs (7 days moving average)"
# plot ["2014-01-01":"2016-01-20"] \
# 'data-mov_avg.csv' using 1:31 title 'Fedora' with lines lw 3,\
# 'data-mov_avg.csv' using 1:($21+$22+$23+$24+$25+$26+$27+$28) title 'fed18' with filledcurves x1,\
# 'data-mov_avg.csv' using 1:($22+$23+$24+$25+$26+$27+$28) title 'fed19' with filledcurves x1,\
# 'data-mov_avg.csv' using 1:($23+$24+$25+$26+$27+$28) title 'fed20' with filledcurves x1,\
# 'data-mov_avg.csv' using 1:($24+$25+$26+$27+$28) title 'fed21' with filledcurves x1,\
# 'data-mov_avg.csv' using 1:($25+$26+$27+$28) title 'fed22' with filledcurves x1,\
# 'data-mov_avg.csv' using 1:($26+$27+$28) title 'fed23' with filledcurves x1,\
# 'data-mov_avg.csv' using 1:($27+$28) title 'fed24' with filledcurves x1,\
# 'data-mov_avg.csv' using 1:($28) title 'rawhide' with filledcurves x1
# unset output
# set output "/var/www/html/csv-reports/fedora-hardware-full-mov_avg.png"
# set title "Fedora Hardware via Unique IPs (7 days moving average)"
# plot ["2007-05-17":"2016-01-20"] \
# 'data-mov_avg.csv' using 1:32 title 'alpha' with lines lw 3,\
# 'data-mov_avg.csv' using 1:33 title 'ARM' with lines lw 3,\
# 'data-mov_avg.csv' using 1:34 title 'ARM64' with lines lw 3,\
# 'data-mov_avg.csv' using 1:35 title 'ia64' with lines lw 3,\
# 'data-mov_avg.csv' using 1:36 title 'mips' with lines lw 3,\
# 'data-mov_avg.csv' using 1:48 title 'ppc' with lines lw 3,\
# 'data-mov_avg.csv' using 1:38 title 's390' with lines lw 3,\
# 'data-mov_avg.csv' using 1:39 title 'sparc' with lines lw 3,\
# 'data-mov_avg.csv' using 1:40 title 'tilegx' with lines lw 3,\
# 'data-mov_avg.csv' using 1:44 title 'x86_32' with lines lw 3,\
# 'data-mov_avg.csv' using 1:46 title 'x86_64' with lines lw 3,\
# 'data-mov_avg.csv' using 1:49 title 'unknown' with lines lw 3
# unset output
# set output "/var/www/html/csv-reports/fedora-hardware-2nd-mov_avg.png"
# set title "Fedora Secondary via Unique IPs (7 days moving average)"
# plot ["2007-05-17":"2016-01-20"] \
# 'data-mov_avg.csv' using 1:32 title 'alpha' with lines lw 3,\
# 'data-mov_avg.csv' using 1:33 title 'ARM' with lines lw 3,\
# 'data-mov_avg.csv' using 1:34 title 'ARM64' with lines lw 3,\
# 'data-mov_avg.csv' using 1:35 title 'ia64' with lines lw 3,\
# 'data-mov_avg.csv' using 1:36 title 'mips' with lines lw 3,\
# 'data-mov_avg.csv' using 1:48 title 'ppc' with lines lw 3,\
# 'data-mov_avg.csv' using 1:38 title 's390' with lines lw 3,\
# 'data-mov_avg.csv' using 1:39 title 'sparc' with lines lw 3,\
# 'data-mov_avg.csv' using 1:40 title 'tilegx' with lines lw 3,\
# 'data-mov_avg.csv' using 1:49 title 'unknown' with lines lw 3
# unset output
# set output "/var/www/html/csv-reports/fedora-all-stacked-mov_avg.png"
# set title "Fedora Epel Yum Unique IPs (7 days moving average)"
# plot ["2007-05-17":"2016-01-20"] \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28+$29) title 'unknown_rel' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28) title 'rawhide' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27) title 'fed24' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26) title 'fed23' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25) title 'fed22' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24) title 'fed21' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23) title 'fed20' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22) title 'fed19' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21) title 'fed18' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20) title 'fed17' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19) title 'fed16' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18) title 'fed15' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17) title 'fed14' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16) title 'fed13' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15) title 'fed12' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14) title 'fed11' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13) title 'fed10' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12) title 'fed09' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11) title 'fed08' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10) title 'fed07' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9) title 'fed06' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8) title 'fed05' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7) title 'fed04' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6) title 'fed03' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5) title 'epel7' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4) title 'epel6' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3) title 'epel5' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2) title 'epel4' w filledcurves x1
# unset output
# set output "/var/www/html/csv-reports/fedora-epel-stacked-mov_avg.png"
# set title "Fedora Epel (consolidated) Yum Unique IPs (7 days moving average)"
# plot ["2007-05-17":"2016-01-20"] \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28) title 'unknown_rel' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27) title 'rawhide' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26) title 'fedora' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5) title 'epel' w filledcurves x1
# unset output
# set output "/var/www/html/csv-reports/fedora-stacked-mov_avg.png"
# set title "Fedora Yum Unique IPs (7 day moving avg)"
# plot ["2007-05-17":"2016-01-20"] \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28+$29) title 'unknown_rel' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28) title 'rawhide' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27) title 'f24' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26) title 'f23' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25) title 'f22' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24) title 'fed21' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23) title 'fed20' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22) title 'fed19' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21) title 'fed18' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20) title 'fed17' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19) title 'fed16' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18) title 'fed15' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17) title 'fed14' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16) title 'fed13' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15) title 'fed12' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14) title 'fed11' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13) title 'fed10' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12) title 'fed09' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11) title 'fed08' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10) title 'fed07' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9) title 'fed06' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8) title 'fed05' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7) title 'fed04' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6) title 'fed03' w filledcurves x1
# unset output
# set output "/var/www/html/csv-reports/fedora-select-stacked-mov_avg.png"
# set title "Fedora Epel Yum Unique IPs (7 days moving average)"
# plot ["2007-05-17":"2016-01-20"] \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28+$29) title 'unknown_rel' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28) title 'rawhide' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27) title 'fed18-24' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20) title 'fed12-17' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14) title 'fed08-11' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10) title 'fed03-07' w filledcurves x1
# unset output
# ##
# ## EPEL
# ##
# set output "/var/www/html/csv-reports/epel-all.png"
# set title "Epel Yum Unique IPs"
# plot ["2007-05-17":"2016-01-20"] \
# 'data-all.csv' using 1:2 title 'epel4' with lines lw 3,\
# 'data-all.csv' using 1:3 title 'epel5' with lines lw 3,\
# 'data-all.csv' using 1:4 title 'epel6' with lines lw 3,\
# 'data-all.csv' using 1:5 title 'epel7' with lines lw 3,\
# 'data-all.csv' using 1:30 title 'EPEL' with lines lw 3
# unset output
# set output "/var/www/html/csv-reports/epel-all-mov_avg.png"
# set title "Epel Yum Unique IPs (7 days moving average)"
# plot ["2007-05-17":"2016-01-20"] \
# 'data-mov_avg.csv' using 1:2 title 'epel4' with lines lw 3,\
# 'data-mov_avg.csv' using 1:3 title 'epel5' with lines lw 3,\
# 'data-mov_avg.csv' using 1:4 title 'epel6' with lines lw 3,\
# 'data-mov_avg.csv' using 1:5 title 'epel7' with lines lw 3,\
# 'data-mov_avg.csv' using 1:30 title 'EPEL' with lines lw 3
# unset output
# set output "/var/www/html/csv-reports/epel-arch.png"
# set title "Epel Yum Unique IPs"
# plot ["2007-05-17":"2016-01-20"] \
# 'data-all.csv' using 1:43 title 'x86_32' with lines lw 3,\
# 'data-all.csv' using 1:45 title 'x86_64' with lines lw 3,\
# 'data-all.csv' using 1:47 title 'ppc' with lines lw 3,\
# 'data-all.csv' using 1:30 title 'EPEL' with lines lw 3
# unset output
# set output "/var/www/html/csv-reports/epel-arch-mov_avg.png"
# set title "Epel Yum Unique IPs"
# plot ["2007-05-17":"2016-01-20"] \
# 'data-mov_avg.csv' using 1:43 title 'x86_32' with lines lw 3,\
# 'data-mov_avg.csv' using 1:45 title 'x86_64' with lines lw 3,\
# 'data-mov_avg.csv' using 1:47 title 'ppc' with lines lw 3,\
# 'data-mov_avg.csv' using 1:30 title 'EPEL' with lines lw 3
# unset output
# set output "/var/www/html/csv-reports/epel-stacked.png"
# set title "Epel Yum Unique IPs"
# plot ["2007-05-17":"2016-01-20"] \
# 'data-all.csv' using 1:($2+$3+$4+$5) title 'epel7' w filledcurves x1, \
# 'data-all.csv' using 1:($2+$3+$4) title 'epel6' w filledcurves x1, \
# 'data-all.csv' using 1:($2+$3) title 'epel5' w filledcurves x1, \
# 'data-all.csv' using 1:($2) title 'epel4' w filledcurves x1
# unset output
# set output "/var/www/html/csv-reports/epel-stacked-mov_avg.png"
# set title "Epel Yum Unique IPs"
# plot ["2007-05-17":"2016-01-20"] \
# 'data-mov_avg.csv' using 1:($2+$3+$4+$5) title 'epel7' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3+$4) title 'epel6' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2+$3) title 'epel5' w filledcurves x1, \
# 'data-mov_avg.csv' using 1:($2) title 'epel4' w filledcurves x1
# unset output

View file

@ -0,0 +1,70 @@
# Moving Average
import pandas
import matplotlib.pyplot as plt
import math
rolling = 7
tree = {}
df = pandas.read_csv("data-all.csv")
dates = df['1970-01-01']
epel4 = pandas.rolling_mean(df['02-epel4'],rolling)
epel5 = pandas.rolling_mean(df['03-epel5'],rolling)
epel6 = pandas.rolling_mean(df['04-epel6'],rolling)
epel7 = pandas.rolling_mean(df['05-epel7'],rolling)
fed03 = pandas.rolling_mean(df['06-f03'], rolling)
fed04 = pandas.rolling_mean(df['07-f04'], rolling)
fed05 = pandas.rolling_mean(df['08-f05'], rolling)
fed06 = pandas.rolling_mean(df['09-f06'], rolling)
fed07 = pandas.rolling_mean(df['10-f07'], rolling)
fed08 = pandas.rolling_mean(df['11-f08'], rolling)
fed09 = pandas.rolling_mean(df['12-f09'], rolling)
fed10 = pandas.rolling_mean(df['13-f10'], rolling)
fed11 = pandas.rolling_mean(df['14-f11'], rolling)
fed12 = pandas.rolling_mean(df['15-f12'], rolling)
fed13 = pandas.rolling_mean(df['16-f13'], rolling)
fed14 = pandas.rolling_mean(df['17-f14'], rolling)
fed15 = pandas.rolling_mean(df['18-f15'], rolling)
fed16 = pandas.rolling_mean(df['19-f16'], rolling)
fed17 = pandas.rolling_mean(df['20-f17'], rolling)
fed18 = pandas.rolling_mean(df['21-f18'], rolling)
fed19 = pandas.rolling_mean(df['22-f19'], rolling)
fed20 = pandas.rolling_mean(df['23-f20'], rolling)
fed21 = pandas.rolling_mean(df['24-f21'], rolling)
fed22 = pandas.rolling_mean(df['25-f22'], rolling)
fed23 = pandas.rolling_mean(df['26-f23'], rolling)
fed24 = pandas.rolling_mean(df['27-f24'], rolling)
rawhide = pandas.rolling_mean(df['28-rawhide'], rolling)
unk_rel = pandas.rolling_mean(df['29-unk_rel'], rolling)
EPEL = pandas.rolling_mean(df['30-epel'], rolling)
Fedora = pandas.rolling_mean(df['31-fedora'], rolling)
alpha = pandas.rolling_mean(df['32-alpha'], rolling)
ARM = pandas.rolling_mean(df['33-arm'], rolling)
ARM64 = pandas.rolling_mean(df['34-arm64'], rolling)
ia64 = pandas.rolling_mean(df['35-ia64'], rolling)
mips = pandas.rolling_mean(df['36-mips'], rolling)
ppc = pandas.rolling_mean(df['37-ppc'], rolling)
s390 = pandas.rolling_mean(df['38-s390'], rolling)
sparc = pandas.rolling_mean(df['39-sparc'], rolling)
tilegx = pandas.rolling_mean(df['40-tilegx'], rolling)
x86_32 = pandas.rolling_mean(df['41-x86_32'], rolling)
x86_64 = pandas.rolling_mean(df['42-x86_64'], rolling)
x86_32_e = pandas.rolling_mean(df['43-x86_32_e'], rolling)
x86_32_f = pandas.rolling_mean(df['44-x86_32_f'], rolling)
x86_64_e = pandas.rolling_mean(df['45-x86_64_e'], rolling)
x86_64_f = pandas.rolling_mean(df['46-x86_64_f'], rolling)
ppc_e = pandas.rolling_mean(df['47-ppc_e'], rolling)
ppc_f = pandas.rolling_mean(df['48-ppc_f'], rolling)
unk_arc = pandas.rolling_mean(df['49-unk_arc'], rolling)
centos = pandas.rolling_mean(df['50-centos'], rolling)
rhel = pandas.rolling_mean(df['51-rhel'], rolling)
for i in xrange(0,len(dates)):
if math.isnan(epel4[i]):
csv_line = ",".join([dates[i],"0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0"])
else:
csv_line = ",".join([dates[i],str(int(epel4[i])),str(int(epel5[i])),str(int(epel6[i])),str(int(epel7[i])),str(int(fed03[i])),str(int(fed04[i])),str(int(fed05[i])),str(int(fed06[i])),str(int(fed07[i])),str(int(fed08[i])),str(int(fed09[i])),str(int(fed10[i])),str(int(fed11[i])),str(int(fed12[i])),str(int(fed13[i])),str(int(fed14[i])),str(int(fed15[i])),str(int(fed16[i])),str(int(fed17[i])),str(int(fed18[i])),str(int(fed19[i])),str(int(fed20[i])),str(int(fed21[i])),str(int(fed22[i])),str(int(fed23[i])),str(int(fed24[i])),str(int(rawhide[i])),str(int(unk_rel[i])),str(int(EPEL[i])),str(int(Fedora[i])),str(int(alpha[i])),str(int(ARM[i])),str(int(ARM64[i])),str(int(ia64[i])),str(int(mips[i])),str(int(ppc[i])),str(int(s390[i])),str(int(sparc[i])),str(int(tilegx[i])),str(int(x86_32[i])),str(int(x86_64[i])), str(int(x86_32_e[i])),str(int(x86_32_f[i])),str(int(x86_64_e[i])),str(int(x86_64_f[i])),str(int(ppc_e[i])),str(int(ppc_f[i])), str(int(unk_arc[i])),str(int(centos[i])),str(int(rhel[i]))])
print csv_line

View file

@ -0,0 +1,567 @@
#!/usr/bin/python
# This file is part of Fedora Project Infrastructure Ansible
# Repository.
#
# Fedora Project Infrastructure Ansible Repository is free software:
# you can redistribute it and/or modify it under the terms of the GNU
# General Public License as published by the Free Software Foundation,
# either version 3 of the License, or (at your option) any later
# version.
#
# Fedora Project Infrastructure Ansible Repository is distributed in
# the hope that it will be useful, but WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License
# along with Fedora Project Infrastructure Ansible Repository. If
# not, see <http://www.gnu.org/licenses/>.
import sys
import re
import optparse
import os
import string
'''
Mirror list will go through the file given as an argument and parse
out which releases and architectures were looked for, and by how much.
'''
log_line = [
r"(?P<host>[\d\.]+)\s",
r"(?P<identity>\S*)\s",
r"(?P<user>\S*)\s",
r"\[(?P<time>.*?)\]\s",
r'"(?P<request>.*?)"\s',
r"(?P<status>\d+)\s",
r"(?P<bytes>\S*)\s",
r'"(?P<referrer>.*?)"\s',
r'"(?P<user_agent>.*?)"\s*',
]
pattern = re.compile("".join(log_line))
repo_dict = {
"epel4" : "epel4",
"epel4.0" : "epel4",
"epel4.1" : "epel4",
"epel4.2" : "epel4",
"epel4.3" : "epel4",
"epel4.4" : "epel4",
"epel4.5" : "epel4",
"epel4.6" : "epel4",
"epel4.7" : "epel4",
"epel4.8" : "epel4",
"epel4.9" : "epel4",
"epel4.10" : "epel4",
"epel4.11" : "epel4",
"epel4.12" : "epel4",
"epel4.13" : "epel4",
"epel4.14" : "epel4",
"epel4.15" : "epel4",
"epel4.16" : "epel4",
"epel4.17" : "epel4",
"epel4.18" : "epel4",
"epel4.19" : "epel4",
"epel4.20" : "epel4",
"epel5" : "epel5",
"epel5.0" : "epel5",
"epel5.1" : "epel5",
"epel5.2" : "epel5",
"epel5.3" : "epel5",
"epel5.4" : "epel5",
"epel5.5" : "epel5",
"epel5.6" : "epel5",
"epel5.7" : "epel5",
"epel5.8" : "epel5",
"epel5.9" : "epel5",
"epel5.10" : "epel5",
"epel5.11" : "epel5",
"epel5.12" : "epel5",
"epel5.13" : "epel5",
"epel5.14" : "epel5",
"epel5.15" : "epel5",
"epel5.16" : "epel5",
"epel5.17" : "epel5",
"epel5.18" : "epel5",
"epel5.19" : "epel5",
"epel5.20" : "epel5",
"epel6" : "epel6",
"epel6.0" : "epel6",
"epel6.1" : "epel6",
"epel6.2" : "epel6",
"epel6.3" : "epel6",
"epel6.4" : "epel6",
"epel6.5" : "epel6",
"epel6.6" : "epel6",
"epel6.7" : "epel6",
"epel6.8" : "epel6",
"epel6.9" : "epel6",
"epel6.10" : "epel6",
"epel6.11" : "epel6",
"epel6.12" : "epel6",
"epel6.13" : "epel6",
"epel6.14" : "epel6",
"epel6.15" : "epel6",
"epel6.16" : "epel6",
"epel6.17" : "epel6",
"epel6.18" : "epel6",
"epel6.19" : "epel6",
"epel6.20" : "epel6",
"epel7" : "epel7",
"epel7.0" : "epel7",
"epel7.1" : "epel7",
"epel7.2" : "epel7",
"epel7.3" : "epel7",
"epel7.4" : "epel7",
"epel7.5" : "epel7",
"epel7.6" : "epel7",
"epel7.7" : "epel7",
"epel7.8" : "epel7",
"epel7.9" : "epel7",
"epel7.10" : "epel7",
"epel7.11" : "epel7",
"epel7.12" : "epel7",
"epel7.13" : "epel7",
"epel7.14" : "epel7",
"epel7.15" : "epel7",
"epel7.16" : "epel7",
"epel7.17" : "epel7",
"epel7.18" : "epel7",
"epel7.19" : "epel7",
"epel7.20" : "epel7",
"epel8" : "epel8",
"epel8.0" : "epel8",
"epel8.1" : "epel8",
"epel8.2" : "epel8",
"epel8.3" : "epel8",
"epel8.4" : "epel8",
"epel8.5" : "epel8",
"epel8.6" : "epel8",
"epel8.7" : "epel8",
"epel8.8" : "epel8",
"epel8.9" : "epel8",
"epel8.10" : "epel8",
"epel8.11" : "epel8",
"epel8.12" : "epel8",
"epel8.13" : "epel8",
"epel8.14" : "epel8",
"epel8.15" : "epel8",
"epel8.16" : "epel8",
"epel8.17" : "epel8",
"epel8.18" : "epel8",
"epel8.19" : "epel8",
"epel8.20" : "epel8",
"epel9" : "epel9",
"epel9.0" : "epel9",
"epel9.1" : "epel9",
"epel9.2" : "epel9",
"epel9.3" : "epel9",
"epel9.4" : "epel9",
"epel9.5" : "epel9",
"epel9.6" : "epel9",
"epel9.7" : "epel9",
"epel9.8" : "epel9",
"epel9.9" : "epel9",
"epel9.10" : "epel9",
"epel9.11" : "epel9",
"epel9.12" : "epel9",
"epel9.13" : "epel9",
"epel9.14" : "epel9",
"epel9.15" : "epel9",
"epel9.16" : "epel9",
"epel9.17" : "epel9",
"epel9.18" : "epel9",
"epel9.19" : "epel9",
"epel9.20" : "epel9",
"rawhide" : "rawhide",
"frawhide" : "rawhide",
"3" : "f03",
"4" : "f04",
"5" : "f05",
"6" : "f06",
"7" : "f07",
"8" : "f08",
"9" : "f09",
"10" : "f10",
"11" : "f11",
"12" : "f12",
"13" : "f13",
"14" : "f14",
"15" : "f15",
"16" : "f16",
"17" : "f17",
"18" : "f18",
"19" : "f19",
"20" : "f20",
"21" : "f21",
"22" : "f22",
"23" : "f23",
"24" : "f24",
"25" : "f25",
"26" : "f26",
"27" : "f27",
"28" : "f28",
"29" : "f29",
"30" : "f30",
"6.89" : "f07",
"6.90" : "f07",
"6.91" : "f07",
"6.92" : "f07",
"6.93" : "f07",
"7.89" : "f08",
"7.90" : "f08",
"7.91" : "f08",
"7.92" : "f08",
"7.93" : "f08",
"8.90" : "f09",
"8.91" : "f09",
"8.92" : "f09",
"8.93" : "f09",
"9.90" : "f10",
"9.90.1" : "f10",
"9.91" : "f10",
"9.92" : "f10",
"9.93" : "f10",
"10.89" : "f11",
"10.90" : "f11",
"10.91" : "f11",
"10.92" : "f11",
"10.93" : "f11",
"11.89" : "f12",
"11.90" : "f12",
"11.91" : "f12",
"11.92" : "f12",
"11.93" : "f12",
"12.89" : "f13",
"12.90" : "f13",
"12.91" : "f13",
"12.92" : "f13",
"12.93" : "f13",
"f6.89" : "f07",
"f6.90" : "f07",
"f6.91" : "f07",
"f6.92" : "f07",
"f6.93" : "f07",
"f7.89" : "f08",
"f7.90" : "f08",
"f7.91" : "f08",
"f7.92" : "f08",
"f7.93" : "f08",
"f8.90" : "f09",
"f8.91" : "f09",
"f8.92" : "f09",
"f8.93" : "f09",
"f9.90" : "f10",
"f9.90.1" : "f10",
"f9.91" : "f10",
"f9.92" : "f10",
"f9.93" : "f10",
"f10.89" : "f11",
"f10.90" : "f11",
"f10.91" : "f11",
"f10.92" : "f11",
"f10.93" : "f11",
"f11.89" : "f12",
"f11.90" : "f12",
"f11.91" : "f12",
"f11.92" : "f12",
"f11.93" : "f12",
"f12.89" : "f13",
"f12.90" : "f13",
"f12.91" : "f13",
"f12.92" : "f13",
"f12.93" : "f13",
'f3' : 'f03',
'f4' : 'f04',
'f5' : 'f05',
'f6' : 'f06',
'f7' : 'f07',
'f8' : 'f08',
'f9' : 'f09',
'f03' : 'f03',
'f04' : 'f04',
'f05' : 'f05',
'f06' : 'f06',
'f07' : 'f07',
'f08' : 'f08',
'f09' : 'f09',
'f10' : 'f10',
'f11' : 'f11',
'f12' : 'f12',
'f13' : 'f13',
'f14' : 'f14',
'f15' : 'f15',
'f16' : 'f16',
'f17' : 'f17',
'f18' : 'f18',
'f19' : 'f19',
'f20' : 'f20',
'f21' : 'f21',
'f22' : 'f22',
'f23' : 'f23',
'f24' : 'f24',
'f25' : 'f25',
'f26' : 'f26',
'f27' : 'f27',
'f28' : 'f28',
'f29' : 'f29',
'f30' : 'f30',
'rhel4' : 'rhel4',
'rhel5' : 'rhel5',
'rhel6' : 'rhel6',
'rhel7' : 'rhel7',
'rhel8' : 'rhel8',
'rhel9' : 'rhel9',
}
repo_keys = repo_dict.keys()
def breakoutdate(givendate):
Apache_Months = {
'Jan' : '01',
'Feb' : '02',
'Mar' : '03',
'Apr' : '04',
'May' : '05',
'Jun' : '06',
'Jul' : '07',
'Aug' : '08',
'Sep' : '09',
'Oct' : '10',
'Nov' : '11',
'Dec' : '12',
}
date_part = givendate.split()
try:
[day, month, year] = givendate.split(":")[0].split('/')
except:
# string out of index because date corrupted?
[day, month, year ] = ['01', '01', '1970'] # epoch
ret_str = "%s-%s-%s" % (year, Apache_Months[month], day)
return ret_str
def breakoutrepo(request):
try:
parts = request.split()[1].split("?")[1].split("&")
repo=""
arch=""
for i in parts:
if 'repo=' in i:
repo = i.split('=')[1]
if 'arch=' in i:
arch = i.split('=')[1]
return (repo,arch)
except:
return ("unknown_repo","unknown_arch")
def figureoutrepo(asked_repo):
global repo_dict
global repo_keys
crap_chars = ['/', '$', '!', '#', '%', '&', "'", '"', "(", ")", "*", "+", ",", "_", ":", ";", "<", ">", "=", "?", "@", "[", "^", "|"]
spew = asked_repo.lower()
for char in crap_chars:
if char in spew:
spew.split(char)[0]
f_phrases = ["core", "fedora", "extras", "legacy", "fc"]
for word in f_phrases:
if word in spew:
spew = spew.replace(word, "f")
repo_phrases = [".newkey", "install", "alpha", "beta", "client", "debug", "devel", "info", "optional", "preview", "released", "source", "testing", "updates"]
for word in repo_phrases:
if word in spew:
spew = spew.replace(word, "")
if "centosplus" in spew:
spew = spew.replace("centosplus", "centos")
if "client" in spew:
spew = re.sub("client.*", "", spew)
if "cloud" in spew:
spew = re.sub("cloud.*", "", spew)
if "server" in spew:
spew = re.sub("server.*", "", spew)
if "workstation" in spew:
spew = re.sub("workstation.*", "", spew)
if "-" in spew:
spew = re.sub("-+", "", spew)
sanitize = spew.strip()
if sanitize in repo_dict.keys():
return repo_dict[sanitize]
else:
# sys.stderr.write("asked_repo: %s. Thought it was %s\n" % (asked_repo,spew))
return "unknown_repo"
def figureoutarch(asked_arch):
arch_dict = {
'i386' : 'i386',
'i486' : 'i386',
'i586' : 'i386',
'i686' : 'i386',
'athlon' : 'i386',
'pentium' : 'i386',
'pentium3' : 'i386',
'pentium4' : 'i386',
'pentium5' : 'i386',
'ia32' : 'i386',
'x86_32' : 'i386',
'x86_64' : 'x86_64',
'amd64' : 'x86_64',
'aarch64' : 'aarch64',
'alpha' : 'alpha',
'arm' : 'arm',
'arm64' : 'aarch64',
'armhfp' : 'arm',
'armv3l' : 'arm',
'armv5tel' : 'arm',
'armv7hl' : 'arm',
'ia64' : 'ia64',
'mips' : 'mips',
'mips64' : 'mips64',
'mips64el' : 'mips64',
'powepc' : 'ppc',
'ppc' : 'ppc',
'ppc32' : 'ppc',
'ppc64' : 'ppc64',
's390' : 's390',
's390x' : 's390',
'sparc' : 'sparc',
'sparc64' : 'sparc64',
'tilegx' : 'tilegx',
}
spew = asked_arch.split("/")[0]
spew = spew.split("!")[0]
spew = spew.split("#")[0]
spew = spew.split("%")[0]
spew = spew.split("&")[0]
spew = spew.split("'")[0]
spew = spew.split("(")[0]
spew = spew.split("*")[0]
spew = spew.split("+")[0]
spew = spew.split(",")[0]
spew = spew.split("-")[0]
spew = spew.split(".")[0]
spew = spew.split(":")[0]
spew = spew.split(";")[0]
spew = spew.split("<")[0]
spew = spew.split("=")[0]
spew = spew.split(">")[0]
spew = spew.split("?")[0]
spew = spew.split("@")[0]
spew = spew.split("[")[0]
spew = spew.split("]")[0]
spew = spew.split("^")[0]
spew = spew.split('"')[0]
spew = spew.split('\\')[0]
spew = spew.split('|')[0]
spew = spew.split('$')[0]
sanitize = spew.lower()
if sanitize in arch_dict.keys():
return arch_dict[sanitize]
else:
#sys.stderr.write("asked_arch: %s\n" % asked_arch)
return "unknown_arch"
def parseline(our_line):
##
## Figure out if line is something we want to work on more
global pattern
if (('/metalink' in our_line) or ('/mirrorlist' in our_line)):
our_blob = pattern.match(our_line)
if our_blob:
our_dict = our_blob.groupdict()
ip = our_dict['host']
time = breakoutdate(our_dict['time'])
r,a = breakoutrepo(our_dict['request'])
repo = figureoutrepo(r)
arch = figureoutarch(a)
return "%s %s %s %s" % (time,ip,repo,arch)
else:
return ""
else:
return ""
def parselog(our_file, out_file):
our_file = our_file
yumclients_set = set()
output_file = out_file
try:
data = open(our_file, "r")
except:
sys.stderr.write("Unable to open %s\n" % our_file )
sys.exit(-1)
for line in data:
parsed = parseline(line)
if parsed == "":
pass
else:
yumclients_set.add(parsed)
data.close()
our_list = list(yumclients_set)
our_list.sort()
try:
output = open(output_file,"a")
sys.stderr.write("Outputting data: %s\n" % our_file)
except:
sys.stderr.write("Unable to open outputfile\n")
sys.exit(-1)
for line in our_list:
output.write(line + os.linesep)
output.close()
return
def main():
parser = optparse.OptionParser(
description = "A program to parse Fedora mirrorlist apache common log format files.",
prog = "mirrorlist.py",
version = "1.0.2",
usage = "%prog [-o output-filename] logfile1 [logfile2...]"
)
parser.add_option("-o", "--output",
default = "output.txt",
help = "Sets the name of the output file for the run.",
dest = "output")
(options, args) = parser.parse_args()
if options.output:
out_file = options.output
else:
out_file = "output.txt"
for our_file in args:
parselog(our_file,out_file)
if __name__ == '__main__':
main()