From 4f8471d08edb1e46e67fce33cf672ec62d78cac3 Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Wed, 4 May 2016 17:00:23 +0000 Subject: [PATCH] add the scripts for data-analysis --- .../files/condense-mirrorlogs.cron | 1 + .../files/condense-mirrorlogs.sh | 60 ++ roles/web-data-analysis/files/mirror-data.awk | 196 ++++++ roles/web-data-analysis/files/mirror-data.gp | 505 ++++++++++++++++ .../files/mirror-moving_avg.py | 70 +++ roles/web-data-analysis/files/mirrorlist.py | 567 ++++++++++++++++++ 6 files changed, 1399 insertions(+) create mode 100644 roles/web-data-analysis/files/condense-mirrorlogs.cron create mode 100644 roles/web-data-analysis/files/condense-mirrorlogs.sh create mode 100644 roles/web-data-analysis/files/mirror-data.awk create mode 100644 roles/web-data-analysis/files/mirror-data.gp create mode 100644 roles/web-data-analysis/files/mirror-moving_avg.py create mode 100644 roles/web-data-analysis/files/mirrorlist.py diff --git a/roles/web-data-analysis/files/condense-mirrorlogs.cron b/roles/web-data-analysis/files/condense-mirrorlogs.cron new file mode 100644 index 0000000000..29f30a8102 --- /dev/null +++ b/roles/web-data-analysis/files/condense-mirrorlogs.cron @@ -0,0 +1 @@ +0 06 * * * root /usr/local/bin/condense-mirrorlogs.sh diff --git a/roles/web-data-analysis/files/condense-mirrorlogs.sh b/roles/web-data-analysis/files/condense-mirrorlogs.sh new file mode 100644 index 0000000000..e488025a5f --- /dev/null +++ b/roles/web-data-analysis/files/condense-mirrorlogs.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# This file is part of Fedora Project Infrastructure Ansible +# Repository. +# +# Fedora Project Infrastructure Ansible Repository is free software: +# you can redistribute it and/or modify it under the terms of the GNU +# General Public License as published by the Free Software Foundation, +# either version 3 of the License, or (at your option) any later +# version. +# +# Fedora Project Infrastructure Ansible Repository is distributed in +# the hope that it will be useful, but WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License +# along with Fedora Project Infrastructure Ansible Repository. If +# not, see . + +# There is a multiday delay involved in processing the logs. It +# may take up to 4 days to get the logs to the main-server. It may +# take a day to combine all the logs onto combined-httpd. So we assume +# we are 5 days behind. + +NUMDAYS=5 +YEAR=$(/bin/date -d "-${NUMDAYS} days" +%Y) +MONTH=$(/bin/date -d "-${NUMDAYS} days" +%m) +DAY=$(/bin/date -d "-${NUMDAYS} days" +%d) + +NFSDIR=/mnt/fedora_stats/combined-http +TARGET=${NFSDIR}/latest + +LOGFILE=${TARGET}/mirrors.fedoraproject.org-access.log + +WORKDIR=/mnt/fedora_stats/data/ +WORKFILE=${WORKDIR}/${YEAR}/${MONTH}/out-${DAY} + +WEBDIR=/var/www/html/csv-reports/ + +mkdir -p ${WORKDIR}/${YEAR}/${MONTH} + +if [[ ! -f ${LOGFILE} ]]; then + echo "No logfile found for ${YEAR}/${MONTH}/${DAY}. Please fix." +elif [[ -f ${WORKFILE} ]]; then + echo "The workfile for ${YEAR}/${MONTH}/${DAY} already existed." +else + /usr/local/bin/mirrorlist.py -o ${WORKFILE} ${LOGFILE}; +fi + +# So the data isn't strictly across month boundries due to the end of the logfiles +# being at 04:00 versus 23:59. Also log files might get stuck and you end up with days +# or weeks of data in a single file. Doing a continual sort clears up that. +sort -o ${WORKDIR}/${YEAR}/out-${MONTH} -S 4G -u -m ${WORKDIR}/${YEAR}/${MONTH}/out-* +sort -o ${WORKDIR}/out-${YEAR} -S 4G -u -m ${WORKDIR}/${YEAR}/out-* + +awk -f /usr/local/share/web-data-analysis/mirror-data.awk ${WORKDIR}/out-${YEAR} > ${WEBDIR}/mirrordata-${YEAR}.csv +sort -o ${WEBDIR}/mirrordata-all.csv -u ${WEBDIR}/mirrordata-*.csv +gnuplot /usr/local/share/web-data-analysis/mirror-data.gp diff --git a/roles/web-data-analysis/files/mirror-data.awk b/roles/web-data-analysis/files/mirror-data.awk new file mode 100644 index 0000000000..7dc65530f7 --- /dev/null +++ b/roles/web-data-analysis/files/mirror-data.awk @@ -0,0 +1,196 @@ +BEGIN{ + olddate="1970-01-01" + epel=0; + fedora=0; + epel4=0; + epel5=0; + epel6=0; + epel7=0; + f03=0; + f04=0; + f05=0; + f06=0; + f07=0; + f08=0; + f09=0; + f10=0; + f11=0; + f12=0; + f13=0; + f14=0; + f15=0; + f16=0; + f17=0; + f18=0; + f19=0; + f20=0; + f21=0; + f22=0; + f23=0; + f24=0; + f25=0; + f26=0; + f27=0; + f28=0; + f29=0; + f30=0; + rawhide=0; + unknown_release = 0; + # arch + alpha=0; + arm64=0; + arm=0; + ia64=0; + mips=0; + ppc=0; + s390=0; + sparc=0; + tilegx=0; + x86_32=0; + x86_64=0; + # sub arch + ppc_e=0; + ppc_f=0; + x86_32_e=0; + x86_32_f=0; + x86_64_e = 0; + x86_64_f = 0; + unknown_arch = 0; + centos = 0; + rhel = 0; + print olddate ",02-epel4,03-epel5,04-epel6,05-epel7,06-f03,07-f04,08-f05,09-f06,10-f07,11-f08,12-f09,13-f10,14-f11,15-f12,16-f13,17-f14,18-f15,19-f16,20-f17,21-f18,22-f19,23-f20,24-f21,25-f22,26-f23,27-f24,28-rawhide,29-unk_rel,30-epel,31-fedora,32-alpha,33-arm,34-arm64,35-ia64,36-mips,37-ppc,38-s390,39-sparc,40-tilegx,41-x86_32,42-x86_64,43-x86_32_e,44-x86_32_f,45-x86_64_e,46-x86_64_f,47-ppc_e,48-ppc_f,49-unk_arc,50-centos,51-rhel"; +} + +{ + if ($1 == olddate) { + if ($3 ~"epel4") { epel4=epel4+1; epel=epel+1} + else if ($3 ~"epel5") { epel5=epel5+1; epel=epel+1} + else if ($3 ~"epel6") { epel6=epel6+1; epel=epel+1} + else if ($3 ~"epel7") { epel7=epel7+1; epel=epel+1} + else if ($3 ~"f03") { f03=f03+1; fedora=fedora+1} + else if ($3 ~"f04") { f04=f04+1; fedora=fedora+1} + else if ($3 ~"f05") { f05=f05+1; fedora=fedora+1} + else if ($3 ~"f06") { f06=f06+1; fedora=fedora+1} + else if ($3 ~"f07") { f07=f07+1; fedora=fedora+1} + else if ($3 ~"f08") { f08=f08+1; fedora=fedora+1} + else if ($3 ~"f09") { f09=f09+1; fedora=fedora+1} + else if ($3 ~"f10") { f10=f10+1; fedora=fedora+1} + else if ($3 ~"f11") { f11=f11+1; fedora=fedora+1} + else if ($3 ~"f12") { f12=f12+1; fedora=fedora+1} + else if ($3 ~"f13") { f13=f13+1; fedora=fedora+1} + else if ($3 ~"f14") { f14=f14+1; fedora=fedora+1} + else if ($3 ~"f15") { f15=f15+1; fedora=fedora+1} + else if ($3 ~"f16") { f16=f16+1; fedora=fedora+1} + else if ($3 ~"f17") { f17=f17+1; fedora=fedora+1} + else if ($3 ~"f18") { f18=f18+1; fedora=fedora+1} + else if ($3 ~"f19") { f19=f19+1; fedora=fedora+1} + else if ($3 ~"f20") { f20=f20+1; fedora=fedora+1} + else if ($3 ~"f21") { f21=f21+1; fedora=fedora+1} + else if ($3 ~"f22") { f22=f22+1; fedora=fedora+1} + else if ($3 ~"f23") { f23=f23+1; fedora=fedora+1} + else if ($3 ~"f24") { f24=f24+1; fedora=fedora+1} + else if ($3 ~"f25") { f25=f25+1; fedora=fedora+1} + else if ($3 ~"f26") { f26=f26+1; fedora=fedora+1} + else if ($3 ~"f27") { f27=f27+1; fedora=fedora+1} + else if ($3 ~"f28") { f28=f28+1; fedora=fedora+1} + else if ($3 ~"f29") { f29=f29+1; fedora=fedora+1} + else if ($3 ~"f30") { f30=f30+1; fedora=fedora+1} + else if ($3 ~"rawhide") { rawhide=rawhide+1; fedora=fedora+1} + else if ($3 ~"centos") { centos=centos+1; epel=epel+1} + else if ($3 ~"rhel") { rhel=rhel+1; epel=epel+1} + else { unknown_release = unknown_release + 1 ; }; + ## ARCH + if ($4 ~ "arm") { arm = arm + 1} + else if ($4 ~ "aarch64") { arm64 = arm64 +1 } + else if ($4 ~"ia64") { ia64 = ia64 + 1} + else if ($4 ~"mips") { mips = mips + 1} + else if ($4 ~"s390") { s390 = s390 +1 } + else if ($4 ~"sparc") { sparc = sparc +1 } + else if ($4 ~"tilegx") { tilegx = tilegx +1 } + else if (($4 ~"i386") && ($3 ~"epel")) { x86_32 = x86_32 + 1; x86_32_e = x86_32_e + 1} + else if (($4 ~"i386") && ($3 ~/rhel/)) { x86_32 = x86_32 + 1; x86_32_e = x86_32_e + 1} + else if (($4 ~"i386") && ($3 ~/centos/)) { x86_32 = x86_32 + 1; x86_32_e = x86_32_e + 1} + else if (($4 ~"i386") && ($3 ~/f[0-2]/)) { x86_32 = x86_32 + 1; x86_32_f = x86_32_f + 1} + else if (($4 ~"i386") && ($3 ~"rawhide")) { x86_32 = x86_32 + 1; x86_32_f = x86_32_f + 1} + else if (($4 ~"x86_64") && ($3 ~"epel")) { x86_64 = x86_64 + 1; x86_64_e = x86_64_e + 1; } + else if (($4 ~"x86_64") && ($3 ~/rhel/)) { x86_64 = x86_64 + 1; x86_64_e = x86_64_e + 1; } + else if (($4 ~"x86_64") && ($3 ~/centos/)) { x86_64 = x86_64 + 1; x86_64_e = x86_64_e + 1; } + else if (($4 ~"x86_64") && ($3 ~/f[0-2]/)) { x86_64 = x86_64 + 1; x86_64_f = x86_64_f + 1; } + else if (($4 ~"x86_64") && ($3 ~"rawhide")) { x86_64 = x86_64 + 1; x86_64_f = x86_64_f + 1; } + else if (($4 ~"ppc") && ($3 ~"epel")) { ppc = ppc + 1 ; ppc_e = ppc_e + 1;} + else if (($4 ~"ppc") && ($3 ~/f[0-2]/)) { ppc = ppc + 1 ; ppc_f = ppc_f + 1;} + else if (($4 ~"ppc") && ($3 ~"rawhide")) { ppc = ppc + 1 ; ppc_f = ppc_f + 1;} + else if ($4 ~/i386/) {x86_32 = x86_32 +1 ; } + else if ($4 ~/x86_64/) {x86_64 = x86_64 +1; } + else if ($4 ~/ppc/) {ppc = ppc +1} + else if ($4 ~"mips") { mips = mips +1 } + else if ($4 ~"alpha") { alpha = alpha +1 } + else {unknown_arch = unknown_arch +1; }; + } else { + print olddate "," epel4 "," epel5 "," epel6 "," epel7 "," f03 "," f04 "," f05 "," f06 "," f07 "," f08 "," f09 "," f10 "," f11 "," f12 "," f13 "," f14 "," f15 "," f16 "," f17 "," f18 "," f19 "," f20 "," f21 "," f22 "," f23 "," f24 "," rawhide "," unknown_release "," epel "," fedora "," alpha "," arm "," arm64 "," ia64 "," mips "," ppc "," s390 "," sparc "," tilegx "," x86_32 "," x86_64 "," x86_32_e "," x86_32_f "," x86_64_e "," x86_64_f "," ppc_e "," ppc_f "," unknown_arch "," centos "," rhel; + olddate=$1 + epel=0; + fedora=0; + epel4=0; + epel5=0; + epel6=0; + epel7=0; + f03=0; + f04=0; + f05=0; + f06=0; + f07=0; + f08=0; + f09=0; + f10=0; + f11=0; + f12=0; + f13=0; + f14=0; + f15=0; + f16=0; + f17=0; + f18=0; + f19=0; + f20=0; + f21=0; + f22=0; + f23=0; + f24=0; + f25=0; + f26=0; + f27=0; + f28=0; + f29=0; + f30=0; + rawhide=0; + unknown_release = 0; + # arch + arm=0; + arm64=0; + ia64=0; + mips=0; + ppc=0; + s390=0; + sparc=0; + tilegx=0; + x86_32=0; + x86_64=0; + # sub arch + ppc_e=0; + ppc_f=0; + x86_32_e=0; + x86_32_f=0; + x86_64_e = 0; + x86_64_f = 0; + unknown_arch = 0; + centos = 0; + rhel = 0; + } + +} + +END { + print olddate "," epel4 "," epel5 "," epel6 "," epel7 "," f03 "," f04 "," f05 "," f06 "," f07 "," f08 "," f09 "," f10 "," f11 "," f12 "," f13 "," f14 "," f15 "," f16 "," f17 "," f18 "," f19 "," f20 "," f21 "," f22 "," f23 "," f24 "," rawhide "," unknown_release "," epel "," fedora "," alpha "," arm "," arm64 "," ia64 "," mips "," ppc "," s390 "," sparc "," tilegx "," x86_32 "," x86_64 "," x86_32_e "," x86_32_f "," x86_64_e "," x86_64_f "," ppc_e "," ppc_f "," unknown_arch "," centos, "," rhel; +} + diff --git a/roles/web-data-analysis/files/mirror-data.gp b/roles/web-data-analysis/files/mirror-data.gp new file mode 100644 index 0000000000..10185358e2 --- /dev/null +++ b/roles/web-data-analysis/files/mirror-data.gp @@ -0,0 +1,505 @@ +set grid +set xdata time +set format x "%Y-%m-%d" +set timefmt "%Y-%m-%d" + +set datafile separator "," +set term png size 1600,1200 + +## +set output "/var/www/html/csv-reports/fedora-all.png" +set title "Fedora+Epel Yum Unique IPs" +plot ["2007-05-17":"2016-01-20"] \ + 'data-all.csv' using 1:2 title 'epel4' with lines lw 3,\ + 'data-all.csv' using 1:3 title 'epel5' with lines lw 3,\ + 'data-all.csv' using 1:4 title 'epel6' with lines lw 3,\ + 'data-all.csv' using 1:5 title 'epel7' with lines lw 3,\ + 'data-all.csv' using 1:6 title 'fed03' with lines lw 3,\ + 'data-all.csv' using 1:7 title 'fed04' with lines lw 3,\ + 'data-all.csv' using 1:8 title 'fed05' with lines lw 3,\ + 'data-all.csv' using 1:9 title 'fed06' with lines lw 3,\ + 'data-all.csv' using 1:10 title 'fed07' with lines lw 3,\ + 'data-all.csv' using 1:11 title 'fed08' with lines lw 3,\ + 'data-all.csv' using 1:12 title 'fed09' with lines lw 3,\ + 'data-all.csv' using 1:13 title 'fed10' with lines lw 3,\ + 'data-all.csv' using 1:14 title 'fed11' with lines lw 3,\ + 'data-all.csv' using 1:15 title 'fed12' with lines lw 3,\ + 'data-all.csv' using 1:16 title 'fed13' with lines lw 3,\ + 'data-all.csv' using 1:17 title 'fed14' with lines lw 3,\ + 'data-all.csv' using 1:18 title 'fed15' with lines lw 3,\ + 'data-all.csv' using 1:19 title 'fed16' with lines lw 3,\ + 'data-all.csv' using 1:20 title 'fed17' with lines lw 3,\ + 'data-all.csv' using 1:21 title 'fed18' with lines lw 3,\ + 'data-all.csv' using 1:22 title 'fed19' with lines lw 3,\ + 'data-all.csv' using 1:23 title 'fed20' with lines lw 3,\ + 'data-all.csv' using 1:24 title 'fed21' with lines lw 3,\ + 'data-all.csv' using 1:25 title 'fed22' with lines lw 3,\ + 'data-all.csv' using 1:26 title 'fed23' with lines lw 3,\ + 'data-all.csv' using 1:27 title 'fed24' with lines lw 3,\ + 'data-all.csv' using 1:28 title 'rawhide' with lines lw 3,\ + 'data-all.csv' using 1:29 title 'unk_rel' with lines lw 3,\ + 'data-all.csv' using 1:30 title 'EPEL' with lines lw 3,\ + 'data-all.csv' using 1:50 title 'centos' with lines lw 3,\ + 'data-all.csv' using 1:51 title 'rhel' with lines lw 3,\ + 'data-all.csv' using 1:31 title 'Fedora' with lines lw 3 +unset output + +set output "/var/www/html/csv-reports/fedora-and-epel.png" +set title "Fedora Epel Totals Unique IPs" +plot ["2007-05-17":"2016-01-20"] \ + 'data-all.csv' using 1:30 title 'EPEL' with lines lw 3,\ + 'data-all.csv' using 1:31 title 'Fedora' with lines lw 3 +unset output + +set output "/var/www/html/csv-reports/fedora-daily.png" +set title "Fedora Daily Totals Unique IPs" +plot ["2007-05-17":"2016-01-20"] \ + 'data-all.csv' using 1:31 title 'Fedora' with lines lw 3 +unset output + +set output "/var/www/html/csv-reports/fedora-os-all.png" +set title "Fedora OS Yum Unique IPs" +plot ["2007-05-17":"2016-01-20"] \ + 'data-all.csv' using 1:6 title 'fed03' with lines lw 3,\ + 'data-all.csv' using 1:7 title 'fed04' with lines lw 3,\ + 'data-all.csv' using 1:8 title 'fed05' with lines lw 3,\ + 'data-all.csv' using 1:9 title 'fed06' with lines lw 3,\ + 'data-all.csv' using 1:10 title 'fed07' with lines lw 3,\ + 'data-all.csv' using 1:11 title 'fed08' with lines lw 3,\ + 'data-all.csv' using 1:12 title 'fed09' with lines lw 3,\ + 'data-all.csv' using 1:13 title 'fed10' with lines lw 3,\ + 'data-all.csv' using 1:14 title 'fed11' with lines lw 3,\ + 'data-all.csv' using 1:15 title 'fed12' with lines lw 3,\ + 'data-all.csv' using 1:16 title 'fed13' with lines lw 3,\ + 'data-all.csv' using 1:17 title 'fed14' with lines lw 3,\ + 'data-all.csv' using 1:18 title 'fed15' with lines lw 3,\ + 'data-all.csv' using 1:19 title 'fed16' with lines lw 3,\ + 'data-all.csv' using 1:20 title 'fed17' with lines lw 3,\ + 'data-all.csv' using 1:21 title 'fed18' with lines lw 3,\ + 'data-all.csv' using 1:22 title 'fed19' with lines lw 3,\ + 'data-all.csv' using 1:23 title 'fed20' with lines lw 3,\ + 'data-all.csv' using 1:24 title 'fed21' with lines lw 3,\ + 'data-all.csv' using 1:25 title 'fed22' with lines lw 3,\ + 'data-all.csv' using 1:26 title 'fed23' with lines lw 3,\ + 'data-all.csv' using 1:27 title 'fed24' with lines lw 3,\ + 'data-all.csv' using 1:28 title 'rawhide' with lines lw 3,\ + 'data-all.csv' using 1:29 title 'unk_rel' with lines lw 3 +unset output + +set output "/var/www/html/csv-reports/fedora-os-latest.png" +set title "Fedora Selected Versions Unique IPs" +plot ["2013-01-01":"2016-01-20"] \ + 'data-all.csv' using 1:31 title 'Fedora' with lines lw 3,\ + 'data-all.csv' using 1:21 title 'fed18' with lines lw 3,\ + 'data-all.csv' using 1:22 title 'fed19' with lines lw 3,\ + 'data-all.csv' using 1:23 title 'fed20' with lines lw 3,\ + 'data-all.csv' using 1:24 title 'fed21' with lines lw 3,\ + 'data-all.csv' using 1:25 title 'fed22' with lines lw 3,\ + 'data-all.csv' using 1:26 title 'fed23' with lines lw 3,\ + 'data-all.csv' using 1:27 title 'fed24' with lines lw 3,\ + 'data-all.csv' using 1:28 title 'rawhide' with lines lw 3 +unset output + +set output "/var/www/html/csv-reports/fedora-os-latest-stacked.png" +set title "Fedora Selected Versions Unique IPs" +plot ["2013-01-01":"2016-01-20"] \ + 'data-all.csv' using 1:31 title 'Fedora' with lines lw 3,\ + 'data-all.csv' using 1:($21+$22+$23+$24+$25+$26+$27+$28) title 'fed18' with filledcurves x1,\ + 'data-all.csv' using 1:($22+$23+$24+$25+$26+$27+$28) title 'fed19' with filledcurves x1,\ + 'data-all.csv' using 1:($23+$24+$25+$26+$27+$28) title 'fed20' with filledcurves x1,\ + 'data-all.csv' using 1:($24+$25+$26+$27+$28) title 'fed21' with filledcurves x1,\ + 'data-all.csv' using 1:($25+$26+$27+$28) title 'fed22' with filledcurves x1,\ + 'data-all.csv' using 1:($26+$27+$28) title 'fed23' with filledcurves x1,\ + 'data-all.csv' using 1:($27+$28) title 'fed24' with filledcurves x1,\ + 'data-all.csv' using 1:($28) title 'rawhide' with filledcurves x1 +unset output + +set output "/var/www/html/csv-reports/fedora-hardware-full.png" +set title "Fedora Hardware via Unique IPs" +plot ["2007-05-17":"2016-01-20"] \ + 'data-all.csv' using 1:32 title 'alpha' with lines lw 3,\ + 'data-all.csv' using 1:33 title 'ARM' with lines lw 3,\ + 'data-all.csv' using 1:34 title 'ARM64' with lines lw 3,\ + 'data-all.csv' using 1:35 title 'ia64' with lines lw 3,\ + 'data-all.csv' using 1:36 title 'mips' with lines lw 3,\ + 'data-all.csv' using 1:48 title 'ppc' with lines lw 3,\ + 'data-all.csv' using 1:38 title 's390' with lines lw 3,\ + 'data-all.csv' using 1:39 title 'sparc' with lines lw 3,\ + 'data-all.csv' using 1:40 title 'tilegx' with lines lw 3,\ + 'data-all.csv' using 1:44 title 'x86_32' with lines lw 3,\ + 'data-all.csv' using 1:46 title 'x86_64' with lines lw 3,\ + 'data-all.csv' using 1:49 title 'unknown' with lines lw 3 +unset output + +set output "/var/www/html/csv-reports/fedora-hardware-2nd.png" +set title "Fedora Secondary via Unique IPs" +plot ["2007-05-17":"2016-01-20"] \ + 'data-all.csv' using 1:32 title 'alpha' with lines lw 3,\ + 'data-all.csv' using 1:33 title 'ARM' with lines lw 3,\ + 'data-all.csv' using 1:34 title 'ARM64' with lines lw 3,\ + 'data-all.csv' using 1:35 title 'ia64' with lines lw 3,\ + 'data-all.csv' using 1:36 title 'mips' with lines lw 3,\ + 'data-all.csv' using 1:48 title 'ppc' with lines lw 3,\ + 'data-all.csv' using 1:38 title 's390' with lines lw 3,\ + 'data-all.csv' using 1:39 title 'sparc' with lines lw 3,\ + 'data-all.csv' using 1:40 title 'tilegx' with lines lw 3,\ + 'data-all.csv' using 1:49 title 'unknown' with lines lw 3 +unset output + +set output "/var/www/html/csv-reports/fedora-epel-stacked.png" +set title "Fedora Yum Unique IPs" +plot ["2007-05-17":"2016-01-20"] \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28+$29) title 'unknown_rel' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28) title 'rawhide' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27) title 'fed24' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26) title 'fed23' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25) title 'fed22' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24) title 'fed21' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23) title 'fed20' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22) title 'fed19' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21) title 'fed18' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20) title 'fed17' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19) title 'fed16' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18) title 'fed15' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17) title 'fed14' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16) title 'fed13' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15) title 'fed12' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14) title 'fed11' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13) title 'fed10' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12) title 'fed09' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11) title 'fed08' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10) title 'fed07' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9) title 'fed06' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7+$8) title 'fed05' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6+$7) title 'fed04' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5+$6) title 'fed03' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4+$5) title 'epel7' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3+$4) title 'epel6' w filledcurves x1, \ + 'data-all.csv' using 1:($2+$3) title 'epel5' w filledcurves x1, \ + 'data-all.csv' using 1:($2) title 'epel4' w filledcurves x1 +unset output + + +set output "/var/www/html/csv-reports/fedora-stacked.png" +set title "Fedora Epel Yum Unique IPs" +plot ["2007-05-17":"2016-01-20"] \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28+$29) title 'unknown_rel' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28) title 'rawhide' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27) title 'f24' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26) title 'f23' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25) title 'f22' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24) title 'fed21' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23) title 'fed20' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22) title 'fed19' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21) title 'fed18' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20) title 'fed17' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19) title 'fed16' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18) title 'fed15' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17) title 'fed14' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16) title 'fed13' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15) title 'fed12' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14) title 'fed11' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13) title 'fed10' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12) title 'fed09' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11) title 'fed08' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10) title 'fed07' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9) title 'fed06' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8) title 'fed05' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7) title 'fed04' w filledcurves x1, \ + 'data-all.csv' using 1:($6) title 'fed03' w filledcurves x1 +unset output + +set output "/var/www/html/csv-reports/fedora-select-stacked.png" +set title "Fedora Epel Yum Unique IPs" +plot ["2007-05-17":"2016-01-20"] \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28+$29) title 'unknown_rel' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28) title 'rawhide' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27) title 'fed18-24' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20) title 'fed12-17' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14) title 'fed08-11' w filledcurves x1, \ + 'data-all.csv' using 1:($6+$7+$8+$9+$10) title 'fed03-07' w filledcurves x1 +unset output + +# ## +# set output "/var/www/html/csv-reports/fedora-all-mov_avg.png" +# set title "Fedora+Epel Yum Unique IPs (7 day moving average)" +# plot ["2007-05-17":"2016-01-20"] \ +# 'data-mov_avg.csv' using 1:2 title 'epel4' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:3 title 'epel5' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:4 title 'epel6' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:5 title 'epel7' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:6 title 'fed03' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:7 title 'fed04' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:8 title 'fed05' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:9 title 'fed06' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:10 title 'fed07' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:11 title 'fed08' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:12 title 'fed09' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:13 title 'fed10' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:14 title 'fed11' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:15 title 'fed12' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:16 title 'fed13' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:17 title 'fed14' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:18 title 'fed15' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:19 title 'fed16' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:20 title 'fed17' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:21 title 'fed18' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:22 title 'fed19' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:23 title 'fed20' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:24 title 'fed21' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:25 title 'fed22' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:26 title 'fed23' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:27 title 'fed24' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:28 title 'rawhide' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:29 title 'unk_rel' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:30 title 'EPEL' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:50 title 'centos' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:51 title 'rhel' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:31 title 'Fedora' with lines lw 3 +# unset output + + +# set output "/var/www/html/csv-reports/fedora-and-epel-mov_avg.png" +# set title "Fedora Epel Totals Unique IPs (7 days moving average)" +# plot ["2007-05-17":"2016-01-20"] \ +# 'data-mov_avg.csv' using 1:30 title 'EPEL' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:31 title 'Fedora' with lines lw 3 +# unset output + +# set output "/var/www/html/csv-reports/fedora-daily-mov_avg.png" +# set title "Fedora Daily Totals Unique IPs" +# plot ["2007-05-17":"2016-01-20"] \ +# 'data-mov_avg.csv' using 1:31 title 'Fedora' with lines lw 3 +# unset output + +# set output "/var/www/html/csv-reports/fedora-os-all-mov_avg.png" +# set title "Fedora OS Yum Unique IPs (7 days moving average)" +# plot ["2007-05-17":"2016-01-20"] \ +# 'data-mov_avg.csv' using 1:6 title 'fed03' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:7 title 'fed04' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:8 title 'fed05' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:9 title 'fed06' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:10 title 'fed07' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:11 title 'fed08' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:12 title 'fed09' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:13 title 'fed10' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:14 title 'fed11' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:15 title 'fed12' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:16 title 'fed13' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:17 title 'fed14' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:18 title 'fed15' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:19 title 'fed16' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:20 title 'fed17' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:21 title 'fed18' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:22 title 'fed19' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:23 title 'fed20' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:24 title 'fed21' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:25 title 'fed22' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:26 title 'fed23' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:27 title 'fed24' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:28 title 'rawhide' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:29 title 'unk_rel' with lines lw 3 +# unset output + +# set output "/var/www/html/csv-reports/fedora-os-latest-mov_avg.png" +# set title "Fedora Selected Versions Unique IPs (7 days moving average)" +# plot ["2014-01-01":"2016-01-20"] \ +# 'data-mov_avg.csv' using 1:31 title 'Fedora' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:21 title 'fed18' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:22 title 'fed19' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:23 title 'fed20' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:24 title 'fed21' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:25 title 'fed22' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:26 title 'fed23' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:27 title 'fed24' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:28 title 'rawhide' with lines lw 3 +# unset output + +# set output "/var/www/html/csv-reports/fedora-os-latest-stacked-mov_avg.png" +# set title "Fedora Selected Versions Unique IPs (7 days moving average)" +# plot ["2014-01-01":"2016-01-20"] \ +# 'data-mov_avg.csv' using 1:31 title 'Fedora' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:($21+$22+$23+$24+$25+$26+$27+$28) title 'fed18' with filledcurves x1,\ +# 'data-mov_avg.csv' using 1:($22+$23+$24+$25+$26+$27+$28) title 'fed19' with filledcurves x1,\ +# 'data-mov_avg.csv' using 1:($23+$24+$25+$26+$27+$28) title 'fed20' with filledcurves x1,\ +# 'data-mov_avg.csv' using 1:($24+$25+$26+$27+$28) title 'fed21' with filledcurves x1,\ +# 'data-mov_avg.csv' using 1:($25+$26+$27+$28) title 'fed22' with filledcurves x1,\ +# 'data-mov_avg.csv' using 1:($26+$27+$28) title 'fed23' with filledcurves x1,\ +# 'data-mov_avg.csv' using 1:($27+$28) title 'fed24' with filledcurves x1,\ +# 'data-mov_avg.csv' using 1:($28) title 'rawhide' with filledcurves x1 +# unset output + +# set output "/var/www/html/csv-reports/fedora-hardware-full-mov_avg.png" +# set title "Fedora Hardware via Unique IPs (7 days moving average)" +# plot ["2007-05-17":"2016-01-20"] \ +# 'data-mov_avg.csv' using 1:32 title 'alpha' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:33 title 'ARM' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:34 title 'ARM64' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:35 title 'ia64' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:36 title 'mips' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:48 title 'ppc' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:38 title 's390' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:39 title 'sparc' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:40 title 'tilegx' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:44 title 'x86_32' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:46 title 'x86_64' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:49 title 'unknown' with lines lw 3 +# unset output + +# set output "/var/www/html/csv-reports/fedora-hardware-2nd-mov_avg.png" +# set title "Fedora Secondary via Unique IPs (7 days moving average)" +# plot ["2007-05-17":"2016-01-20"] \ +# 'data-mov_avg.csv' using 1:32 title 'alpha' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:33 title 'ARM' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:34 title 'ARM64' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:35 title 'ia64' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:36 title 'mips' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:48 title 'ppc' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:38 title 's390' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:39 title 'sparc' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:40 title 'tilegx' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:49 title 'unknown' with lines lw 3 +# unset output + +# set output "/var/www/html/csv-reports/fedora-all-stacked-mov_avg.png" +# set title "Fedora Epel Yum Unique IPs (7 days moving average)" +# plot ["2007-05-17":"2016-01-20"] \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28+$29) title 'unknown_rel' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28) title 'rawhide' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27) title 'fed24' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26) title 'fed23' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25) title 'fed22' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24) title 'fed21' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23) title 'fed20' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22) title 'fed19' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21) title 'fed18' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20) title 'fed17' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19) title 'fed16' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18) title 'fed15' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17) title 'fed14' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16) title 'fed13' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15) title 'fed12' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14) title 'fed11' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13) title 'fed10' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12) title 'fed09' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11) title 'fed08' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10) title 'fed07' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9) title 'fed06' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8) title 'fed05' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7) title 'fed04' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6) title 'fed03' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5) title 'epel7' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4) title 'epel6' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3) title 'epel5' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2) title 'epel4' w filledcurves x1 +# unset output + +# set output "/var/www/html/csv-reports/fedora-epel-stacked-mov_avg.png" +# set title "Fedora Epel (consolidated) Yum Unique IPs (7 days moving average)" +# plot ["2007-05-17":"2016-01-20"] \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28) title 'unknown_rel' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27) title 'rawhide' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5+$6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26) title 'fedora' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5) title 'epel' w filledcurves x1 +# unset output + +# set output "/var/www/html/csv-reports/fedora-stacked-mov_avg.png" +# set title "Fedora Yum Unique IPs (7 day moving avg)" +# plot ["2007-05-17":"2016-01-20"] \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28+$29) title 'unknown_rel' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28) title 'rawhide' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27) title 'f24' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26) title 'f23' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25) title 'f22' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24) title 'fed21' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23) title 'fed20' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22) title 'fed19' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21) title 'fed18' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20) title 'fed17' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19) title 'fed16' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18) title 'fed15' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17) title 'fed14' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16) title 'fed13' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15) title 'fed12' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14) title 'fed11' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13) title 'fed10' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12) title 'fed09' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11) title 'fed08' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10) title 'fed07' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9) title 'fed06' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8) title 'fed05' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7) title 'fed04' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6) title 'fed03' w filledcurves x1 +# unset output + +# set output "/var/www/html/csv-reports/fedora-select-stacked-mov_avg.png" +# set title "Fedora Epel Yum Unique IPs (7 days moving average)" +# plot ["2007-05-17":"2016-01-20"] \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28+$29) title 'unknown_rel' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27+$28) title 'rawhide' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20+$21+$22+$23+$24+$25+$26+$27) title 'fed18-24' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14+$15+$16+$17+$18+$19+$20) title 'fed12-17' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10+$11+$12+$13+$14) title 'fed08-11' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($6+$7+$8+$9+$10) title 'fed03-07' w filledcurves x1 +# unset output + +# ## +# ## EPEL +# ## + +# set output "/var/www/html/csv-reports/epel-all.png" +# set title "Epel Yum Unique IPs" +# plot ["2007-05-17":"2016-01-20"] \ +# 'data-all.csv' using 1:2 title 'epel4' with lines lw 3,\ +# 'data-all.csv' using 1:3 title 'epel5' with lines lw 3,\ +# 'data-all.csv' using 1:4 title 'epel6' with lines lw 3,\ +# 'data-all.csv' using 1:5 title 'epel7' with lines lw 3,\ +# 'data-all.csv' using 1:30 title 'EPEL' with lines lw 3 +# unset output + +# set output "/var/www/html/csv-reports/epel-all-mov_avg.png" +# set title "Epel Yum Unique IPs (7 days moving average)" +# plot ["2007-05-17":"2016-01-20"] \ +# 'data-mov_avg.csv' using 1:2 title 'epel4' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:3 title 'epel5' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:4 title 'epel6' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:5 title 'epel7' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:30 title 'EPEL' with lines lw 3 +# unset output + +# set output "/var/www/html/csv-reports/epel-arch.png" +# set title "Epel Yum Unique IPs" +# plot ["2007-05-17":"2016-01-20"] \ +# 'data-all.csv' using 1:43 title 'x86_32' with lines lw 3,\ +# 'data-all.csv' using 1:45 title 'x86_64' with lines lw 3,\ +# 'data-all.csv' using 1:47 title 'ppc' with lines lw 3,\ +# 'data-all.csv' using 1:30 title 'EPEL' with lines lw 3 +# unset output + +# set output "/var/www/html/csv-reports/epel-arch-mov_avg.png" +# set title "Epel Yum Unique IPs" +# plot ["2007-05-17":"2016-01-20"] \ +# 'data-mov_avg.csv' using 1:43 title 'x86_32' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:45 title 'x86_64' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:47 title 'ppc' with lines lw 3,\ +# 'data-mov_avg.csv' using 1:30 title 'EPEL' with lines lw 3 +# unset output + +# set output "/var/www/html/csv-reports/epel-stacked.png" +# set title "Epel Yum Unique IPs" +# plot ["2007-05-17":"2016-01-20"] \ +# 'data-all.csv' using 1:($2+$3+$4+$5) title 'epel7' w filledcurves x1, \ +# 'data-all.csv' using 1:($2+$3+$4) title 'epel6' w filledcurves x1, \ +# 'data-all.csv' using 1:($2+$3) title 'epel5' w filledcurves x1, \ +# 'data-all.csv' using 1:($2) title 'epel4' w filledcurves x1 +# unset output + +# set output "/var/www/html/csv-reports/epel-stacked-mov_avg.png" +# set title "Epel Yum Unique IPs" +# plot ["2007-05-17":"2016-01-20"] \ +# 'data-mov_avg.csv' using 1:($2+$3+$4+$5) title 'epel7' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3+$4) title 'epel6' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2+$3) title 'epel5' w filledcurves x1, \ +# 'data-mov_avg.csv' using 1:($2) title 'epel4' w filledcurves x1 +# unset output + diff --git a/roles/web-data-analysis/files/mirror-moving_avg.py b/roles/web-data-analysis/files/mirror-moving_avg.py new file mode 100644 index 0000000000..bbae2531f0 --- /dev/null +++ b/roles/web-data-analysis/files/mirror-moving_avg.py @@ -0,0 +1,70 @@ +# Moving Average + +import pandas +import matplotlib.pyplot as plt +import math + +rolling = 7 + +tree = {} + +df = pandas.read_csv("data-all.csv") + +dates = df['1970-01-01'] +epel4 = pandas.rolling_mean(df['02-epel4'],rolling) +epel5 = pandas.rolling_mean(df['03-epel5'],rolling) +epel6 = pandas.rolling_mean(df['04-epel6'],rolling) +epel7 = pandas.rolling_mean(df['05-epel7'],rolling) +fed03 = pandas.rolling_mean(df['06-f03'], rolling) +fed04 = pandas.rolling_mean(df['07-f04'], rolling) +fed05 = pandas.rolling_mean(df['08-f05'], rolling) +fed06 = pandas.rolling_mean(df['09-f06'], rolling) +fed07 = pandas.rolling_mean(df['10-f07'], rolling) +fed08 = pandas.rolling_mean(df['11-f08'], rolling) +fed09 = pandas.rolling_mean(df['12-f09'], rolling) +fed10 = pandas.rolling_mean(df['13-f10'], rolling) +fed11 = pandas.rolling_mean(df['14-f11'], rolling) +fed12 = pandas.rolling_mean(df['15-f12'], rolling) +fed13 = pandas.rolling_mean(df['16-f13'], rolling) +fed14 = pandas.rolling_mean(df['17-f14'], rolling) +fed15 = pandas.rolling_mean(df['18-f15'], rolling) +fed16 = pandas.rolling_mean(df['19-f16'], rolling) +fed17 = pandas.rolling_mean(df['20-f17'], rolling) +fed18 = pandas.rolling_mean(df['21-f18'], rolling) +fed19 = pandas.rolling_mean(df['22-f19'], rolling) +fed20 = pandas.rolling_mean(df['23-f20'], rolling) +fed21 = pandas.rolling_mean(df['24-f21'], rolling) +fed22 = pandas.rolling_mean(df['25-f22'], rolling) +fed23 = pandas.rolling_mean(df['26-f23'], rolling) +fed24 = pandas.rolling_mean(df['27-f24'], rolling) +rawhide = pandas.rolling_mean(df['28-rawhide'], rolling) +unk_rel = pandas.rolling_mean(df['29-unk_rel'], rolling) +EPEL = pandas.rolling_mean(df['30-epel'], rolling) +Fedora = pandas.rolling_mean(df['31-fedora'], rolling) +alpha = pandas.rolling_mean(df['32-alpha'], rolling) +ARM = pandas.rolling_mean(df['33-arm'], rolling) +ARM64 = pandas.rolling_mean(df['34-arm64'], rolling) +ia64 = pandas.rolling_mean(df['35-ia64'], rolling) +mips = pandas.rolling_mean(df['36-mips'], rolling) +ppc = pandas.rolling_mean(df['37-ppc'], rolling) +s390 = pandas.rolling_mean(df['38-s390'], rolling) +sparc = pandas.rolling_mean(df['39-sparc'], rolling) +tilegx = pandas.rolling_mean(df['40-tilegx'], rolling) +x86_32 = pandas.rolling_mean(df['41-x86_32'], rolling) +x86_64 = pandas.rolling_mean(df['42-x86_64'], rolling) +x86_32_e = pandas.rolling_mean(df['43-x86_32_e'], rolling) +x86_32_f = pandas.rolling_mean(df['44-x86_32_f'], rolling) +x86_64_e = pandas.rolling_mean(df['45-x86_64_e'], rolling) +x86_64_f = pandas.rolling_mean(df['46-x86_64_f'], rolling) +ppc_e = pandas.rolling_mean(df['47-ppc_e'], rolling) +ppc_f = pandas.rolling_mean(df['48-ppc_f'], rolling) +unk_arc = pandas.rolling_mean(df['49-unk_arc'], rolling) +centos = pandas.rolling_mean(df['50-centos'], rolling) +rhel = pandas.rolling_mean(df['51-rhel'], rolling) + +for i in xrange(0,len(dates)): + if math.isnan(epel4[i]): + csv_line = ",".join([dates[i],"0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0","0"]) + else: + csv_line = ",".join([dates[i],str(int(epel4[i])),str(int(epel5[i])),str(int(epel6[i])),str(int(epel7[i])),str(int(fed03[i])),str(int(fed04[i])),str(int(fed05[i])),str(int(fed06[i])),str(int(fed07[i])),str(int(fed08[i])),str(int(fed09[i])),str(int(fed10[i])),str(int(fed11[i])),str(int(fed12[i])),str(int(fed13[i])),str(int(fed14[i])),str(int(fed15[i])),str(int(fed16[i])),str(int(fed17[i])),str(int(fed18[i])),str(int(fed19[i])),str(int(fed20[i])),str(int(fed21[i])),str(int(fed22[i])),str(int(fed23[i])),str(int(fed24[i])),str(int(rawhide[i])),str(int(unk_rel[i])),str(int(EPEL[i])),str(int(Fedora[i])),str(int(alpha[i])),str(int(ARM[i])),str(int(ARM64[i])),str(int(ia64[i])),str(int(mips[i])),str(int(ppc[i])),str(int(s390[i])),str(int(sparc[i])),str(int(tilegx[i])),str(int(x86_32[i])),str(int(x86_64[i])), str(int(x86_32_e[i])),str(int(x86_32_f[i])),str(int(x86_64_e[i])),str(int(x86_64_f[i])),str(int(ppc_e[i])),str(int(ppc_f[i])), str(int(unk_arc[i])),str(int(centos[i])),str(int(rhel[i]))]) + print csv_line diff --git a/roles/web-data-analysis/files/mirrorlist.py b/roles/web-data-analysis/files/mirrorlist.py new file mode 100644 index 0000000000..e313988179 --- /dev/null +++ b/roles/web-data-analysis/files/mirrorlist.py @@ -0,0 +1,567 @@ +#!/usr/bin/python + +# This file is part of Fedora Project Infrastructure Ansible +# Repository. +# +# Fedora Project Infrastructure Ansible Repository is free software: +# you can redistribute it and/or modify it under the terms of the GNU +# General Public License as published by the Free Software Foundation, +# either version 3 of the License, or (at your option) any later +# version. +# +# Fedora Project Infrastructure Ansible Repository is distributed in +# the hope that it will be useful, but WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License +# along with Fedora Project Infrastructure Ansible Repository. If +# not, see . + +import sys +import re +import optparse +import os +import string + + +''' + +Mirror list will go through the file given as an argument and parse +out which releases and architectures were looked for, and by how much. + +''' + +log_line = [ + r"(?P[\d\.]+)\s", + r"(?P\S*)\s", + r"(?P\S*)\s", + r"\[(?P