Merge branch 'master' of /git/ansible

inventory/group_vars/hotness

@@ -15,7 +15,6 @@ sudoers: "{{ private }}/files/sudo/hotness-sudoers"
 
 # These people get told when something goes wrong.
 fedmsg_error_recipients:
-- rbean@redhat.com
 - pingou@fedoraproject.org
 - phracek@redhat.com
 - thozza@redhat.com

inventory/group_vars/hotness-stg

@@ -15,7 +15,6 @@ sudoers: "{{ private }}/files/sudo/hotness-sudoers"
 
 # These people get told when something goes wrong.
 fedmsg_error_recipients:
-- rbean@redhat.com
 - pingou@fedoraproject.org
 - phracek@redhat.com
 - thozza@redhat.com

inventory/group_vars/mbs-frontend

@@ -24,8 +24,8 @@ fas_client_groups: sysadmin-noc,sysadmin-releng
 # These are consumed by a task in roles/fedmsg/base/main.yml
 fedmsg_certs:
 - service: mbs
-  owner: root
-  group: apache 
+  owner: fedmsg
+  group: fedmsg
   can_send:
   - mbs.module.state.change
   # Only the backend sends this message..

inventory/group_vars/mbs-frontend-stg

@@ -24,8 +24,8 @@ fas_client_groups: sysadmin-noc,sysadmin-releng
 # These are consumed by a task in roles/fedmsg/base/main.yml
 fedmsg_certs:
 - service: mbs
-  owner: root
-  group: apache 
+  owner: fedmsg
+  group: fedmsg
   can_send:
   - mbs.module.state.change
   # Only the backend sends this message..

playbooks/clear_memcached.yml

@@ -1,5 +1,5 @@
 - name: clear memcache
-  hosts: memcached
+  hosts: memcached:memcached-stg
   serial: 1
 
   tasks:

roles/cgit/clean_lock_cron/tasks/main.yml

@@ -3,3 +3,4 @@
 
 - name: install the cron file
   copy: src=clean-lock.cron dest=/etc/cron.d/cgit-clean-lock.cron mode=0644
+  tags: cgit

roles/cgit/make_pkgs_list/files/make-cgit-pkgs-list.sh

@@ -8,14 +8,21 @@
 newfile=`mktemp`
 target=/srv/git/repositories
 
+# These are the pagure folders that we don't want to bother showing in cgit (it
+# makes things too slow...)
+blacklist='forks tickets docs requests'
+
 for d in `ls $target`; do
-  if [ ! -L $target/$d ] && [ -d $target/$d ]; then
+  # If it's not a link, it is a directory, and it's not in the blacklist..
+  if [ ! -L $target/$d ] && [ -d $target/$d ] && [[ ! $blacklist == *"$d"* ]]; then
+    # Then take every file inside and stuff it into our tmpfile.
     for f in `ls $target/$d/`; do
      echo "$d/$f" >> $newfile;
     done;
   fi;
 done;
 
+# When we're done with everything in $target, make that avail to cgit.
 mv -Z $newfile /srv/git/pkgs-git-repos-list
 chown apache:apache /srv/git/pkgs-git-repos-list
 chmod 644 /srv/git/pkgs-git-repos-list

roles/cgit/make_pkgs_list/tasks/main.yml

@@ -3,20 +3,25 @@
 
 - name: install semanage
   yum: pkg=/usr/sbin/semanage state=present
+  tags: cgit
 
 - name: create the git root directory (/srv/git)
   file: dest=/srv/git state=directory mode=0755
+  tags: cgit
 
 - name: ensure the repo list file exists
   copy: content="" dest=/srv/git/pkgs-git-repos-list force=no owner=apache group=apache mode=0644
+  tags: cgit
 
 - name: install the script
   copy: src=make-cgit-pkgs-list.sh dest=/usr/local/bin/make-cgit-pkgs-list.sh mode=0755
   when: inventory_hostname.startswith('pkgs')
+  tags: cgit
 
 - name: install the script for fedorapeople
   copy: src=make-people-git.sh dest=/usr/local/bin/make-people-git.sh mode=0755
   when: inventory_hostname.startswith('people')
+  tags: cgit
 
 #
 # For the batcave we just have a small static list
@@ -24,10 +29,12 @@
 - name: install the file for batcave
   copy: src=cgit-projects-batcave dest=/etc/cgit-projects-batcave mode=0644
   when: inventory_hostname.startswith('batcave')
+  tags: cgit
 
 - name: install the file for infinote
   copy: src=cgit-projects-infinote dest=/etc/cgit-projects-infinote mode=0644
   when: inventory_hostname.startswith('infinote')
+  tags: cgit
 
 - name: install the cron job
   cron: >
@@ -36,6 +43,7 @@
     user=root
     job="/usr/local/bin/lock-wrapper make-cgit-pkgs-list '/usr/local/bin/make-cgit-pkgs-list.sh | /usr/local/bin/nag-once make-cgit-pkgs-list 1d 2>&1'"
   when: inventory_hostname.startswith('pkgs')
+  tags: cgit
 
 - name: install the cron job for fedora people cgit
   cron: >
@@ -44,6 +52,7 @@
     user=root
     job="/usr/local/bin/lock-wrapper make-people-git '/usr/local/bin/make-people-git.sh | /usr/local/bin/nag-once make-people-git 1d 2>&1'"
   when: inventory_hostname.startswith('people')
+  tags: cgit
 
 - name: check the selinux context of the repo list
   command: matchpathcon /srv/git/pkgs-git-repos-list

roles/check-compose/files/checkcomp_consumer.py

@@ -1,3 +0,0 @@
-config = {
-    'checkcomp_consumer.prod.enabled': True,
-}

roles/check-compose/tasks/main.yml

@@ -78,7 +78,7 @@
   - restart fedmsg-hub
 
 - name: Enable fedmsg consumer
-  copy: src=checkcomp_consumer.py dest=/etc/fedmsg.d/checkcomp_consumer.py owner=root group=root mode=0644
+  template: src=checkcomp_consumer.py.j2 dest=/etc/fedmsg.d/checkcomp_consumer.py owner=root group=root mode=0644
   notify:
   - restart fedmsg-hub
   tags:

roles/check-compose/templates/checkcomp_consumer.py.j2

@@ -0,0 +1,9 @@
+config = {
+{% if deployment_type is defined and deployment_type == 'prod' %}
+    'checkcomp_consumer.prod.enabled': True,
+{% elif deployment_type is defined and deployment_type == 'stg' %}
+    'checkcomp_consumer.stg.enabled': True,
+{% else %}
+    checkcomp_consumer.test.enabled': True,
+{% endif %}
+}

roles/copr/backend/files/provision/files/mock/site-defaults.cfg

@@ -14,14 +14,12 @@ config_opts['plugin_conf']['compress_logs_enable'] = True
 config_opts['plugin_conf']['compress_logs_opts'] = {}
 config_opts['plugin_conf']['compress_logs_opts']['command'] = "/usr/bin/gzip -9 --force"
 
-{% if ansible_architecture == 'x86_64' %}
 config_opts['plugin_conf']['tmpfs_enable'] = True
 config_opts['plugin_conf']['tmpfs_opts'] = {}
 config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024
 config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = '75g'
 config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755'
 config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = False
-{% endif %}
 
 {% if ansible_distribution == 'Fedora' and ansible_distribution_major_version|int > 21 %}
 config_opts['yum_command'] = '/usr/bin/yum-deprecated'

roles/copr/backend/files/provision/provision_builder_tasks_ppc64le.yml

@@ -4,9 +4,6 @@
 - name: set bigger timeout for yum
   ini_file: dest=/etc/yum.conf section=main option=timeout value=1000
 
-- name: install distribution-gpg-keys which are right now not in fedora main
-  shell: dnf install -y https://kojipkgs.fedoraproject.org//packages/distribution-gpg-keys/1.5/1.fc24/noarch/distribution-gpg-keys-1.5-1.fc24.noarch.rpm
-
 - name: install pkgs
   yum: state=present pkg={{ item }}
   with_items:
@@ -63,6 +60,7 @@
 
 - name: symlink F26 configs to rawhide ones
   shell: for cfg in /etc/mock/fedora-rawhide-*.cfg; do ln -s $cfg `echo $cfg|sed 's/rawhide/26/'`; done
+  when: prepare_base_image is defined
 
 # ansible doesn't support simultaneously usage of async and with_* options
 # it's not even planned for implementation, see https://github.com/ansible/ansible/issues/5841

roles/copr/backend/templates/provision/nova_cloud_vars_ppc64le.yml

@@ -11,7 +11,7 @@ OS_PASSWORD_OLD: "{{ copr_nova_password|default('variable OS_PASSWORD_OLD is und
 OS_PASSWORD: "{{ copr_password|default('variable OS_PASSWORD is undefined')}}"
 
 
-image_name: "builder-f24-ppc64le"
+image_name: "builder-f24-ppc64le-swapmounted"
 flavor_name: "{{ copr_builder_flavor_name }}"
 network_name: "{{ copr_builder_network_name }}"
 key_name: "{{ copr_builder_key_name }}"

roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf

@@ -3,9 +3,8 @@ RequestHeader set X-Scheme https early
 RequestHeader set X-Forwarded-Proto https early
 ProxyPreserveHost On
 
-{% if env == "maybe-in-staging" %}
-RewriteRule ^/v2/([a-zA-Z]*)/blobs/([a-zA-Z]*) https://ourregistry/v2/$1/blobs/$2 [R]
-{% endif %}
+RewriteCond %{HTTP:VIA} !cdn77
+RewriteRule ^/v2/(.*)/blobs/([a-zA-Z0-9:]*) https://cdn.registry.fedoraproject.org/v2/$1/blobs/$2 [R]
 
 # This is terible, but Docker.
 RewriteEngine on

roles/mbs/common/tasks/main.yml

@@ -15,6 +15,15 @@
   - mbs
   - mbs/common
 
+- name: kill development config
+  file: path=/etc/fedmsg.d/module_build_service.py state=absent
+  notify:
+  - restart apache
+  - restart fedmsg-hub
+  tags:
+  - mbs
+  - mbs/common
+
 - name: copy app configuration
   template: >
     src=config.py dest=/etc/module-build-service/config.py
@@ -26,10 +35,22 @@
   - mbs
   - mbs/common
 
-- name: copy client secrets that aren't really secret.
+- name: copy fedmsg configuration
+  template: >
+    src=mbs-fedmsg.py dest=/etc/fedmsg.d/mbs-fedmsg.py
+    owner=root group=fedmsg mode=0644
+  notify:
+  - restart apache
+  - restart fedmsg-hub
+  tags:
+  - mbs
+  - mbs/common
+
+- name: copy client secrets
   template: >
     src=client_secrets.json.{{env}} dest=/etc/module-build-service/client_secrets.json
-    owner=root group=root mode=0644
+    owner=root group=fedmsg mode=0640
+  when: inventory_hostname.startswith('mbs-frontend')
   notify:
   - restart apache
   tags:

roles/mbs/common/templates/client_secrets.json.production

@@ -1,12 +1,11 @@
 {
     "web": {
         "auth_uri": "https://id.fedoraproject.org/openidc/Authorization",
-        "client_id": "mbs-authorizer",
-        "client_secret": "notsecret",
-        "redirect_uris": [
-            "http://localhost:13747/"
-        ],
+        "client_id": "mbs-prod",
+        "client_secret": "{{ mbs_prod_oidc_client_secret }}",
+        "redirect_uris": [],
         "token_uri": "https://id.fedoraproject.org/openidc/Token",
-        "token_introspection_uri": "https://id.fedoraproject.org/openidc/TokenInfo"
+        "token_introspection_uri": "https://id.fedoraproject.org/openidc/TokenInfo",
+        "userinfo_uri": "https://id.fedoraproject.org/openidc/UserInfo"
     }
 }

roles/mbs/common/templates/client_secrets.json.staging

@@ -1,12 +1,11 @@
 {
     "web": {
         "auth_uri": "https://id.stg.fedoraproject.org/openidc/Authorization",
-        "client_id": "mbs-authorizer",
-        "client_secret": "notsecret",
-        "redirect_uris": [
-            "http://localhost:13747/"
-        ],
+        "client_id": "mbs-stg",
+        "client_secret": "{{ mbs_stg_oidc_client_secret }}",
+        "redirect_uris": [],
         "token_uri": "https://id.stg.fedoraproject.org/openidc/Token",
-        "token_introspection_uri": "https://id.stg.fedoraproject.org/openidc/TokenInfo"
+        "token_introspection_uri": "https://id.stg.fedoraproject.org/openidc/TokenInfo",
+        "userinfo_uri": "https://id.stg.fedoraproject.org/openidc/UserInfo"
     }
 }

roles/mbs/common/templates/config.py

@@ -98,8 +98,10 @@ class ProdConfiguration(BaseConfiguration):
     LOG_LEVEL = 'debug'
     LOG_BACKEND = 'console'
 
+    # Yes, use tls.
     PDC_INSECURE = False
-    PDC_DEVELOP = False
+    # No, don't try to obtain a token (we just read.  we don't write.)
+    PDC_DEVELOP = True
 
     KOJI_CONFIG = path.join(confdir, 'koji.conf')
 {% if env == 'staging' %}

roles/mbs/common/templates/mbs-fedmsg.py

@@ -0,0 +1,7 @@
+import socket
+
+config = {
+    # So that the MBS can find it's cert in /etc/fedmsg.d/ssl.py
+    'cert_prefix': 'mbs',
+    'name': 'mbs.%s' % socket.gethostname().split('.', 1)[0],
+}

roles/mbs/frontend/tasks/main.yml

@@ -4,7 +4,7 @@
 - name: disable the scheduler on the frontend
   copy: >
     src={{ item }} dest=/etc/fedmsg.d/{{ item }}
-    owner=apache group=apache mode=0600
+    owner=fedmsg group=fedmsg mode=0644
   with_items:
   - mbs-scheduler.py
   notify:
@@ -41,6 +41,7 @@
   with_items:
   - httpd_can_network_connect_db
   - httpd_can_network_memcache
+  - httpd_can_network_connect
   - httpd_can_sendmail
   tags:
   - mbs

roles/opendkim/files/opendkim.conf

@@ -131,3 +131,7 @@ OversignHeaders	From
 ##  caching service. Useful if the nameserver being used by the filter is
 ##  not local.
 # QueryCache	yes
+#
+# We need to increase the default header size because notifs adds a X-fedmsg header for
+# each message in a digest.
+MaximumHeaders 262144

roles/openqa/dispatcher/tasks/main.yml

@@ -16,8 +16,10 @@
 # - wikitcms_password
 ##      string - password for relval_user
 # - deployment_type
-##      string - Fedora Infrastructure thing; for this role, decides
-##               whether and where to submit wiki results
+##      string - Fedora Infrastructure thing; for this role, the
+##               fedora_openqa config file will be set appropriately
+##               for infra deployments if this is set, so don't set
+##               it for private deployments
 #
 # When all of the above are set, a wikitcms 'credentials' file will
 # be created and result submission to the wiki will be enabled. If
@@ -75,25 +77,29 @@
     chdir: /root/openQA-python-client
   when: "gitclient|changed or instclient.rc > 0"
 
-# We check this out to a different place from the 'server' task, because
-# otherwise it's hard to make sure we install each time it changes.
-- name: Check out openqa_fedora_tools
+- name: Remove old openqa_fedora_tools checkout
+  file: path=/root/openqa_fedora_tools-dispatcher state=absent
+
+- name: Remove old fedora-openqa-schedule script
+  file: path=/usr/bin/fedora-openqa-schedule state=absent
+
+- name: Check out fedora_openqa (scheduler / reporter tool)
   git:
-    repo: https://bitbucket.org/rajcze/openqa_fedora_tools
-    dest: /root/openqa_fedora_tools-dispatcher
+    repo: https://pagure.io/fedora-qa/fedora_openqa.git
+    dest: /root/fedora_openqa
   register: gittools
 
-- name: Check if openqa_fedora_tools has ever been installed
-  stat: path=/usr/bin/fedora-openqa-schedule
+- name: Check if fedora_openqa has ever been installed
+  stat: path=/usr/bin/fedora-openqa
   register: insttools
   changed_when: "1 != 1"
   failed_when: "1 != 1"
   check_mode: no
 
-- name: Install openqa_fedora_tools
+- name: Install fedora_openqa
   command: "python setup.py install"
   args:
-    chdir: /root/openqa_fedora_tools-dispatcher/scheduler
+    chdir: /root/fedora_openqa
   when: "gittools|changed or not insttools.stat.exists"
   notify:
   - restart fedmsg-hub
@@ -103,7 +109,7 @@
   tags:
   - config
 
-- name: Create fedora-openqa-schedule config directory
+- name: Create fedora_openqa config directory
   file: path=/etc/fedora-openqa state=directory owner=root group=root mode=0755
 
 - name: Write schedule.conf

roles/openqa/dispatcher/templates/openqa_consumer.py.j2

@@ -1,13 +1,16 @@
 config = {
 {% if openqa_consumer %}
-    'fedora_openqa_schedule.consumer.enabled': True,
+    'fedora_openqa.scheduler.prod.enabled': True,
 {% endif %}
-{% if wikitcms_user is defined and wikitcms_password is defined %}
 {% if deployment_type is defined and deployment_type == 'prod' %}
-    'fedora_openqa_schedule.wiki.consumer.prod.enabled': True,
-    'fedora_openqa_schedule.resultsdb.reporter.prod.enabled': True,
-{% elif deployment_type is defined and deployment_type == 'stg' %}
-    'fedora_openqa_schedule.wiki.consumer.stg.enabled': True,
+{% if wikitcms_user is defined and wikitcms_password is defined %}
+    'fedora_openqa.reporter.wiki.prod.enabled': True,
 {% endif %}
+    'fedora_openqa.reporter.resultsdb.prod.enabled': True,
+{% elif deployment_type is defined and deployment_type == 'stg' %}
+{% if wikitcms_user is defined and wikitcms_password is defined %}
+    'fedora_openqa.reporter.wiki.stg.enabled': True,
+{% endif %}
+    'fedora_openqa.reporter.resultsdb.stg.enabled': True,
 {% endif %}
 }

roles/openqa/dispatcher/templates/schedule.conf.j2

@@ -1,14 +1,24 @@
 [report]
-openqa_url: https://{{ external_hostname|default(ansible_nodename) }}
-{% if wikitcms_user is defined and wikitcms_password is defined %}
-submit_wiki: true
-submit_resultsdb: true
-{% else %}
-submit_wiki: false
-submit_resultsdb: false
-{% endif %}
 {% if deployment_type is defined and deployment_type == 'prod' %}
 resultsdb_url: http://resultsdb01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
+wiki_hostname: fedoraproject.org
+[consumers]
+prod_oqa_hostname: localhost
+prod_oqa_baseurl: https://{{ external_hostname|default(ansible_nodename) }}
+prod_wiki_hostname: fedoraproject.org
+prod_wiki_report: true
+prod_rdb_url: http://resultsdb01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
+prod_rdb_report: true
 {% elif deployment_type is defined and deployment_type == 'stg' %}
 resultsdb_url: http://resultsdb-stg01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
+wiki_hostname: stg.fedoraproject.org
+[consumers]
+# as we use the production scheduler
+prod_oqa_hostname: localhost
+stg_oqa_hostname: localhost
+stg_oqa_baseurl: https://{{ external_hostname|default(ansible_nodename) }}
+stg_wiki_hostname: stg.fedoraproject.org
+stg_wiki_report: true
+stg_rdb_url: http://resultsdb-stg01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
+stg_rdb_report: true
 {% endif %}

roles/openqa/server/files/createhdds

@@ -1,4 +1,4 @@
 #!/bin/sh
 
 cd /var/lib/openqa/share/factory/hdd/fixed
-LIBGUESTFS_BACKEND=direct withlock /var/lock/createhdds.lock /root/openqa_fedora_tools/tools/createhdds.py all --clean
+LIBGUESTFS_BACKEND=direct withlock /var/lock/createhdds.lock /root/createhdds/createhdds.py all --clean

roles/openqa/server/tasks/main.yml

@@ -120,11 +120,13 @@
   become_user: geekotest
   when: "(testsbranch.stdout.find('Not a git repository') != -1) or (testsbranch.stdout.find('On branch master') != -1)"
 
-- name: Check out openqa_fedora_tools
+- name: Remove old openqa_fedora_tools checkout
+  file: path=/root/openqa_fedora_tools state=absent
+
+- name: Check out createhdds
   git:
-    repo: https://bitbucket.org/rajcze/openqa_fedora_tools
-    dest: /root/openqa_fedora_tools
-  register: git_result
+    repo: https://pagure.io/fedora-qa/createhdds.git
+    dest: /root/createhdds
 
 - name: Create asset directories
   file: path={{ item }} state=directory owner=geekotest group=root mode=0755
@@ -139,7 +141,7 @@
   copy: src=createhdds dest=/etc/cron.daily/createhdds owner=root group=root mode=0755
 
 - name: Check if any hard disk images need (re)building
-  command: "/root/openqa_fedora_tools/tools/createhdds.py check"
+  command: "/root/createhdds/createhdds.py check"
   args:
     chdir: /var/lib/openqa/share/factory/hdd/fixed
   register: diskcheck

roles/pkgdb2/templates/pkgdb2.cfg

@@ -27,7 +27,9 @@ AUTOAPPROVE_PKGERS = [
 
 #### FAS group for the pkgdb admins
 {% if env == 'staging' %}
-ADMIN_GROUP = ['cvsadmin']
+# Factory 2 added to make it easier for us to create repos for the modularity
+# team.  Only in staging.  Feel free to remove!
+ADMIN_GROUP = ['cvsadmin', 'factory2']
 {% else %}
 ADMIN_GROUP = ['sysadmin-main', 'cvsadmin']
 {% endif %}

roles/web-data-analysis/files/condense-hotspot.cron

@@ -0,0 +1 @@
+0 07  * * * root /usr/local/bin/condense-hotspot.sh

roles/web-data-analysis/files/condense-hotspot.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+# This file is part of Fedora Project Infrastructure Ansible
+# Repository.
+#
+# Fedora Project Infrastructure Ansible Repository is free software:
+# you can redistribute it and/or modify it under the terms of the GNU
+# General Public License as published by the Free Software Foundation,
+# either version 3 of the License, or (at your option) any later
+# version.
+#
+# Fedora Project Infrastructure Ansible Repository is distributed in
+# the hope that it will be useful, but WITHOUT ANY WARRANTY; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fedora Project Infrastructure Ansible Repository.  If
+# not, see <http://www.gnu.org/licenses/>.
+
+# There is a multiday delay involved in processing the logs. It
+# may take up to 4 days to get the logs to the main-server. It may
+# take a day to combine all the logs onto combined-httpd. So we assume 
+# we are 5 days behind.
+
+let NUMDAYS=5
+let OLDDAYS=$(( $NUMDAYS+1 ))
+
+PROJECT=hotspot
+WEBLOG=fedoraproject.org
+
+# This is the year/month/day for a N days ago.
+YEAR=$(/bin/date -d "-${NUMDAYS} days" +%Y)
+MONTH=$(/bin/date -d "-${NUMDAYS} days" +%m)
+DAY=$(/bin/date -d "-${NUMDAYS} days" +%d)
+
+# And we have have to deal with year/month/day boundaries for our later grep.
+OLDDATE=$(/bin/date -d "-${OLDDAYS} days" +%Y-%m-%d)
+OLDYEAR=$(/bin/date -d "-${OLDDAYS} days" +%Y)
+
+NFSDIR=/mnt/fedora_stats/combined-http
+TARGET=${NFSDIR}/${YEAR}/${MONTH}/${DAY}
+
+LOGFILE=${TARGET}/${WEBLOG}-access.log
+
+WORKDIR=/mnt/fedora_stats/data/${PROJECT}
+WORKFILE=${WORKDIR}/${YEAR}/${MONTH}/raw-${DAY}
+
+WEBDIR=/var/www/html/csv-reports/${PROJECT}
+
+TEMPDIR=$( mktemp -d /tmp/web-data-analysis.XXXXXXXXX )
+
+LBIN=/usr/local/bin/
+LSHARE=/usr/local/share/web-data-analysis
+
+mkdir -p ${WORKDIR}/${YEAR}/${MONTH}
+if [[ ! -f ${WORKDIR}/${YEAR}/out-${YEAR}-${MONTH} ]]; then
+    touch ${WORKDIR}/${YEAR}/out-${YEAR}-${MONTH}
+fi
+
+if [[ ! -f ${WORKDIR}/out-${YEAR} ]]; then
+    touch ${WORKDIR}/out-${YEAR}
+fi
+
+if [[ ! -f ${LOGFILE} ]]; then
+    echo "No logfile found for ${YEAR}/${MONTH}/${DAY}. Please fix."
+else
+    awk -f ${LSHARE}/${PROJECT}.awk ${LOGFILE} > ${WORKFILE}
+fi
+
+# So the data isn't strictly across month boundries due to the end of
+# the logfiles being at 04:00 versus 23:59. Also log files might get
+# stuck and you end up with days or weeks of data in a single
+# file. Because the data is pretty small we can get away with adding up the data every day.
+
+find ${WORKDIR} -type f | grep raw- | xargs cat  | sort -u | awk 'BEGIN{x=0; y=0}; {if (x != $1){ print x,y; x=$1; y=$2} else {y=y+$2}}' > ${WORKDIR}/worked-all
+
+
+awk -f ${LSHARE}/${PROJECT}-data.awk ${WORKDIR}/worked-all | grep -v "1970-01-01,0,0,0" | sort -u > ${WEBDIR}/${PROJECT}data-all.csv
+
+# Make the seven day moving average file
+/usr/local/bin/hotspot-moving_avg.py > ${WEBDIR}/${PROJECT}data-all-7day-ma.csv
+
+gnuplot  ${LSHARE}/${PROJECT}.gp
+
+# cleanup the temp data
+rm -rf ${TEMPDIR}

roles/web-data-analysis/files/hotspot-data.awk

@@ -0,0 +1,33 @@
+BEGIN{
+  date=strftime("%F",86401);
+  count=1;
+  sum=0;
+  most=0;
+  least=0
+  print "1970-01-01,AVG,LEAST,MAX"
+}
+
+{
+  newdate=strftime("%F",$1);                # convert this to a printable date
+  if (date != newdate){
+    print date "," int(sum/count) "," least "," most;
+    date=newdate;
+    count=1;    # start count to 0. we should have 288 per day but logs are stupid
+    sum=$2;     # start the sum
+    most=$2;    # what is going to be our most per day
+    least=$2;   # what is going to be our least per day
+  } else {
+    count=count+1;
+    sum=sum+$2;
+    if ($2 > most){
+      most=$2;
+    };
+    if ($2 < least) {
+      least=$2;
+    }
+  }
+}
+
+END{
+  print date "," int(sum/count) "," least "," most;
+}

roles/web-data-analysis/files/hotspot-moving_avg.py

@@ -0,0 +1,50 @@
+#!/usr/bin/python
+
+# This file is part of Fedora Project Infrastructure Ansible
+# Repository.
+#
+# Fedora Project Infrastructure Ansible Repository is free software:
+# you can redistribute it and/or modify it under the terms of the GNU
+# General Public License as published by the Free Software Foundation,
+# either version 3 of the License, or (at your option) any later
+# version.
+#
+# Fedora Project Infrastructure Ansible Repository is distributed in
+# the hope that it will be useful, but WITHOUT ANY WARRANTY; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fedora Project Infrastructure Ansible Repository.  If
+# not, see <http://www.gnu.org/licenses/>.
+
+# This is a complete horrible hack to get something done. Patches are
+# really welcome.
+
+import pandas
+#import matplotlib.pyplot as plt
+import math
+
+rolling = 7
+
+tree = {}
+
+df = pandas.read_csv("/var/www/html/csv-reports/hotspot/hotspotdata-all.csv")
+
+dates = df['1970-01-01']
+AVG   = pandas.rolling_mean(df['AVG'],rolling)
+LEAST = pandas.rolling_mean(df['LEAST'],rolling)
+MAX   = pandas.rolling_mean(df['MAX'],rolling)
+
+
+for i in xrange(0,len(dates)):
+    if math.isnan(MAX[i]):
+        csv_line = ",".join([dates[i],"0","0"])
+    else:
+        csv_line = ",".join([dates[i],
+                             str(int(AVG[i])),
+                             str(int(LEAST[i])),
+                             str(int(MAX[i])),
+                         ])
+    print csv_line

roles/web-data-analysis/files/hotspot.awk

@@ -0,0 +1,95 @@
+#
+# Take the apache log line
+# 83.163.161.147 - - [30/Sep/2012:13:54:19 +0000] "GET /static/hotspot.txt HTTP/1.1" 200 3 "-" "dnssec-trigger/0.11"
+# Convert to
+# 1349013000 1
+
+function convertdate(str) {
+  gsub(/\[/, "", str)
+  gsub(/\]/, "", str)
+  split(str,a,":");
+  split(a[1],b,"/");
+  temp="";
+  switch (b[2]) {
+  case "Jan":
+    temp="01"
+    break;
+  case "Feb":
+    temp="02"
+    break;
+  case "Mar":
+    temp="03"
+    break;
+  case "Apr":
+    temp="04"
+    break;
+  case "May":
+    temp="05"
+    break;
+  case "Jun":
+    temp="06"
+    break;
+  case "Jul":
+    temp="07"
+    break;
+  case "Aug":
+    temp="08"
+    break;
+  case "Sep":
+    temp="09"
+    break;
+  case "Oct":
+    temp="10"
+    break;
+  case "Nov":
+    temp="11"
+    break;
+  case "Dec":
+    temp="12"
+    break;
+  default:
+    temp="00"
+    break;
+  }
+  x=b[3]" "temp" "b[1]" "a[2]" "a[3] " "a[4]
+  y=int(mktime(x)/300) # 300 seconds make 5 minutes (I NEED A GLOBAL VAR)
+  return y
+}
+
+
+BEGIN{
+  timestamp=0;
+  num_ts = 0;
+  ts_hotspots=0;
+  total_hotsponts=0;
+}
+
+#
+# We assume that every 300 seconds a system will log in at least 1
+# time because the Networkmanager addon does so.
+# Convert our date stamp to the nearest 5 minute block and add data to
+# it. If the log file goes backwards or jumps etc this will mean
+# multiple outputs for a timestamp. A later process will need to deal
+# with that. All this will do is output how many it saw at that block
+# in the log file.
+#
+
+$7 ~/hotspot.txt/ && $6 ~/GET/ {
+  date = convertdate($4)
+  if (timestamp != date) {
+    num_ts = num_ts +1;
+    print (timestamp*300),ts_hotspots # GLOBAL VAR GOES HERE
+    timestamp = date;
+    ts_hotspots = 1;
+  } else {
+    ts_hotspots = ts_hotspots +1;
+    total_hotspots = total_hotspots +1;
+  }
+}
+
+END {
+  num_ts = num_ts +1;
+  print int(timestamp*300),ts_hotspots # LOOK GLOBAL VAR AGAIN                                                                                                             
+}
+
+## END OF FILE

roles/web-data-analysis/files/hotspot.gp

@@ -0,0 +1,25 @@
+set grid
+set xdata time
+set format x "%Y-%m-%d"
+set timefmt "%Y-%m-%d"
+
+set datafile separator ","
+set term png size 1600,1200
+
+##
+set output "/var/www/html/csv-reports/images/hotspot-all.png"
+set title "IPs grabbing hotspot per day"
+plot ["2014-12-01":"2017-12-31"] \
+     '/var/www/html/csv-reports/hotspot/hotspotdata-all.csv' using 1:2 title 'Average every 5min' with lines lw 4, \
+     '/var/www/html/csv-reports/hotspot/hotspotdata-all.csv' using 1:3 title 'Least 5min' with lines lw 4, \
+     '/var/www/html/csv-reports/hotspot/hotspotdata-all.csv' using 1:4 title 'Max 5min' with lines lw 4
+unset output
+
+##
+set output "/var/www/html/csv-reports/images/hotspot-all-ma.png"
+set title "Moving Average of IPs grabbing hotspot"
+plot ["2014-12-01":"2017-12-31"] \
+     '/var/www/html/csv-reports/hotspot/hotspotdatadata-all-7day-ma.csv' using 1:2 title 'Average every 5min' with lines lw 4, \
+     '/var/www/html/csv-reports/hotspot/hotspotdatadata-all-7day-ma.csv' using 1:3 title 'Least 5min' with lines lw 4, \
+     '/var/www/html/csv-reports/hotspot/hotspotdatadata-all-7day-ma.csv' using 1:4 title 'Max 5min' with lines lw 4
+unset output

roles/web-data-analysis/tasks/main.yml

@@ -39,25 +39,25 @@
 
 - name: scripts to condense data down for further processing
   copy: src={{item}} dest=/usr/local/bin/ mode=0755
-  with_items: [condense-mirrorlogs.sh, condense-getfedoralogs.sh, run-daily-awstats.sh]
+  with_items: [condense-mirrorlogs.sh, condense-getfedoralogs.sh, run-daily-awstats.sh, condense-hotspot.sh ]
   tags:
     - web-data
 
 - name: python scripts to calculate various data
   copy: src={{item}} dest=/usr/local/bin/ mode=0755
-  with_items: [mirrorlist.py, mirrors-moving_avg.py]
+  with_items: [mirrorlist.py, mirrors-moving_avg.py, hotspot-moving_avg.py]
   tags:
     - web-data
 
 - name: awk files for csv creation
   copy: src={{item}} dest=/usr/local/share/web-data-analysis mode=0644
-  with_items: [mirrors-data.awk, getfedora-data.awk, getfedora.awk]
+  with_items: [mirrors-data.awk, getfedora-data.awk, getfedora.awk, hotspot-data.awk, hotspot.awk ]
   tags:
     - web-data
 
 - name: gnuplot file for image creation
   copy: src={{item}} dest=/usr/local/share/web-data-analysis mode=0644
-  with_items: [ mirrors-data.gp, getfedora-data.gp, mirrors-moving.gp ]
+  with_items: [ mirrors-data.gp, getfedora-data.gp, mirrors-moving.gp, hotspot.gp ]
   tags:
     - web-data