From cc95c6482b74fbdbe9469b359ce642df67c10c8c Mon Sep 17 00:00:00 2001
From: Adam Williamson <awilliam@redhat.com>
Date: Tue, 14 Feb 2017 16:57:23 -0800
Subject: [PATCH 01/38] openqa: update roles for openqa_fedora_tools repo split

We split fedora_openqa and createhdds out into their own repos,
this should adapt to the change (I hope).
---
 roles/openqa/dispatcher/tasks/main.yml        | 30 +++++++++++--------
 .../templates/openqa_consumer.py.j2           |  9 +++---
 .../dispatcher/templates/schedule.conf.j2     | 20 ++++++++-----
 roles/openqa/server/files/createhdds          |  2 +-
 roles/openqa/server/tasks/main.yml            | 12 ++++----
 5 files changed, 43 insertions(+), 30 deletions(-)

diff --git a/roles/openqa/dispatcher/tasks/main.yml b/roles/openqa/dispatcher/tasks/main.yml
index da87403149..1a3fc0de22 100644
--- a/roles/openqa/dispatcher/tasks/main.yml
+++ b/roles/openqa/dispatcher/tasks/main.yml
@@ -16,8 +16,10 @@
 # - wikitcms_password
 ##      string - password for relval_user
 # - deployment_type
-##      string - Fedora Infrastructure thing; for this role, decides
-##               whether and where to submit wiki results
+##      string - Fedora Infrastructure thing; for this role, the
+##               fedora_openqa config file will be set appropriately
+##               for infra deployments if this is set, so don't set
+##               it for private deployments
 #
 # When all of the above are set, a wikitcms 'credentials' file will
 # be created and result submission to the wiki will be enabled. If
@@ -75,25 +77,29 @@
     chdir: /root/openQA-python-client
   when: "gitclient|changed or instclient.rc > 0"
 
-# We check this out to a different place from the 'server' task, because
-# otherwise it's hard to make sure we install each time it changes.
-- name: Check out openqa_fedora_tools
+- name: Remove old openqa_fedora_tools checkout
+  file: path=/root/openqa_fedora_tools-dispatcher state=absent
+
+- name: Remove old fedora-openqa-schedule script
+  file: path=/usr/bin/fedora-openqa-schedule state=absent
+
+- name: Check out fedora_openqa (scheduler / reporter tool)
   git:
-    repo: https://bitbucket.org/rajcze/openqa_fedora_tools
-    dest: /root/openqa_fedora_tools-dispatcher
+    repo: https://pagure.io/fedora-qa/fedora_openqa.git
+    dest: /root/fedora_openqa
   register: gittools
 
-- name: Check if openqa_fedora_tools has ever been installed
-  stat: path=/usr/bin/fedora-openqa-schedule
+- name: Check if fedora_openqa has ever been installed
+  stat: path=/usr/bin/fedora-openqa
   register: insttools
   changed_when: "1 != 1"
   failed_when: "1 != 1"
   check_mode: no
 
-- name: Install openqa_fedora_tools
+- name: Install fedora_openqa
   command: "python setup.py install"
   args:
-    chdir: /root/openqa_fedora_tools-dispatcher/scheduler
+    chdir: /root/fedora_openqa
   when: "gittools|changed or not insttools.stat.exists"
   notify:
   - restart fedmsg-hub
@@ -103,7 +109,7 @@
   tags:
   - config
 
-- name: Create fedora-openqa-schedule config directory
+- name: Create fedora_openqa config directory
   file: path=/etc/fedora-openqa state=directory owner=root group=root mode=0755
 
 - name: Write schedule.conf
diff --git a/roles/openqa/dispatcher/templates/openqa_consumer.py.j2 b/roles/openqa/dispatcher/templates/openqa_consumer.py.j2
index 33219f72dc..b78e9a180c 100644
--- a/roles/openqa/dispatcher/templates/openqa_consumer.py.j2
+++ b/roles/openqa/dispatcher/templates/openqa_consumer.py.j2
@@ -1,13 +1,14 @@
 config = {
 {% if openqa_consumer %}
-    'fedora_openqa_schedule.consumer.enabled': True,
+    'fedora_openqa.scheduler.prod.enabled': True,
 {% endif %}
 {% if wikitcms_user is defined and wikitcms_password is defined %}
 {% if deployment_type is defined and deployment_type == 'prod' %}
-    'fedora_openqa_schedule.wiki.consumer.prod.enabled': True,
-    'fedora_openqa_schedule.resultsdb.reporter.prod.enabled': True,
+    'fedora_openqa.reporter.wiki.prod.enabled': True,
+    'fedora_openqa.reporter.resultsdb.prod.enabled': True,
 {% elif deployment_type is defined and deployment_type == 'stg' %}
-    'fedora_openqa_schedule.wiki.consumer.stg.enabled': True,
+    'fedora_openqa.reporter.wiki.stg.enabled': True,
+    'fedora_openqa.reporter.resultsdb.stg.enabled': True,
 {% endif %}
 {% endif %}
 }
diff --git a/roles/openqa/dispatcher/templates/schedule.conf.j2 b/roles/openqa/dispatcher/templates/schedule.conf.j2
index ab79ae819d..64b5c4bb72 100644
--- a/roles/openqa/dispatcher/templates/schedule.conf.j2
+++ b/roles/openqa/dispatcher/templates/schedule.conf.j2
@@ -1,14 +1,18 @@
 [report]
-openqa_url: https://{{ external_hostname|default(ansible_nodename) }}
-{% if wikitcms_user is defined and wikitcms_password is defined %}
-submit_wiki: true
-submit_resultsdb: true
-{% else %}
-submit_wiki: false
-submit_resultsdb: false
-{% endif %}
 {% if deployment_type is defined and deployment_type == 'prod' %}
 resultsdb_url: http://resultsdb01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
+wiki_hostname: fedoraproject.org
+prod_oqa_baseurl: https://{{ external_hostname|default(ansible_nodename) }}
+prod_wiki_hostname: fedoraproject.org
+prod_wiki_report: true
+prod_rdb_url: http://resultsdb01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
+prod_rdb_report: true
 {% elif deployment_type is defined and deployment_type == 'stg' %}
 resultsdb_url: http://resultsdb-stg01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
+wiki_hostname: stg.fedoraproject.org
+stg_oqa_baseurl: https://{{ external_hostname|default(ansible_nodename) }}
+stg_wiki_hostname: stg.fedoraproject.org
+stg_wiki_report: true
+stg_rdb_url: http://resultsdb-stg01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
+stg_rdb_report: true
 {% endif %}
diff --git a/roles/openqa/server/files/createhdds b/roles/openqa/server/files/createhdds
index c76019d8b6..794d80450e 100644
--- a/roles/openqa/server/files/createhdds
+++ b/roles/openqa/server/files/createhdds
@@ -1,4 +1,4 @@
 #!/bin/sh
 
 cd /var/lib/openqa/share/factory/hdd/fixed
-LIBGUESTFS_BACKEND=direct withlock /var/lock/createhdds.lock /root/openqa_fedora_tools/tools/createhdds.py all --clean
+LIBGUESTFS_BACKEND=direct withlock /var/lock/createhdds.lock /root/createhdds/createhdds.py all --clean
diff --git a/roles/openqa/server/tasks/main.yml b/roles/openqa/server/tasks/main.yml
index 957290b6be..27dbe1a39c 100644
--- a/roles/openqa/server/tasks/main.yml
+++ b/roles/openqa/server/tasks/main.yml
@@ -120,11 +120,13 @@
   become_user: geekotest
   when: "(testsbranch.stdout.find('Not a git repository') != -1) or (testsbranch.stdout.find('On branch master') != -1)"
 
-- name: Check out openqa_fedora_tools
+- name: Remove old openqa_fedora_tools checkout
+  file: path=/root/openqa_fedora_tools state=absent
+
+- name: Check out createhdds
   git:
-    repo: https://bitbucket.org/rajcze/openqa_fedora_tools
-    dest: /root/openqa_fedora_tools
-  register: git_result
+    repo: https://pagure.io/fedora-qa/createhdds.git
+    dest: /root/createhdds
 
 - name: Create asset directories
   file: path={{ item }} state=directory owner=geekotest group=root mode=0755
@@ -139,7 +141,7 @@
   copy: src=createhdds dest=/etc/cron.daily/createhdds owner=root group=root mode=0755
 
 - name: Check if any hard disk images need (re)building
-  command: "/root/openqa_fedora_tools/tools/createhdds.py check"
+  command: "/root/createhdds/createhdds.py check"
   args:
     chdir: /var/lib/openqa/share/factory/hdd/fixed
   register: diskcheck

From fff2dddf8fc851e19146e9f9967114f52300ff33 Mon Sep 17 00:00:00 2001
From: Adam Williamson <awilliam@redhat.com>
Date: Tue, 14 Feb 2017 17:08:09 -0800
Subject: [PATCH 02/38] openqa/dispatcher: tweak template logic a bit

---
 roles/openqa/dispatcher/templates/openqa_consumer.py.j2 | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/roles/openqa/dispatcher/templates/openqa_consumer.py.j2 b/roles/openqa/dispatcher/templates/openqa_consumer.py.j2
index b78e9a180c..d0b62d2282 100644
--- a/roles/openqa/dispatcher/templates/openqa_consumer.py.j2
+++ b/roles/openqa/dispatcher/templates/openqa_consumer.py.j2
@@ -2,13 +2,15 @@ config = {
 {% if openqa_consumer %}
     'fedora_openqa.scheduler.prod.enabled': True,
 {% endif %}
-{% if wikitcms_user is defined and wikitcms_password is defined %}
 {% if deployment_type is defined and deployment_type == 'prod' %}
+{% if wikitcms_user is defined and wikitcms_password is defined %}
     'fedora_openqa.reporter.wiki.prod.enabled': True,
+{% endif %}
     'fedora_openqa.reporter.resultsdb.prod.enabled': True,
 {% elif deployment_type is defined and deployment_type == 'stg' %}
+{% if wikitcms_user is defined and wikitcms_password is defined %}
     'fedora_openqa.reporter.wiki.stg.enabled': True,
+{% endif %}
     'fedora_openqa.reporter.resultsdb.stg.enabled': True,
 {% endif %}
-{% endif %}
 }

From a4366ff9f56b8c68467dbd4a3a612ca0322aa0df Mon Sep 17 00:00:00 2001
From: Adam Williamson <awilliam@redhat.com>
Date: Tue, 14 Feb 2017 17:12:37 -0800
Subject: [PATCH 03/38] check-compose: use appropriate fedmsg consumer

Don't use prod consumer on staging...
---
 roles/check-compose/files/checkcomp_consumer.py        | 3 ---
 roles/check-compose/tasks/main.yml                     | 2 +-
 roles/check-compose/templates/checkcomp_consumer.py.j2 | 9 +++++++++
 3 files changed, 10 insertions(+), 4 deletions(-)
 delete mode 100644 roles/check-compose/files/checkcomp_consumer.py
 create mode 100644 roles/check-compose/templates/checkcomp_consumer.py.j2

diff --git a/roles/check-compose/files/checkcomp_consumer.py b/roles/check-compose/files/checkcomp_consumer.py
deleted file mode 100644
index 5d3068cffb..0000000000
--- a/roles/check-compose/files/checkcomp_consumer.py
+++ /dev/null
@@ -1,3 +0,0 @@
-config = {
-    'checkcomp_consumer.prod.enabled': True,
-}
diff --git a/roles/check-compose/tasks/main.yml b/roles/check-compose/tasks/main.yml
index 0d91c4833b..7c8e52b8bd 100644
--- a/roles/check-compose/tasks/main.yml
+++ b/roles/check-compose/tasks/main.yml
@@ -78,7 +78,7 @@
   - restart fedmsg-hub
 
 - name: Enable fedmsg consumer
-  copy: src=checkcomp_consumer.py dest=/etc/fedmsg.d/checkcomp_consumer.py owner=root group=root mode=0644
+  template: src=checkcomp_consumer.py.j2 dest=/etc/fedmsg.d/checkcomp_consumer.py owner=root group=root mode=0644
   notify:
   - restart fedmsg-hub
   tags:
diff --git a/roles/check-compose/templates/checkcomp_consumer.py.j2 b/roles/check-compose/templates/checkcomp_consumer.py.j2
new file mode 100644
index 0000000000..d5af64308a
--- /dev/null
+++ b/roles/check-compose/templates/checkcomp_consumer.py.j2
@@ -0,0 +1,9 @@
+config = {
+{% if deployment_type is defined and deployment_type == 'prod' %}
+    'checkcomp_consumer.prod.enabled': True,
+{% elif deployment_type is defined and deployment_type == 'stg' %}
+    'checkcomp_consumer.stg.enabled': True,
+{% else %}
+    checkcomp_consumer.test.enabled': True,
+{% endif %}
+}

From 369c00eb808e70321e1569c4a1fb39ff3bd856fa Mon Sep 17 00:00:00 2001
From: Adam Williamson <awilliam@redhat.com>
Date: Tue, 14 Feb 2017 23:05:55 -0800
Subject: [PATCH 04/38] openqa/dispatcher: use 'localhost' as oqa hostname in
 infra

---
 roles/openqa/dispatcher/templates/schedule.conf.j2 | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/roles/openqa/dispatcher/templates/schedule.conf.j2 b/roles/openqa/dispatcher/templates/schedule.conf.j2
index 64b5c4bb72..07d8fdbd79 100644
--- a/roles/openqa/dispatcher/templates/schedule.conf.j2
+++ b/roles/openqa/dispatcher/templates/schedule.conf.j2
@@ -2,12 +2,14 @@
 {% if deployment_type is defined and deployment_type == 'prod' %}
 resultsdb_url: http://resultsdb01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
 wiki_hostname: fedoraproject.org
+prod_oqa_hostname: localhost
 prod_oqa_baseurl: https://{{ external_hostname|default(ansible_nodename) }}
 prod_wiki_hostname: fedoraproject.org
 prod_wiki_report: true
 prod_rdb_url: http://resultsdb01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
 prod_rdb_report: true
 {% elif deployment_type is defined and deployment_type == 'stg' %}
+stg_oqa_hostname: localhost
 resultsdb_url: http://resultsdb-stg01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
 wiki_hostname: stg.fedoraproject.org
 stg_oqa_baseurl: https://{{ external_hostname|default(ansible_nodename) }}

From 7dd87e5a12a264447d36a528de4c942e5743fae0 Mon Sep 17 00:00:00 2001
From: Adam Williamson <awilliam@redhat.com>
Date: Tue, 14 Feb 2017 23:30:09 -0800
Subject: [PATCH 05/38] openqa/dispatcher: fix schedule.conf again

gah, section names.
---
 roles/openqa/dispatcher/templates/schedule.conf.j2 | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/roles/openqa/dispatcher/templates/schedule.conf.j2 b/roles/openqa/dispatcher/templates/schedule.conf.j2
index 07d8fdbd79..92e13f9e41 100644
--- a/roles/openqa/dispatcher/templates/schedule.conf.j2
+++ b/roles/openqa/dispatcher/templates/schedule.conf.j2
@@ -2,6 +2,7 @@
 {% if deployment_type is defined and deployment_type == 'prod' %}
 resultsdb_url: http://resultsdb01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
 wiki_hostname: fedoraproject.org
+[consumers]
 prod_oqa_hostname: localhost
 prod_oqa_baseurl: https://{{ external_hostname|default(ansible_nodename) }}
 prod_wiki_hostname: fedoraproject.org
@@ -9,9 +10,10 @@ prod_wiki_report: true
 prod_rdb_url: http://resultsdb01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
 prod_rdb_report: true
 {% elif deployment_type is defined and deployment_type == 'stg' %}
-stg_oqa_hostname: localhost
 resultsdb_url: http://resultsdb-stg01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
 wiki_hostname: stg.fedoraproject.org
+[consumers]
+stg_oqa_hostname: localhost
 stg_oqa_baseurl: https://{{ external_hostname|default(ansible_nodename) }}
 stg_wiki_hostname: stg.fedoraproject.org
 stg_wiki_report: true

From bfc98713865f77c15c2a8df9d2634cc6492399f7 Mon Sep 17 00:00:00 2001
From: clime <clime@redhat.com>
Date: Tue, 14 Feb 2017 16:44:29 +0100
Subject: [PATCH 06/38] enable tmpfs plugin also for ppc64 builders now that we
 have the 1T swap properly mounted on them

---
 roles/copr/backend/files/provision/files/mock/site-defaults.cfg | 2 --
 1 file changed, 2 deletions(-)

diff --git a/roles/copr/backend/files/provision/files/mock/site-defaults.cfg b/roles/copr/backend/files/provision/files/mock/site-defaults.cfg
index 399126e6fd..d8491025e5 100644
--- a/roles/copr/backend/files/provision/files/mock/site-defaults.cfg
+++ b/roles/copr/backend/files/provision/files/mock/site-defaults.cfg
@@ -14,14 +14,12 @@ config_opts['plugin_conf']['compress_logs_enable'] = True
 config_opts['plugin_conf']['compress_logs_opts'] = {}
 config_opts['plugin_conf']['compress_logs_opts']['command'] = "/usr/bin/gzip -9 --force"
 
-{% if ansible_architecture == 'x86_64' %}
 config_opts['plugin_conf']['tmpfs_enable'] = True
 config_opts['plugin_conf']['tmpfs_opts'] = {}
 config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024
 config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = '75g'
 config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755'
 config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = False
-{% endif %}
 
 {% if ansible_distribution == 'Fedora' and ansible_distribution_major_version|int > 21 %}
 config_opts['yum_command'] = '/usr/bin/yum-deprecated'

From e957d5785b47f911a74984e5c53fea864b171a37 Mon Sep 17 00:00:00 2001
From: clime <clime@redhat.com>
Date: Tue, 14 Feb 2017 17:06:52 +0100
Subject: [PATCH 07/38] copr-backend: update provision configs for swapmounted
 ppc64le image

---
 .../files/provision/provision_builder_tasks_ppc64le.yml       | 4 +---
 .../backend/templates/provision/nova_cloud_vars_ppc64le.yml   | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/roles/copr/backend/files/provision/provision_builder_tasks_ppc64le.yml b/roles/copr/backend/files/provision/provision_builder_tasks_ppc64le.yml
index 9effa0fbcb..1b2e89dc0d 100644
--- a/roles/copr/backend/files/provision/provision_builder_tasks_ppc64le.yml
+++ b/roles/copr/backend/files/provision/provision_builder_tasks_ppc64le.yml
@@ -4,9 +4,6 @@
 - name: set bigger timeout for yum
   ini_file: dest=/etc/yum.conf section=main option=timeout value=1000
 
-- name: install distribution-gpg-keys which are right now not in fedora main
-  shell: dnf install -y https://kojipkgs.fedoraproject.org//packages/distribution-gpg-keys/1.5/1.fc24/noarch/distribution-gpg-keys-1.5-1.fc24.noarch.rpm
-
 - name: install pkgs
   yum: state=present pkg={{ item }}
   with_items:
@@ -63,6 +60,7 @@
 
 - name: symlink F26 configs to rawhide ones
   shell: for cfg in /etc/mock/fedora-rawhide-*.cfg; do ln -s $cfg `echo $cfg|sed 's/rawhide/26/'`; done
+  when: prepare_base_image is defined
 
 # ansible doesn't support simultaneously usage of async and with_* options
 # it's not even planned for implementation, see https://github.com/ansible/ansible/issues/5841
diff --git a/roles/copr/backend/templates/provision/nova_cloud_vars_ppc64le.yml b/roles/copr/backend/templates/provision/nova_cloud_vars_ppc64le.yml
index f7a7da1cfc..2a65936f9e 100644
--- a/roles/copr/backend/templates/provision/nova_cloud_vars_ppc64le.yml
+++ b/roles/copr/backend/templates/provision/nova_cloud_vars_ppc64le.yml
@@ -11,7 +11,7 @@ OS_PASSWORD_OLD: "{{ copr_nova_password|default('variable OS_PASSWORD_OLD is und
 OS_PASSWORD: "{{ copr_password|default('variable OS_PASSWORD is undefined')}}"
 
 
-image_name: "builder-f24-ppc64le"
+image_name: "builder-f24-ppc64le-swapmounted"
 flavor_name: "{{ copr_builder_flavor_name }}"
 network_name: "{{ copr_builder_network_name }}"
 key_name: "{{ copr_builder_key_name }}"

From 1db24ba991020ef3ac6359ba5adb655793d3e8d8 Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Wed, 15 Feb 2017 11:47:36 +0000
Subject: [PATCH 08/38] No more hotness for rbean.

---
 inventory/group_vars/hotness     | 1 -
 inventory/group_vars/hotness-stg | 1 -
 2 files changed, 2 deletions(-)

diff --git a/inventory/group_vars/hotness b/inventory/group_vars/hotness
index 0cb42a9d74..8844ef65fd 100644
--- a/inventory/group_vars/hotness
+++ b/inventory/group_vars/hotness
@@ -15,7 +15,6 @@ sudoers: "{{ private }}/files/sudo/hotness-sudoers"
 
 # These people get told when something goes wrong.
 fedmsg_error_recipients:
-- rbean@redhat.com
 - pingou@fedoraproject.org
 - phracek@redhat.com
 - thozza@redhat.com
diff --git a/inventory/group_vars/hotness-stg b/inventory/group_vars/hotness-stg
index a6938f282e..3e8ed20de2 100644
--- a/inventory/group_vars/hotness-stg
+++ b/inventory/group_vars/hotness-stg
@@ -15,7 +15,6 @@ sudoers: "{{ private }}/files/sudo/hotness-sudoers"
 
 # These people get told when something goes wrong.
 fedmsg_error_recipients:
-- rbean@redhat.com
 - pingou@fedoraproject.org
 - phracek@redhat.com
 - thozza@redhat.com

From 668b66570be3180dc4dfc7fd3251f21f12bfb550 Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Wed, 15 Feb 2017 12:10:59 +0000
Subject: [PATCH 09/38] Blacklist pagure folders in dist-git to make cache
 building faster.

---
 roles/cgit/make_pkgs_list/files/make-cgit-pkgs-list.sh | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/roles/cgit/make_pkgs_list/files/make-cgit-pkgs-list.sh b/roles/cgit/make_pkgs_list/files/make-cgit-pkgs-list.sh
index bfc306186f..3e2b6f814a 100644
--- a/roles/cgit/make_pkgs_list/files/make-cgit-pkgs-list.sh
+++ b/roles/cgit/make_pkgs_list/files/make-cgit-pkgs-list.sh
@@ -8,14 +8,21 @@
 newfile=`mktemp`
 target=/srv/git/repositories
 
+# These are the pagure folders that we don't want to bother showing in cgit (it
+# makes things too slow...)
+blacklist='forks tickets docs requests'
+
 for d in `ls $target`; do
-  if [ ! -L $target/$d ] && [ -d $target/$d ]; then
+  # If it's not a link, it is a directory, and it's not in the blacklist..
+  if [ ! -L $target/$d ] && [ -d $target/$d ] && [[ ! $blacklist == *"$d"* ]]; then
+    # Then take every file inside and stuff it into our tmpfile.
     for f in `ls $target/$d/`; do
      echo "$d/$f" >> $newfile;
     done;
   fi;
 done;
 
+# When we're done with everything in $target, make that avail to cgit.
 mv -Z $newfile /srv/git/pkgs-git-repos-list
 chown apache:apache /srv/git/pkgs-git-repos-list
 chmod 644 /srv/git/pkgs-git-repos-list

From a2d16d03ec0835aca86a845988b2bd65d3af7b6a Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Wed, 15 Feb 2017 12:12:23 +0000
Subject: [PATCH 10/38] Tag this role.

---
 roles/cgit/make_pkgs_list/tasks/main.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/roles/cgit/make_pkgs_list/tasks/main.yml b/roles/cgit/make_pkgs_list/tasks/main.yml
index 061a9e1886..a7e0d832c5 100644
--- a/roles/cgit/make_pkgs_list/tasks/main.yml
+++ b/roles/cgit/make_pkgs_list/tasks/main.yml
@@ -3,20 +3,25 @@
 
 - name: install semanage
   yum: pkg=/usr/sbin/semanage state=present
+  tags: cgit
 
 - name: create the git root directory (/srv/git)
   file: dest=/srv/git state=directory mode=0755
+  tags: cgit
 
 - name: ensure the repo list file exists
   copy: content="" dest=/srv/git/pkgs-git-repos-list force=no owner=apache group=apache mode=0644
+  tags: cgit
 
 - name: install the script
   copy: src=make-cgit-pkgs-list.sh dest=/usr/local/bin/make-cgit-pkgs-list.sh mode=0755
   when: inventory_hostname.startswith('pkgs')
+  tags: cgit
 
 - name: install the script for fedorapeople
   copy: src=make-people-git.sh dest=/usr/local/bin/make-people-git.sh mode=0755
   when: inventory_hostname.startswith('people')
+  tags: cgit
 
 #
 # For the batcave we just have a small static list
@@ -24,10 +29,12 @@
 - name: install the file for batcave
   copy: src=cgit-projects-batcave dest=/etc/cgit-projects-batcave mode=0644
   when: inventory_hostname.startswith('batcave')
+  tags: cgit
 
 - name: install the file for infinote
   copy: src=cgit-projects-infinote dest=/etc/cgit-projects-infinote mode=0644
   when: inventory_hostname.startswith('infinote')
+  tags: cgit
 
 - name: install the cron job
   cron: >
@@ -36,6 +43,7 @@
     user=root
     job="/usr/local/bin/lock-wrapper make-cgit-pkgs-list '/usr/local/bin/make-cgit-pkgs-list.sh | /usr/local/bin/nag-once make-cgit-pkgs-list 1d 2>&1'"
   when: inventory_hostname.startswith('pkgs')
+  tags: cgit
 
 - name: install the cron job for fedora people cgit
   cron: >
@@ -44,6 +52,7 @@
     user=root
     job="/usr/local/bin/lock-wrapper make-people-git '/usr/local/bin/make-people-git.sh | /usr/local/bin/nag-once make-people-git 1d 2>&1'"
   when: inventory_hostname.startswith('people')
+  tags: cgit
 
 - name: check the selinux context of the repo list
   command: matchpathcon /srv/git/pkgs-git-repos-list

From ee403b03140416be3dd54437f4d10b77b3aca1d3 Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Wed, 15 Feb 2017 12:13:03 +0000
Subject: [PATCH 11/38] This one, too.

---
 roles/cgit/clean_lock_cron/tasks/main.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/roles/cgit/clean_lock_cron/tasks/main.yml b/roles/cgit/clean_lock_cron/tasks/main.yml
index be680c416d..9a189f65c0 100644
--- a/roles/cgit/clean_lock_cron/tasks/main.yml
+++ b/roles/cgit/clean_lock_cron/tasks/main.yml
@@ -3,3 +3,4 @@
 
 - name: install the cron file
   copy: src=clean-lock.cron dest=/etc/cron.d/cgit-clean-lock.cron mode=0644
+  tags: cgit

From c1b28e4568f7b67922d9fd2f8afd6ed9d2340517 Mon Sep 17 00:00:00 2001
From: Patrick Uiterwijk <puiterwijk@redhat.com>
Date: Wed, 15 Feb 2017 15:48:04 +0000
Subject: [PATCH 12/38] Redirect blob requests to CDN

Signed-off-by: Patrick Uiterwijk <puiterwijk@redhat.com>
---
 .../reverseproxy/templates/reversepassproxy.registry.conf    | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf b/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf
index 149fd04d43..8894e439eb 100644
--- a/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf
+++ b/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf
@@ -3,9 +3,8 @@ RequestHeader set X-Scheme https early
 RequestHeader set X-Forwarded-Proto https early
 ProxyPreserveHost On
 
-{% if env == "maybe-in-staging" %}
-RewriteRule ^/v2/([a-zA-Z]*)/blobs/([a-zA-Z]*) https://ourregistry/v2/$1/blobs/$2 [R]
-{% endif %}
+RewriteCond %{HTTP:VIA} !cdn77
+RewriteRule ^/v2/([a-zA-Z]*)/blobs/([a-zA-Z]*) https://cdn.registry.fedoraproject.org/v2/$1/blobs/$2 [R]
 
 # This is terible, but Docker.
 RewriteEngine on

From 9f6dafbe935122c80341d1cacd62f4d41f02ba09 Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Wed, 15 Feb 2017 15:52:03 +0000
Subject: [PATCH 13/38] Set OIDC secrets in place for MBS service provider.

---
 roles/mbs/common/tasks/main.yml                       |  5 +++--
 .../common/templates/client_secrets.json.production   | 11 +++++------
 .../mbs/common/templates/client_secrets.json.staging  | 11 +++++------
 3 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/roles/mbs/common/tasks/main.yml b/roles/mbs/common/tasks/main.yml
index 5a1126d3bb..a2518479c9 100644
--- a/roles/mbs/common/tasks/main.yml
+++ b/roles/mbs/common/tasks/main.yml
@@ -26,10 +26,11 @@
   - mbs
   - mbs/common
 
-- name: copy client secrets that aren't really secret.
+- name: copy client secrets
   template: >
     src=client_secrets.json.{{env}} dest=/etc/module-build-service/client_secrets.json
-    owner=root group=root mode=0644
+    owner=root group=apache mode=0640
+  when: inventory_hostname.startswith('mbs-web')
   notify:
   - restart apache
   tags:
diff --git a/roles/mbs/common/templates/client_secrets.json.production b/roles/mbs/common/templates/client_secrets.json.production
index e25c1b49a8..9b12f4cc78 100644
--- a/roles/mbs/common/templates/client_secrets.json.production
+++ b/roles/mbs/common/templates/client_secrets.json.production
@@ -1,12 +1,11 @@
 {
     "web": {
         "auth_uri": "https://id.fedoraproject.org/openidc/Authorization",
-        "client_id": "mbs-authorizer",
-        "client_secret": "notsecret",
-        "redirect_uris": [
-            "http://localhost:13747/"
-        ],
+        "client_id": "mbs-prod",
+        "client_secret": "{{ mbs_prod_oidc_client_secret }}",
+        "redirect_uris": [],
         "token_uri": "https://id.fedoraproject.org/openidc/Token",
-        "token_introspection_uri": "https://id.fedoraproject.org/openidc/TokenInfo"
+        "token_introspection_uri": "https://id.fedoraproject.org/openidc/TokenInfo",
+        "userinfo_uri": "https://id.fedoraproject.org/openidc/UserInfo"
     }
 }
diff --git a/roles/mbs/common/templates/client_secrets.json.staging b/roles/mbs/common/templates/client_secrets.json.staging
index 7fd5069dae..f78371caf8 100644
--- a/roles/mbs/common/templates/client_secrets.json.staging
+++ b/roles/mbs/common/templates/client_secrets.json.staging
@@ -1,12 +1,11 @@
 {
     "web": {
         "auth_uri": "https://id.stg.fedoraproject.org/openidc/Authorization",
-        "client_id": "mbs-authorizer",
-        "client_secret": "notsecret",
-        "redirect_uris": [
-            "http://localhost:13747/"
-        ],
+        "client_id": "mbs-stg",
+        "client_secret": "{{ mbs_stg_oidc_client_secret }}",
+        "redirect_uris": [],
         "token_uri": "https://id.stg.fedoraproject.org/openidc/Token",
-        "token_introspection_uri": "https://id.stg.fedoraproject.org/openidc/TokenInfo"
+        "token_introspection_uri": "https://id.stg.fedoraproject.org/openidc/TokenInfo",
+        "userinfo_uri": "https://id.stg.fedoraproject.org/openidc/UserInfo"
     }
 }

From ffd0b74867f97ed0926e03a87e41fb7c1be61ac6 Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Wed, 15 Feb 2017 15:53:34 +0000
Subject: [PATCH 14/38] Typofix in hostname.

---
 roles/mbs/common/tasks/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/mbs/common/tasks/main.yml b/roles/mbs/common/tasks/main.yml
index a2518479c9..df5312890b 100644
--- a/roles/mbs/common/tasks/main.yml
+++ b/roles/mbs/common/tasks/main.yml
@@ -30,7 +30,7 @@
   template: >
     src=client_secrets.json.{{env}} dest=/etc/module-build-service/client_secrets.json
     owner=root group=apache mode=0640
-  when: inventory_hostname.startswith('mbs-web')
+  when: inventory_hostname.startswith('mbs-frontend')
   notify:
   - restart apache
   tags:

From aa1514ea24444dfb870c08e4f3b837781ef22de3 Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Wed, 15 Feb 2017 15:58:01 +0000
Subject: [PATCH 15/38] Typofix, wrong group.

---
 roles/mbs/common/tasks/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/mbs/common/tasks/main.yml b/roles/mbs/common/tasks/main.yml
index df5312890b..d7ecb0437e 100644
--- a/roles/mbs/common/tasks/main.yml
+++ b/roles/mbs/common/tasks/main.yml
@@ -29,7 +29,7 @@
 - name: copy client secrets
   template: >
     src=client_secrets.json.{{env}} dest=/etc/module-build-service/client_secrets.json
-    owner=root group=apache mode=0640
+    owner=root group=fedmsg mode=0640
   when: inventory_hostname.startswith('mbs-frontend')
   notify:
   - restart apache

From e1601ca869840fd3ca91b1c3d92c0879e63d56f2 Mon Sep 17 00:00:00 2001
From: Patrick Uiterwijk <puiterwijk@redhat.com>
Date: Wed, 15 Feb 2017 15:59:42 +0000
Subject: [PATCH 16/38] More flexible regex

Signed-off-by: Patrick Uiterwijk <puiterwijk@redhat.com>
---
 .../httpd/reverseproxy/templates/reversepassproxy.registry.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf b/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf
index 8894e439eb..8fd2808ce4 100644
--- a/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf
+++ b/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf
@@ -4,7 +4,7 @@ RequestHeader set X-Forwarded-Proto https early
 ProxyPreserveHost On
 
 RewriteCond %{HTTP:VIA} !cdn77
-RewriteRule ^/v2/([a-zA-Z]*)/blobs/([a-zA-Z]*) https://cdn.registry.fedoraproject.org/v2/$1/blobs/$2 [R]
+RewriteRule ^/v2/(.*)/blobs/([a-zA-Z0-9:]*) https://cdn.registry.fedoraproject.org/v2/$1/blobs/$2 [R]
 
 # This is terible, but Docker.
 RewriteEngine on

From 2b041d9cfd9b0c6da6abc2fcc7a657982e332c98 Mon Sep 17 00:00:00 2001
From: Adam Williamson <awilliam@redhat.com>
Date: Wed, 15 Feb 2017 08:20:28 -0800
Subject: [PATCH 17/38] openqa/dispatcher: more config fixing

goddamnit, adam, why'd you make so many options.
---
 roles/openqa/dispatcher/templates/schedule.conf.j2 | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/roles/openqa/dispatcher/templates/schedule.conf.j2 b/roles/openqa/dispatcher/templates/schedule.conf.j2
index 92e13f9e41..f599284ff0 100644
--- a/roles/openqa/dispatcher/templates/schedule.conf.j2
+++ b/roles/openqa/dispatcher/templates/schedule.conf.j2
@@ -13,6 +13,8 @@ prod_rdb_report: true
 resultsdb_url: http://resultsdb-stg01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
 wiki_hostname: stg.fedoraproject.org
 [consumers]
+# as we use the production scheduler
+prod_oqa_hostname: localhost
 stg_oqa_hostname: localhost
 stg_oqa_baseurl: https://{{ external_hostname|default(ansible_nodename) }}
 stg_wiki_hostname: stg.fedoraproject.org

From 81961f7ccd4978711aa98332bfd111867c6ddabf Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Wed, 15 Feb 2017 16:39:55 +0000
Subject: [PATCH 18/38] mbs/frontend needs httpd_can_network_connect to query
 dist-git for validation.

---
 roles/mbs/frontend/tasks/main.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/roles/mbs/frontend/tasks/main.yml b/roles/mbs/frontend/tasks/main.yml
index 1d24257955..92edf75aaa 100644
--- a/roles/mbs/frontend/tasks/main.yml
+++ b/roles/mbs/frontend/tasks/main.yml
@@ -41,6 +41,7 @@
   with_items:
   - httpd_can_network_connect_db
   - httpd_can_network_memcache
+  - httpd_can_network_connect
   - httpd_can_sendmail
   tags:
   - mbs

From 5e452622a260605bfb94b2f26cd764d3e96661c1 Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Wed, 15 Feb 2017 17:19:39 +0000
Subject: [PATCH 19/38] fedmsg config for mbs.

---
 roles/mbs/common/tasks/main.yml          | 11 +++++++++++
 roles/mbs/common/templates/mbs-fedmsg.py |  4 ++++
 2 files changed, 15 insertions(+)
 create mode 100644 roles/mbs/common/templates/mbs-fedmsg.py

diff --git a/roles/mbs/common/tasks/main.yml b/roles/mbs/common/tasks/main.yml
index d7ecb0437e..d1807900e6 100644
--- a/roles/mbs/common/tasks/main.yml
+++ b/roles/mbs/common/tasks/main.yml
@@ -26,6 +26,17 @@
   - mbs
   - mbs/common
 
+- name: copy fedmsg configuration
+  template: >
+    src=mbs-fedmsg.py dest=/etc/fedmsg.d/mbs-fedmsg.py
+    owner=root group=fedmsg mode=0644
+  notify:
+  - restart apache
+  - restart fedmsg-hub
+  tags:
+  - mbs
+  - mbs/common
+
 - name: copy client secrets
   template: >
     src=client_secrets.json.{{env}} dest=/etc/module-build-service/client_secrets.json
diff --git a/roles/mbs/common/templates/mbs-fedmsg.py b/roles/mbs/common/templates/mbs-fedmsg.py
new file mode 100644
index 0000000000..709fd2cb98
--- /dev/null
+++ b/roles/mbs/common/templates/mbs-fedmsg.py
@@ -0,0 +1,4 @@
+config = {
+    # So that the MBS can find it's cert in /etc/fedmsg.d/ssl.py
+    'name': 'mbs',
+}

From d7f9660ced9eb46686c22ddb8802284b330ea986 Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Wed, 15 Feb 2017 17:22:23 +0000
Subject: [PATCH 20/38] Fix permissions.

---
 roles/mbs/frontend/tasks/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/mbs/frontend/tasks/main.yml b/roles/mbs/frontend/tasks/main.yml
index 92edf75aaa..761318c437 100644
--- a/roles/mbs/frontend/tasks/main.yml
+++ b/roles/mbs/frontend/tasks/main.yml
@@ -4,7 +4,7 @@
 - name: disable the scheduler on the frontend
   copy: >
     src={{ item }} dest=/etc/fedmsg.d/{{ item }}
-    owner=apache group=apache mode=0600
+    owner=fedmsg group=fedmsg mode=0644
   with_items:
   - mbs-scheduler.py
   notify:

From a8f17c9cd4ad6dd1fd21ba5a7d9ba219ec3a1b2d Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Wed, 15 Feb 2017 17:25:35 +0000
Subject: [PATCH 21/38] Remove unwanted file from mbs rpm.

---
 roles/mbs/common/tasks/main.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/roles/mbs/common/tasks/main.yml b/roles/mbs/common/tasks/main.yml
index d1807900e6..7e46835ff8 100644
--- a/roles/mbs/common/tasks/main.yml
+++ b/roles/mbs/common/tasks/main.yml
@@ -15,6 +15,15 @@
   - mbs
   - mbs/common
 
+- name: kill development config
+  file: path=/etc/fedmsg.d/module_build_service.py state=absent
+  notify:
+  - restart apache
+  - restart fedmsg-hub
+  tags:
+  - mbs
+  - mbs/common
+
 - name: copy app configuration
   template: >
     src=config.py dest=/etc/module-build-service/config.py

From a59ac60311ccafd614c0833fb54deeeb1b3e1b77 Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Wed, 15 Feb 2017 17:28:33 +0000
Subject: [PATCH 22/38] Wrong value.

---
 roles/mbs/common/templates/mbs-fedmsg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/mbs/common/templates/mbs-fedmsg.py b/roles/mbs/common/templates/mbs-fedmsg.py
index 709fd2cb98..7a3823d48b 100644
--- a/roles/mbs/common/templates/mbs-fedmsg.py
+++ b/roles/mbs/common/templates/mbs-fedmsg.py
@@ -1,4 +1,4 @@
 config = {
     # So that the MBS can find it's cert in /etc/fedmsg.d/ssl.py
-    'name': 'mbs',
+    'cert_prefix': 'mbs',
 }

From 89ea6230aba97941ed19a730b166a32c06bdca6d Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Wed, 15 Feb 2017 17:39:11 +0000
Subject: [PATCH 23/38] Change owner to match the apache process.

---
 inventory/group_vars/mbs-frontend     | 4 ++--
 inventory/group_vars/mbs-frontend-stg | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/inventory/group_vars/mbs-frontend b/inventory/group_vars/mbs-frontend
index 34e3981f0a..b84a047deb 100644
--- a/inventory/group_vars/mbs-frontend
+++ b/inventory/group_vars/mbs-frontend
@@ -24,8 +24,8 @@ fas_client_groups: sysadmin-noc,sysadmin-releng
 # These are consumed by a task in roles/fedmsg/base/main.yml
 fedmsg_certs:
 - service: mbs
-  owner: root
-  group: apache 
+  owner: fedmsg
+  group: fedmsg
   can_send:
   - mbs.module.state.change
   # Only the backend sends this message..
diff --git a/inventory/group_vars/mbs-frontend-stg b/inventory/group_vars/mbs-frontend-stg
index 1c104763c7..0b692933c7 100644
--- a/inventory/group_vars/mbs-frontend-stg
+++ b/inventory/group_vars/mbs-frontend-stg
@@ -24,8 +24,8 @@ fas_client_groups: sysadmin-noc,sysadmin-releng
 # These are consumed by a task in roles/fedmsg/base/main.yml
 fedmsg_certs:
 - service: mbs
-  owner: root
-  group: apache 
+  owner: fedmsg
+  group: fedmsg
   can_send:
   - mbs.module.state.change
   # Only the backend sends this message..

From 01a57debe5815864d1292bc320932c029520f864 Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Wed, 15 Feb 2017 17:42:20 +0000
Subject: [PATCH 24/38] Apparently we need both.

---
 roles/mbs/common/templates/mbs-fedmsg.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/roles/mbs/common/templates/mbs-fedmsg.py b/roles/mbs/common/templates/mbs-fedmsg.py
index 7a3823d48b..3485e012de 100644
--- a/roles/mbs/common/templates/mbs-fedmsg.py
+++ b/roles/mbs/common/templates/mbs-fedmsg.py
@@ -1,4 +1,5 @@
 config = {
     # So that the MBS can find it's cert in /etc/fedmsg.d/ssl.py
     'cert_prefix': 'mbs',
+    'name': 'mbs',
 }

From 87c3a80974a0dafd84767b6ec3b1b2862a2d3485 Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Wed, 15 Feb 2017 17:58:09 +0000
Subject: [PATCH 25/38] Name is longer than just the prefix.

---
 roles/mbs/common/templates/mbs-fedmsg.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/roles/mbs/common/templates/mbs-fedmsg.py b/roles/mbs/common/templates/mbs-fedmsg.py
index 3485e012de..3715f21b0e 100644
--- a/roles/mbs/common/templates/mbs-fedmsg.py
+++ b/roles/mbs/common/templates/mbs-fedmsg.py
@@ -1,5 +1,7 @@
+import socket
+
 config = {
     # So that the MBS can find it's cert in /etc/fedmsg.d/ssl.py
     'cert_prefix': 'mbs',
-    'name': 'mbs',
+    'name': 'mbs-%s' % socket.gethostname(),
 }

From 9db908833369a2d6365d00a25b3ef56cae6a451f Mon Sep 17 00:00:00 2001
From: Kevin Fenzi <kevin@scrye.com>
Date: Wed, 15 Feb 2017 18:00:19 +0000
Subject: [PATCH 26/38] increase opendkim header size

---
 roles/opendkim/files/opendkim.conf | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/roles/opendkim/files/opendkim.conf b/roles/opendkim/files/opendkim.conf
index 03353770c0..c50d1d1692 100644
--- a/roles/opendkim/files/opendkim.conf
+++ b/roles/opendkim/files/opendkim.conf
@@ -131,3 +131,7 @@ OversignHeaders	From
 ##  caching service. Useful if the nameserver being used by the filter is
 ##  not local.
 # QueryCache	yes
+#
+# We need to increase the default header size because notifs adds a X-fedmsg header for
+# each message in a digest.
+MaximumHeaders 262144

From eab08ddbd65a8f9bdeabefe73d070e42e4f281e9 Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Wed, 15 Feb 2017 18:48:01 +0000
Subject: [PATCH 27/38] Playbook should work in both prod and stg, thank you.

---
 playbooks/clear_memcached.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/playbooks/clear_memcached.yml b/playbooks/clear_memcached.yml
index fe913c3656..eaae858dad 100644
--- a/playbooks/clear_memcached.yml
+++ b/playbooks/clear_memcached.yml
@@ -1,5 +1,5 @@
 - name: clear memcache
-  hosts: memcached
+  hosts: memcached:memcached-stg
   serial: 1
 
   tasks:

From 568d06cd84cf25bdee45759f76b2e16f8d5067a7 Mon Sep 17 00:00:00 2001
From: Stephen Smoogen <smooge@redhat.com>
Date: Wed, 15 Feb 2017 19:26:30 +0000
Subject: [PATCH 28/38] we have a temporary fix for the hotspot data

---
 .../files/condense-hotspot.sh                 | 88 +++++++++++++++++
 .../web-data-analysis/files/hotspot-data.awk  | 40 ++++++++
 .../files/hotspot-moving_avg.py               | 56 +++++++++++
 roles/web-data-analysis/files/hotspot.awk     | 95 +++++++++++++++++++
 roles/web-data-analysis/files/hotspot.gp      | 25 +++++
 5 files changed, 304 insertions(+)
 create mode 100644 roles/web-data-analysis/files/condense-hotspot.sh
 create mode 100644 roles/web-data-analysis/files/hotspot-data.awk
 create mode 100644 roles/web-data-analysis/files/hotspot-moving_avg.py
 create mode 100644 roles/web-data-analysis/files/hotspot.awk
 create mode 100644 roles/web-data-analysis/files/hotspot.gp

diff --git a/roles/web-data-analysis/files/condense-hotspot.sh b/roles/web-data-analysis/files/condense-hotspot.sh
new file mode 100644
index 0000000000..17f3fcb2a7
--- /dev/null
+++ b/roles/web-data-analysis/files/condense-hotspot.sh
@@ -0,0 +1,88 @@
+#!/bin/bash
+
+# This file is part of Fedora Project Infrastructure Ansible
+# Repository.
+#
+# Fedora Project Infrastructure Ansible Repository is free software:
+# you can redistribute it and/or modify it under the terms of the GNU
+# General Public License as published by the Free Software Foundation,
+# either version 3 of the License, or (at your option) any later
+# version.
+#
+# Fedora Project Infrastructure Ansible Repository is distributed in
+# the hope that it will be useful, but WITHOUT ANY WARRANTY; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fedora Project Infrastructure Ansible Repository.  If
+# not, see <http://www.gnu.org/licenses/>.
+
+# There is a multiday delay involved in processing the logs. It
+# may take up to 4 days to get the logs to the main-server. It may
+# take a day to combine all the logs onto combined-httpd. So we assume 
+# we are 5 days behind.
+
+let NUMDAYS=5
+let OLDDAYS=$(( $NUMDAYS+1 ))
+
+PROJECT=hotspot
+WEBLOG=fedoraproject.org
+
+# This is the year/month/day for a N days ago.
+YEAR=$(/bin/date -d "-${NUMDAYS} days" +%Y)
+MONTH=$(/bin/date -d "-${NUMDAYS} days" +%m)
+DAY=$(/bin/date -d "-${NUMDAYS} days" +%d)
+
+# And we have have to deal with year/month/day boundaries for our later grep.
+OLDDATE=$(/bin/date -d "-${OLDDAYS} days" +%Y-%m-%d)
+OLDYEAR=$(/bin/date -d "-${OLDDAYS} days" +%Y)
+
+NFSDIR=/mnt/fedora_stats/combined-http
+TARGET=${NFSDIR}/${YEAR}/${MONTH}/${DAY}
+
+LOGFILE=${TARGET}/${WEBLOG}-access.log
+
+WORKDIR=/mnt/fedora_stats/data/${PROJECT}
+WORKFILE=${WORKDIR}/${YEAR}/${MONTH}/raw-${DAY}
+
+WEBDIR=/var/www/html/csv-reports/${PROJECT}
+
+TEMPDIR=$( mktemp -d /tmp/web-data-analysis.XXXXXXXXX )
+
+LBIN=/usr/local/bin/
+LSHARE=/usr/local/share/web-data-analysis
+
+mkdir -p ${WORKDIR}/${YEAR}/${MONTH}
+if [[ ! -f ${WORKDIR}/${YEAR}/out-${YEAR}-${MONTH} ]]; then
+    touch ${WORKDIR}/${YEAR}/out-${YEAR}-${MONTH}
+fi
+
+if [[ ! -f ${WORKDIR}/out-${YEAR} ]]; then
+    touch ${WORKDIR}/out-${YEAR}
+fi
+
+if [[ ! -f ${LOGFILE} ]]; then
+    echo "No logfile found for ${YEAR}/${MONTH}/${DAY}. Please fix."
+else
+    awk -f ${LSHARE}/${PROJECT}.awk ${LOGFILE} > ${WORKFILE}
+fi
+
+# So the data isn't strictly across month boundries due to the end of
+# the logfiles being at 04:00 versus 23:59. Also log files might get
+# stuck and you end up with days or weeks of data in a single
+# file. Because the data is pretty small we can get away with adding up the data every day.
+
+find ${WORKEDIR} -type f | grep raw- | xargs cat  | sort -u | awk 'BEGIN{x=0; y=0}; {if (x != $1){ print x,y; x=$1; y=$2} else {y=y+$2}}' > ${WORKEDIR}/worked-all
+
+
+awk -f ${LSHARE}/${PROJECT}-data.awk ${WORKEDIR}/worked-all | sort -u > ${WEBDIR}/${PROJECT}data-all.csv
+
+# Make the seven day moving average file
+/usr/local/bin/hotspot-moving_avg.py > ${WEBDIR}/${PROJECT}data-all-7day-ma.csv
+
+gnuplot  ${LSHARE}/${PROJECT}.gp
+
+# cleanup the temp data
+rm -rf ${TEMPDIR}
diff --git a/roles/web-data-analysis/files/hotspot-data.awk b/roles/web-data-analysis/files/hotspot-data.awk
new file mode 100644
index 0000000000..496def828e
--- /dev/null
+++ b/roles/web-data-analysis/files/hotspot-data.awk
@@ -0,0 +1,40 @@
+BEGIN{
+  date=strftime("%F",0);
+  count=1;
+  sum=0;
+  most=0;
+  least=0
+}
+
+{
+  newdate=strftime("%F",$1);                # convert this to a printable date
+  if (date == strftime("%F",0)){            # we hit a min time and need to just print same stuff.
+    print date ",AVG,LEAST,MAX"
+    date=newdate;
+    count=1;    # start count to 0. we should have 288 per day but logs are stupid
+    sum=$2;     # start the sum
+    most=$2;    # what is going to be our most per day
+    least=$2;   # what is going to be our least per day
+  } else {
+    if (date != newdate){
+      print date "," int(sum/count) "," least "," most;
+      date=newdate;
+      count=1;    # start count to 0. we should have 288 per day but logs are stupid
+      sum=$2;     # start the sum
+      most=$2;    # what is going to be our most per day
+      least=$2;   # what is going to be our least per day
+    } else {
+      count=count+1;
+      sum=sum+$2;
+      if ($2 > most){
+	most=$2;
+      };
+      if ($2 < least) {
+	least=$2;
+      }
+    }
+  }
+}
+END{
+  print date "," int(sum/count) "," least "," most;
+}
diff --git a/roles/web-data-analysis/files/hotspot-moving_avg.py b/roles/web-data-analysis/files/hotspot-moving_avg.py
new file mode 100644
index 0000000000..1489df62c9
--- /dev/null
+++ b/roles/web-data-analysis/files/hotspot-moving_avg.py
@@ -0,0 +1,56 @@
+#!/usr/bin/python
+
+# This file is part of Fedora Project Infrastructure Ansible
+# Repository.
+#
+# Fedora Project Infrastructure Ansible Repository is free software:
+# you can redistribute it and/or modify it under the terms of the GNU
+# General Public License as published by the Free Software Foundation,
+# either version 3 of the License, or (at your option) any later
+# version.
+#
+# Fedora Project Infrastructure Ansible Repository is distributed in
+# the hope that it will be useful, but WITHOUT ANY WARRANTY; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fedora Project Infrastructure Ansible Repository.  If
+# not, see <http://www.gnu.org/licenses/>.
+
+# This is a complete horrible hack to get something done. Patches are
+# really welcome.
+
+import pandas
+#import matplotlib.pyplot as plt
+import math
+
+# Moving Average
+
+import pandas
+import matplotlib.pyplot as plt
+import math
+
+rolling = 7
+
+tree = {}
+
+df = pandas.read_csv("hotspot-new.csv")
+
+dates = df['1970-01-01']
+AVG   = pandas.rolling_mean(df['AVG'],rolling)
+LEAST = pandas.rolling_mean(df['LEAST'],rolling)
+MAX   = pandas.rolling_mean(df['MAX'],rolling)
+
+
+for i in xrange(0,len(dates)):
+    if math.isnan(MAX[i]):
+        csv_line = ",".join([dates[i],"0","0"])
+    else:
+        csv_line = ",".join([dates[i],
+                             str(int(AVG[i])),
+                             str(int(LEAST[i])),
+                             str(int(MAX[i])),
+                         ])
+    print csv_line
diff --git a/roles/web-data-analysis/files/hotspot.awk b/roles/web-data-analysis/files/hotspot.awk
new file mode 100644
index 0000000000..f47da6958e
--- /dev/null
+++ b/roles/web-data-analysis/files/hotspot.awk
@@ -0,0 +1,95 @@
+#
+# Take the apache log line
+# 83.163.161.147 - - [30/Sep/2012:13:54:19 +0000] "GET /static/hotspot.txt HTTP/1.1" 200 3 "-" "dnssec-trigger/0.11"
+# Convert to
+# 1349013000 1
+
+function convertdate(str) {
+  gsub(/\[/, "", str)
+  gsub(/\]/, "", str)
+  split(str,a,":");
+  split(a[1],b,"/");
+  temp="";
+  switch (b[2]) {
+  case "Jan":
+    temp="01"
+    break;
+  case "Feb":
+    temp="02"
+    break;
+  case "Mar":
+    temp="03"
+    break;
+  case "Apr":
+    temp="04"
+    break;
+  case "May":
+    temp="05"
+    break;
+  case "Jun":
+    temp="06"
+    break;
+  case "Jul":
+    temp="07"
+    break;
+  case "Aug":
+    temp="08"
+    break;
+  case "Sep":
+    temp="09"
+    break;
+  case "Oct":
+    temp="10"
+    break;
+  case "Nov":
+    temp="11"
+    break;
+  case "Dec":
+    temp="12"
+    break;
+  default:
+    temp="00"
+    break;
+  }
+  x=b[3]" "temp" "b[1]" "a[2]" "a[3] " "a[4]
+  y=int(mktime(x)/300) # 300 seconds make 5 minutes (I NEED A GLOBAL VAR)
+  return y
+}
+
+
+BEGIN{
+  timestamp=0;
+  num_ts = 0;
+  ts_hotspots=0;
+  total_hotsponts=0;
+}
+
+#
+# We assume that every 300 seconds a system will log in at least 1
+# time because the Networkmanager addon does so.
+# Convert our date stamp to the nearest 5 minute block and add data to
+# it. If the log file goes backwards or jumps etc this will mean
+# multiple outputs for a timestamp. A later process will need to deal
+# with that. All this will do is output how many it saw at that block
+# in the log file.
+#
+
+$7 ~/hotspot.txt/ && $6 ~/GET/ {
+  date = convertdate($4)
+  if (timestamp != date) {
+    num_ts = num_ts +1;
+    print (timestamp*300),ts_hotspots # GLOBAL VAR GOES HERE
+    timestamp = date;
+    ts_hotspots = 1;
+  } else {
+    ts_hotspots = ts_hotspots +1;
+    total_hotspots = total_hotspots +1;
+  }
+}
+
+END {
+  num_ts = num_ts +1;
+  print int(timestamp*300),ts_hotspots # LOOK GLOBAL VAR AGAIN                                                                                                             
+}
+
+## END OF FILE
diff --git a/roles/web-data-analysis/files/hotspot.gp b/roles/web-data-analysis/files/hotspot.gp
new file mode 100644
index 0000000000..2b4d8c9c5c
--- /dev/null
+++ b/roles/web-data-analysis/files/hotspot.gp
@@ -0,0 +1,25 @@
+set grid
+set xdata time
+set format x "%Y-%m-%d"
+set timefmt "%Y-%m-%d"
+
+set datafile separator ","
+set term png size 1600,1200
+
+##
+set output "hotspot-all.png"
+set title "IPs grabbing hotspot per day"
+plot ["2014-12-01":"2017-12-31"] \
+     '/var/www/html/csv-reports/hotspot/hotspotdata-all.csv' using 1:2 title 'Average every 5min' with lines lw 4, \
+     '/var/www/html/csv-reports/hotspot/hotspotdata-all.csv' using 1:3 title 'Least 5min' with lines lw 4, \
+     '/var/www/html/csv-reports/hotspot/hotspotdata-all.csv' using 1:4 title 'Max 5min' with lines lw 4
+unset output
+
+##
+set output "hotspot-all-ma.png"
+set title "Moving Average of IPs grabbing hotspot"
+plot ["2014-12-01":"2017-12-31"] \
+     '/var/www/html/csv-reports/hotspot/hotspotdatadata-all-7day-ma.csv' using 1:2 title 'Average every 5min' with lines lw 4, \
+     '/var/www/html/csv-reports/hotspot/hotspotdatadata-all-7day-ma.csv' using 1:3 title 'Least 5min' with lines lw 4, \
+     '/var/www/html/csv-reports/hotspot/hotspotdatadata-all-7day-ma.csv' using 1:4 title 'Max 5min' with lines lw 4
+unset output

From 08a5d0c36bdb7d782ffe8eab4f86fa65d597cebc Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Wed, 15 Feb 2017 19:26:56 +0000
Subject: [PATCH 29/38] Give factory2 rights on pkgdb2 in staging only.

---
 roles/pkgdb2/templates/pkgdb2.cfg | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/roles/pkgdb2/templates/pkgdb2.cfg b/roles/pkgdb2/templates/pkgdb2.cfg
index c033723ea9..bdb677d508 100644
--- a/roles/pkgdb2/templates/pkgdb2.cfg
+++ b/roles/pkgdb2/templates/pkgdb2.cfg
@@ -27,7 +27,9 @@ AUTOAPPROVE_PKGERS = [
 
 #### FAS group for the pkgdb admins
 {% if env == 'staging' %}
-ADMIN_GROUP = ['cvsadmin']
+# Factory 2 added to make it easier for us to create repos for the modularity
+# team.  Only in staging.  Feel free to remove!
+ADMIN_GROUP = ['cvsadmin', 'factory2']
 {% else %}
 ADMIN_GROUP = ['sysadmin-main', 'cvsadmin']
 {% endif %}

From 4b0c8808c988da1de8274ef18ea03660167e9f2a Mon Sep 17 00:00:00 2001
From: Stephen Smoogen <smooge@redhat.com>
Date: Wed, 15 Feb 2017 19:30:16 +0000
Subject: [PATCH 30/38] add in our cron and ansible rules

---
 roles/web-data-analysis/files/condense-hotspot.cron | 1 +
 roles/web-data-analysis/tasks/main.yml              | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)
 create mode 100644 roles/web-data-analysis/files/condense-hotspot.cron

diff --git a/roles/web-data-analysis/files/condense-hotspot.cron b/roles/web-data-analysis/files/condense-hotspot.cron
new file mode 100644
index 0000000000..b4fe6d07c1
--- /dev/null
+++ b/roles/web-data-analysis/files/condense-hotspot.cron
@@ -0,0 +1 @@
+0 07  * * * root /usr/local/bin/condense-hotspot.sh
diff --git a/roles/web-data-analysis/tasks/main.yml b/roles/web-data-analysis/tasks/main.yml
index 81cf5b1e8a..fedd326c07 100644
--- a/roles/web-data-analysis/tasks/main.yml
+++ b/roles/web-data-analysis/tasks/main.yml
@@ -39,7 +39,7 @@
 
 - name: scripts to condense data down for further processing
   copy: src={{item}} dest=/usr/local/bin/ mode=0755
-  with_items: [condense-mirrorlogs.sh, condense-getfedoralogs.sh, run-daily-awstats.sh]
+  with_items: [condense-mirrorlogs.sh, condense-getfedoralogs.sh, run-daily-awstats.sh, condense-hotspot.sh ]
   tags:
     - web-data
 
@@ -51,13 +51,13 @@
 
 - name: awk files for csv creation
   copy: src={{item}} dest=/usr/local/share/web-data-analysis mode=0644
-  with_items: [mirrors-data.awk, getfedora-data.awk, getfedora.awk]
+  with_items: [mirrors-data.awk, getfedora-data.awk, getfedora.awk, hotspot-data.awk, hotspot.awk ]
   tags:
     - web-data
 
 - name: gnuplot file for image creation
   copy: src={{item}} dest=/usr/local/share/web-data-analysis mode=0644
-  with_items: [ mirrors-data.gp, getfedora-data.gp, mirrors-moving.gp ]
+  with_items: [ mirrors-data.gp, getfedora-data.gp, mirrors-moving.gp, hotspot.gp ]
   tags:
     - web-data
 

From 87117e1f3a44ca6a0c3d2e7daea8e261768950c6 Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Wed, 15 Feb 2017 19:51:28 +0000
Subject: [PATCH 31/38] Not the fqdn.  Just the first part.

---
 roles/mbs/common/templates/mbs-fedmsg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/mbs/common/templates/mbs-fedmsg.py b/roles/mbs/common/templates/mbs-fedmsg.py
index 3715f21b0e..1d8c824ba0 100644
--- a/roles/mbs/common/templates/mbs-fedmsg.py
+++ b/roles/mbs/common/templates/mbs-fedmsg.py
@@ -3,5 +3,5 @@ import socket
 config = {
     # So that the MBS can find it's cert in /etc/fedmsg.d/ssl.py
     'cert_prefix': 'mbs',
-    'name': 'mbs-%s' % socket.gethostname(),
+    'name': 'mbs.%s' % socket.gethostname().split('.', 1)[0],
 }

From 14a85f11f51cc4e27d5fb0407b93bd4bb701ea0a Mon Sep 17 00:00:00 2001
From: Stephen Smoogen <smooge@redhat.com>
Date: Wed, 15 Feb 2017 19:51:59 +0000
Subject: [PATCH 32/38] ok lets get this working

---
 roles/web-data-analysis/tasks/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/web-data-analysis/tasks/main.yml b/roles/web-data-analysis/tasks/main.yml
index fedd326c07..8afb91f780 100644
--- a/roles/web-data-analysis/tasks/main.yml
+++ b/roles/web-data-analysis/tasks/main.yml
@@ -45,7 +45,7 @@
 
 - name: python scripts to calculate various data
   copy: src={{item}} dest=/usr/local/bin/ mode=0755
-  with_items: [mirrorlist.py, mirrors-moving_avg.py]
+  with_items: [mirrorlist.py, mirrors-moving_avg.py, hotspot-moving_avg.py]
   tags:
     - web-data
 

From c9f72584af9af793f2886e7028710f87d6d2c0bb Mon Sep 17 00:00:00 2001
From: Stephen Smoogen <smooge@redhat.com>
Date: Wed, 15 Feb 2017 19:57:02 +0000
Subject: [PATCH 33/38] try this fix

---
 .../web-data-analysis/files/hotspot-data.awk  | 29 +++++++------------
 1 file changed, 11 insertions(+), 18 deletions(-)

diff --git a/roles/web-data-analysis/files/hotspot-data.awk b/roles/web-data-analysis/files/hotspot-data.awk
index 496def828e..244d616898 100644
--- a/roles/web-data-analysis/files/hotspot-data.awk
+++ b/roles/web-data-analysis/files/hotspot-data.awk
@@ -4,37 +4,30 @@ BEGIN{
   sum=0;
   most=0;
   least=0
+  print date ",AVG,LEAST,MAX"
 }
 
 {
   newdate=strftime("%F",$1);                # convert this to a printable date
-  if (date == strftime("%F",0)){            # we hit a min time and need to just print same stuff.
-    print date ",AVG,LEAST,MAX"
+  if (date != newdate){
+    print date "," int(sum/count) "," least "," most;
     date=newdate;
     count=1;    # start count to 0. we should have 288 per day but logs are stupid
     sum=$2;     # start the sum
     most=$2;    # what is going to be our most per day
     least=$2;   # what is going to be our least per day
   } else {
-    if (date != newdate){
-      print date "," int(sum/count) "," least "," most;
-      date=newdate;
-      count=1;    # start count to 0. we should have 288 per day but logs are stupid
-      sum=$2;     # start the sum
-      most=$2;    # what is going to be our most per day
-      least=$2;   # what is going to be our least per day
-    } else {
-      count=count+1;
-      sum=sum+$2;
-      if ($2 > most){
-	most=$2;
-      };
-      if ($2 < least) {
-	least=$2;
-      }
+    count=count+1;
+    sum=sum+$2;
+    if ($2 > most){
+      most=$2;
+    };
+    if ($2 < least) {
+      least=$2;
     }
   }
 }
+
 END{
   print date "," int(sum/count) "," least "," most;
 }

From 9a200b3bf6e7698d5cefcccb41da6ee527258d73 Mon Sep 17 00:00:00 2001
From: Stephen Smoogen <smooge@redhat.com>
Date: Wed, 15 Feb 2017 20:33:24 +0000
Subject: [PATCH 34/38] this is what happens when you cut and this is what
 happens paste

---
 roles/web-data-analysis/files/condense-hotspot.sh   | 4 ++--
 roles/web-data-analysis/files/hotspot-moving_avg.py | 6 ------
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/roles/web-data-analysis/files/condense-hotspot.sh b/roles/web-data-analysis/files/condense-hotspot.sh
index 17f3fcb2a7..95784984d9 100644
--- a/roles/web-data-analysis/files/condense-hotspot.sh
+++ b/roles/web-data-analysis/files/condense-hotspot.sh
@@ -74,10 +74,10 @@ fi
 # stuck and you end up with days or weeks of data in a single
 # file. Because the data is pretty small we can get away with adding up the data every day.
 
-find ${WORKEDIR} -type f | grep raw- | xargs cat  | sort -u | awk 'BEGIN{x=0; y=0}; {if (x != $1){ print x,y; x=$1; y=$2} else {y=y+$2}}' > ${WORKEDIR}/worked-all
+find ${WORKDIR} -type f | grep raw- | xargs cat  | sort -u | awk 'BEGIN{x=0; y=0}; {if (x != $1){ print x,y; x=$1; y=$2} else {y=y+$2}}' > ${WORKDIR}/worked-all
 
 
-awk -f ${LSHARE}/${PROJECT}-data.awk ${WORKEDIR}/worked-all | sort -u > ${WEBDIR}/${PROJECT}data-all.csv
+awk -f ${LSHARE}/${PROJECT}-data.awk ${WORKDIR}/worked-all | sort -u > ${WEBDIR}/${PROJECT}data-all.csv
 
 # Make the seven day moving average file
 /usr/local/bin/hotspot-moving_avg.py > ${WEBDIR}/${PROJECT}data-all-7day-ma.csv
diff --git a/roles/web-data-analysis/files/hotspot-moving_avg.py b/roles/web-data-analysis/files/hotspot-moving_avg.py
index 1489df62c9..35f1ce3da0 100644
--- a/roles/web-data-analysis/files/hotspot-moving_avg.py
+++ b/roles/web-data-analysis/files/hotspot-moving_avg.py
@@ -26,12 +26,6 @@ import pandas
 #import matplotlib.pyplot as plt
 import math
 
-# Moving Average
-
-import pandas
-import matplotlib.pyplot as plt
-import math
-
 rolling = 7
 
 tree = {}

From da7c4b9456ac0d8c8886cd943c46b1b3bb2f7541 Mon Sep 17 00:00:00 2001
From: Stephen Smoogen <smooge@redhat.com>
Date: Wed, 15 Feb 2017 20:40:09 +0000
Subject: [PATCH 35/38] put the files where the files want to be

---
 roles/web-data-analysis/files/hotspot.gp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/roles/web-data-analysis/files/hotspot.gp b/roles/web-data-analysis/files/hotspot.gp
index 2b4d8c9c5c..dc17471de5 100644
--- a/roles/web-data-analysis/files/hotspot.gp
+++ b/roles/web-data-analysis/files/hotspot.gp
@@ -7,7 +7,7 @@ set datafile separator ","
 set term png size 1600,1200
 
 ##
-set output "hotspot-all.png"
+set output "/var/www/html/csv-reports/images/hotspot-all.png"
 set title "IPs grabbing hotspot per day"
 plot ["2014-12-01":"2017-12-31"] \
      '/var/www/html/csv-reports/hotspot/hotspotdata-all.csv' using 1:2 title 'Average every 5min' with lines lw 4, \
@@ -16,7 +16,7 @@ plot ["2014-12-01":"2017-12-31"] \
 unset output
 
 ##
-set output "hotspot-all-ma.png"
+set output "/var/www/html/csv-reports/images/hotspot-all-ma.png"
 set title "Moving Average of IPs grabbing hotspot"
 plot ["2014-12-01":"2017-12-31"] \
      '/var/www/html/csv-reports/hotspot/hotspotdatadata-all-7day-ma.csv' using 1:2 title 'Average every 5min' with lines lw 4, \

From 4ab43e5ffecfe357fb880f4a1d19c7517d329267 Mon Sep 17 00:00:00 2001
From: Stephen Smoogen <smooge@redhat.com>
Date: Wed, 15 Feb 2017 20:46:59 +0000
Subject: [PATCH 36/38] well lets try some dates

---
 roles/web-data-analysis/files/hotspot-data.awk      | 4 ++--
 roles/web-data-analysis/files/hotspot-moving_avg.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/roles/web-data-analysis/files/hotspot-data.awk b/roles/web-data-analysis/files/hotspot-data.awk
index 244d616898..3e9a029477 100644
--- a/roles/web-data-analysis/files/hotspot-data.awk
+++ b/roles/web-data-analysis/files/hotspot-data.awk
@@ -1,10 +1,10 @@
 BEGIN{
-  date=strftime("%F",0);
+  date=strftime("%F",172800);
   count=1;
   sum=0;
   most=0;
   least=0
-  print date ",AVG,LEAST,MAX"
+  print "1970-01-01,AVG,LEAST,MAX"
 }
 
 {
diff --git a/roles/web-data-analysis/files/hotspot-moving_avg.py b/roles/web-data-analysis/files/hotspot-moving_avg.py
index 35f1ce3da0..13929fa481 100644
--- a/roles/web-data-analysis/files/hotspot-moving_avg.py
+++ b/roles/web-data-analysis/files/hotspot-moving_avg.py
@@ -30,7 +30,7 @@ rolling = 7
 
 tree = {}
 
-df = pandas.read_csv("hotspot-new.csv")
+df = pandas.read_csv("/var/www/html/csv-reports/hotspot/hotspotdata-all.csv")
 
 dates = df['1970-01-01']
 AVG   = pandas.rolling_mean(df['AVG'],rolling)

From 95605fa454b4bafcf5ac4013331959f76873a5a1 Mon Sep 17 00:00:00 2001
From: Ralph Bean <rbean@redhat.com>
Date: Wed, 15 Feb 2017 20:47:01 +0000
Subject: [PATCH 37/38] Tell MBS to not try and obtain a token from PDC (no
 need).

---
 roles/mbs/common/templates/config.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/roles/mbs/common/templates/config.py b/roles/mbs/common/templates/config.py
index d245d456d0..fb6bb81082 100644
--- a/roles/mbs/common/templates/config.py
+++ b/roles/mbs/common/templates/config.py
@@ -98,8 +98,10 @@ class ProdConfiguration(BaseConfiguration):
     LOG_LEVEL = 'debug'
     LOG_BACKEND = 'console'
 
+    # Yes, use tls.
     PDC_INSECURE = False
-    PDC_DEVELOP = False
+    # No, don't try to obtain a token (we just read.  we don't write.)
+    PDC_DEVELOP = True
 
     KOJI_CONFIG = path.join(confdir, 'koji.conf')
 {% if env == 'staging' %}

From 69368f53ba2b4e788d6b581e4ffb1097cea556e6 Mon Sep 17 00:00:00 2001
From: Stephen Smoogen <smooge@redhat.com>
Date: Wed, 15 Feb 2017 20:57:17 +0000
Subject: [PATCH 38/38] sometimes logs give you 0 time

---
 roles/web-data-analysis/files/condense-hotspot.sh | 2 +-
 roles/web-data-analysis/files/hotspot-data.awk    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/roles/web-data-analysis/files/condense-hotspot.sh b/roles/web-data-analysis/files/condense-hotspot.sh
index 95784984d9..1f5976c4b9 100644
--- a/roles/web-data-analysis/files/condense-hotspot.sh
+++ b/roles/web-data-analysis/files/condense-hotspot.sh
@@ -77,7 +77,7 @@ fi
 find ${WORKDIR} -type f | grep raw- | xargs cat  | sort -u | awk 'BEGIN{x=0; y=0}; {if (x != $1){ print x,y; x=$1; y=$2} else {y=y+$2}}' > ${WORKDIR}/worked-all
 
 
-awk -f ${LSHARE}/${PROJECT}-data.awk ${WORKDIR}/worked-all | sort -u > ${WEBDIR}/${PROJECT}data-all.csv
+awk -f ${LSHARE}/${PROJECT}-data.awk ${WORKDIR}/worked-all | grep -v "1970-01-01,0,0,0" | sort -u > ${WEBDIR}/${PROJECT}data-all.csv
 
 # Make the seven day moving average file
 /usr/local/bin/hotspot-moving_avg.py > ${WEBDIR}/${PROJECT}data-all-7day-ma.csv
diff --git a/roles/web-data-analysis/files/hotspot-data.awk b/roles/web-data-analysis/files/hotspot-data.awk
index 3e9a029477..93ec3d3d46 100644
--- a/roles/web-data-analysis/files/hotspot-data.awk
+++ b/roles/web-data-analysis/files/hotspot-data.awk
@@ -1,5 +1,5 @@
 BEGIN{
-  date=strftime("%F",172800);
+  date=strftime("%F",86401);
   count=1;
   sum=0;
   most=0;