diff --git a/inventory/group_vars/hotness b/inventory/group_vars/hotness
index 0cb42a9d74..8844ef65fd 100644
--- a/inventory/group_vars/hotness
+++ b/inventory/group_vars/hotness
@@ -15,7 +15,6 @@ sudoers: "{{ private }}/files/sudo/hotness-sudoers"
# These people get told when something goes wrong.
fedmsg_error_recipients:
-- rbean@redhat.com
- pingou@fedoraproject.org
- phracek@redhat.com
- thozza@redhat.com
diff --git a/inventory/group_vars/hotness-stg b/inventory/group_vars/hotness-stg
index a6938f282e..3e8ed20de2 100644
--- a/inventory/group_vars/hotness-stg
+++ b/inventory/group_vars/hotness-stg
@@ -15,7 +15,6 @@ sudoers: "{{ private }}/files/sudo/hotness-sudoers"
# These people get told when something goes wrong.
fedmsg_error_recipients:
-- rbean@redhat.com
- pingou@fedoraproject.org
- phracek@redhat.com
- thozza@redhat.com
diff --git a/inventory/group_vars/mbs-frontend b/inventory/group_vars/mbs-frontend
index 34e3981f0a..b84a047deb 100644
--- a/inventory/group_vars/mbs-frontend
+++ b/inventory/group_vars/mbs-frontend
@@ -24,8 +24,8 @@ fas_client_groups: sysadmin-noc,sysadmin-releng
# These are consumed by a task in roles/fedmsg/base/main.yml
fedmsg_certs:
- service: mbs
- owner: root
- group: apache
+ owner: fedmsg
+ group: fedmsg
can_send:
- mbs.module.state.change
# Only the backend sends this message..
diff --git a/inventory/group_vars/mbs-frontend-stg b/inventory/group_vars/mbs-frontend-stg
index 1c104763c7..0b692933c7 100644
--- a/inventory/group_vars/mbs-frontend-stg
+++ b/inventory/group_vars/mbs-frontend-stg
@@ -24,8 +24,8 @@ fas_client_groups: sysadmin-noc,sysadmin-releng
# These are consumed by a task in roles/fedmsg/base/main.yml
fedmsg_certs:
- service: mbs
- owner: root
- group: apache
+ owner: fedmsg
+ group: fedmsg
can_send:
- mbs.module.state.change
# Only the backend sends this message..
diff --git a/playbooks/clear_memcached.yml b/playbooks/clear_memcached.yml
index fe913c3656..eaae858dad 100644
--- a/playbooks/clear_memcached.yml
+++ b/playbooks/clear_memcached.yml
@@ -1,5 +1,5 @@
- name: clear memcache
- hosts: memcached
+ hosts: memcached:memcached-stg
serial: 1
tasks:
diff --git a/roles/cgit/clean_lock_cron/tasks/main.yml b/roles/cgit/clean_lock_cron/tasks/main.yml
index be680c416d..9a189f65c0 100644
--- a/roles/cgit/clean_lock_cron/tasks/main.yml
+++ b/roles/cgit/clean_lock_cron/tasks/main.yml
@@ -3,3 +3,4 @@
- name: install the cron file
copy: src=clean-lock.cron dest=/etc/cron.d/cgit-clean-lock.cron mode=0644
+ tags: cgit
diff --git a/roles/cgit/make_pkgs_list/files/make-cgit-pkgs-list.sh b/roles/cgit/make_pkgs_list/files/make-cgit-pkgs-list.sh
index bfc306186f..3e2b6f814a 100644
--- a/roles/cgit/make_pkgs_list/files/make-cgit-pkgs-list.sh
+++ b/roles/cgit/make_pkgs_list/files/make-cgit-pkgs-list.sh
@@ -8,14 +8,21 @@
newfile=`mktemp`
target=/srv/git/repositories
+# These are the pagure folders that we don't want to bother showing in cgit (it
+# makes things too slow...)
+blacklist='forks tickets docs requests'
+
for d in `ls $target`; do
- if [ ! -L $target/$d ] && [ -d $target/$d ]; then
+ # If it's not a link, it is a directory, and it's not in the blacklist..
+ if [ ! -L $target/$d ] && [ -d $target/$d ] && [[ ! $blacklist == *"$d"* ]]; then
+ # Then take every file inside and stuff it into our tmpfile.
for f in `ls $target/$d/`; do
echo "$d/$f" >> $newfile;
done;
fi;
done;
+# When we're done with everything in $target, make that avail to cgit.
mv -Z $newfile /srv/git/pkgs-git-repos-list
chown apache:apache /srv/git/pkgs-git-repos-list
chmod 644 /srv/git/pkgs-git-repos-list
diff --git a/roles/cgit/make_pkgs_list/tasks/main.yml b/roles/cgit/make_pkgs_list/tasks/main.yml
index 061a9e1886..a7e0d832c5 100644
--- a/roles/cgit/make_pkgs_list/tasks/main.yml
+++ b/roles/cgit/make_pkgs_list/tasks/main.yml
@@ -3,20 +3,25 @@
- name: install semanage
yum: pkg=/usr/sbin/semanage state=present
+ tags: cgit
- name: create the git root directory (/srv/git)
file: dest=/srv/git state=directory mode=0755
+ tags: cgit
- name: ensure the repo list file exists
copy: content="" dest=/srv/git/pkgs-git-repos-list force=no owner=apache group=apache mode=0644
+ tags: cgit
- name: install the script
copy: src=make-cgit-pkgs-list.sh dest=/usr/local/bin/make-cgit-pkgs-list.sh mode=0755
when: inventory_hostname.startswith('pkgs')
+ tags: cgit
- name: install the script for fedorapeople
copy: src=make-people-git.sh dest=/usr/local/bin/make-people-git.sh mode=0755
when: inventory_hostname.startswith('people')
+ tags: cgit
#
# For the batcave we just have a small static list
@@ -24,10 +29,12 @@
- name: install the file for batcave
copy: src=cgit-projects-batcave dest=/etc/cgit-projects-batcave mode=0644
when: inventory_hostname.startswith('batcave')
+ tags: cgit
- name: install the file for infinote
copy: src=cgit-projects-infinote dest=/etc/cgit-projects-infinote mode=0644
when: inventory_hostname.startswith('infinote')
+ tags: cgit
- name: install the cron job
cron: >
@@ -36,6 +43,7 @@
user=root
job="/usr/local/bin/lock-wrapper make-cgit-pkgs-list '/usr/local/bin/make-cgit-pkgs-list.sh | /usr/local/bin/nag-once make-cgit-pkgs-list 1d 2>&1'"
when: inventory_hostname.startswith('pkgs')
+ tags: cgit
- name: install the cron job for fedora people cgit
cron: >
@@ -44,6 +52,7 @@
user=root
job="/usr/local/bin/lock-wrapper make-people-git '/usr/local/bin/make-people-git.sh | /usr/local/bin/nag-once make-people-git 1d 2>&1'"
when: inventory_hostname.startswith('people')
+ tags: cgit
- name: check the selinux context of the repo list
command: matchpathcon /srv/git/pkgs-git-repos-list
diff --git a/roles/check-compose/files/checkcomp_consumer.py b/roles/check-compose/files/checkcomp_consumer.py
deleted file mode 100644
index 5d3068cffb..0000000000
--- a/roles/check-compose/files/checkcomp_consumer.py
+++ /dev/null
@@ -1,3 +0,0 @@
-config = {
- 'checkcomp_consumer.prod.enabled': True,
-}
diff --git a/roles/check-compose/tasks/main.yml b/roles/check-compose/tasks/main.yml
index 0d91c4833b..7c8e52b8bd 100644
--- a/roles/check-compose/tasks/main.yml
+++ b/roles/check-compose/tasks/main.yml
@@ -78,7 +78,7 @@
- restart fedmsg-hub
- name: Enable fedmsg consumer
- copy: src=checkcomp_consumer.py dest=/etc/fedmsg.d/checkcomp_consumer.py owner=root group=root mode=0644
+ template: src=checkcomp_consumer.py.j2 dest=/etc/fedmsg.d/checkcomp_consumer.py owner=root group=root mode=0644
notify:
- restart fedmsg-hub
tags:
diff --git a/roles/check-compose/templates/checkcomp_consumer.py.j2 b/roles/check-compose/templates/checkcomp_consumer.py.j2
new file mode 100644
index 0000000000..d5af64308a
--- /dev/null
+++ b/roles/check-compose/templates/checkcomp_consumer.py.j2
@@ -0,0 +1,9 @@
+config = {
+{% if deployment_type is defined and deployment_type == 'prod' %}
+ 'checkcomp_consumer.prod.enabled': True,
+{% elif deployment_type is defined and deployment_type == 'stg' %}
+ 'checkcomp_consumer.stg.enabled': True,
+{% else %}
+ checkcomp_consumer.test.enabled': True,
+{% endif %}
+}
diff --git a/roles/copr/backend/files/provision/files/mock/site-defaults.cfg b/roles/copr/backend/files/provision/files/mock/site-defaults.cfg
index 399126e6fd..d8491025e5 100644
--- a/roles/copr/backend/files/provision/files/mock/site-defaults.cfg
+++ b/roles/copr/backend/files/provision/files/mock/site-defaults.cfg
@@ -14,14 +14,12 @@ config_opts['plugin_conf']['compress_logs_enable'] = True
config_opts['plugin_conf']['compress_logs_opts'] = {}
config_opts['plugin_conf']['compress_logs_opts']['command'] = "/usr/bin/gzip -9 --force"
-{% if ansible_architecture == 'x86_64' %}
config_opts['plugin_conf']['tmpfs_enable'] = True
config_opts['plugin_conf']['tmpfs_opts'] = {}
config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024
config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = '75g'
config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755'
config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = False
-{% endif %}
{% if ansible_distribution == 'Fedora' and ansible_distribution_major_version|int > 21 %}
config_opts['yum_command'] = '/usr/bin/yum-deprecated'
diff --git a/roles/copr/backend/files/provision/provision_builder_tasks_ppc64le.yml b/roles/copr/backend/files/provision/provision_builder_tasks_ppc64le.yml
index 9effa0fbcb..1b2e89dc0d 100644
--- a/roles/copr/backend/files/provision/provision_builder_tasks_ppc64le.yml
+++ b/roles/copr/backend/files/provision/provision_builder_tasks_ppc64le.yml
@@ -4,9 +4,6 @@
- name: set bigger timeout for yum
ini_file: dest=/etc/yum.conf section=main option=timeout value=1000
-- name: install distribution-gpg-keys which are right now not in fedora main
- shell: dnf install -y https://kojipkgs.fedoraproject.org//packages/distribution-gpg-keys/1.5/1.fc24/noarch/distribution-gpg-keys-1.5-1.fc24.noarch.rpm
-
- name: install pkgs
yum: state=present pkg={{ item }}
with_items:
@@ -63,6 +60,7 @@
- name: symlink F26 configs to rawhide ones
shell: for cfg in /etc/mock/fedora-rawhide-*.cfg; do ln -s $cfg `echo $cfg|sed 's/rawhide/26/'`; done
+ when: prepare_base_image is defined
# ansible doesn't support simultaneously usage of async and with_* options
# it's not even planned for implementation, see https://github.com/ansible/ansible/issues/5841
diff --git a/roles/copr/backend/templates/provision/nova_cloud_vars_ppc64le.yml b/roles/copr/backend/templates/provision/nova_cloud_vars_ppc64le.yml
index f7a7da1cfc..2a65936f9e 100644
--- a/roles/copr/backend/templates/provision/nova_cloud_vars_ppc64le.yml
+++ b/roles/copr/backend/templates/provision/nova_cloud_vars_ppc64le.yml
@@ -11,7 +11,7 @@ OS_PASSWORD_OLD: "{{ copr_nova_password|default('variable OS_PASSWORD_OLD is und
OS_PASSWORD: "{{ copr_password|default('variable OS_PASSWORD is undefined')}}"
-image_name: "builder-f24-ppc64le"
+image_name: "builder-f24-ppc64le-swapmounted"
flavor_name: "{{ copr_builder_flavor_name }}"
network_name: "{{ copr_builder_network_name }}"
key_name: "{{ copr_builder_key_name }}"
diff --git a/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf b/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf
index 149fd04d43..8fd2808ce4 100644
--- a/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf
+++ b/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf
@@ -3,9 +3,8 @@ RequestHeader set X-Scheme https early
RequestHeader set X-Forwarded-Proto https early
ProxyPreserveHost On
-{% if env == "maybe-in-staging" %}
-RewriteRule ^/v2/([a-zA-Z]*)/blobs/([a-zA-Z]*) https://ourregistry/v2/$1/blobs/$2 [R]
-{% endif %}
+RewriteCond %{HTTP:VIA} !cdn77
+RewriteRule ^/v2/(.*)/blobs/([a-zA-Z0-9:]*) https://cdn.registry.fedoraproject.org/v2/$1/blobs/$2 [R]
# This is terible, but Docker.
RewriteEngine on
diff --git a/roles/mbs/common/tasks/main.yml b/roles/mbs/common/tasks/main.yml
index 5a1126d3bb..7e46835ff8 100644
--- a/roles/mbs/common/tasks/main.yml
+++ b/roles/mbs/common/tasks/main.yml
@@ -15,6 +15,15 @@
- mbs
- mbs/common
+- name: kill development config
+ file: path=/etc/fedmsg.d/module_build_service.py state=absent
+ notify:
+ - restart apache
+ - restart fedmsg-hub
+ tags:
+ - mbs
+ - mbs/common
+
- name: copy app configuration
template: >
src=config.py dest=/etc/module-build-service/config.py
@@ -26,10 +35,22 @@
- mbs
- mbs/common
-- name: copy client secrets that aren't really secret.
+- name: copy fedmsg configuration
+ template: >
+ src=mbs-fedmsg.py dest=/etc/fedmsg.d/mbs-fedmsg.py
+ owner=root group=fedmsg mode=0644
+ notify:
+ - restart apache
+ - restart fedmsg-hub
+ tags:
+ - mbs
+ - mbs/common
+
+- name: copy client secrets
template: >
src=client_secrets.json.{{env}} dest=/etc/module-build-service/client_secrets.json
- owner=root group=root mode=0644
+ owner=root group=fedmsg mode=0640
+ when: inventory_hostname.startswith('mbs-frontend')
notify:
- restart apache
tags:
diff --git a/roles/mbs/common/templates/client_secrets.json.production b/roles/mbs/common/templates/client_secrets.json.production
index e25c1b49a8..9b12f4cc78 100644
--- a/roles/mbs/common/templates/client_secrets.json.production
+++ b/roles/mbs/common/templates/client_secrets.json.production
@@ -1,12 +1,11 @@
{
"web": {
"auth_uri": "https://id.fedoraproject.org/openidc/Authorization",
- "client_id": "mbs-authorizer",
- "client_secret": "notsecret",
- "redirect_uris": [
- "http://localhost:13747/"
- ],
+ "client_id": "mbs-prod",
+ "client_secret": "{{ mbs_prod_oidc_client_secret }}",
+ "redirect_uris": [],
"token_uri": "https://id.fedoraproject.org/openidc/Token",
- "token_introspection_uri": "https://id.fedoraproject.org/openidc/TokenInfo"
+ "token_introspection_uri": "https://id.fedoraproject.org/openidc/TokenInfo",
+ "userinfo_uri": "https://id.fedoraproject.org/openidc/UserInfo"
}
}
diff --git a/roles/mbs/common/templates/client_secrets.json.staging b/roles/mbs/common/templates/client_secrets.json.staging
index 7fd5069dae..f78371caf8 100644
--- a/roles/mbs/common/templates/client_secrets.json.staging
+++ b/roles/mbs/common/templates/client_secrets.json.staging
@@ -1,12 +1,11 @@
{
"web": {
"auth_uri": "https://id.stg.fedoraproject.org/openidc/Authorization",
- "client_id": "mbs-authorizer",
- "client_secret": "notsecret",
- "redirect_uris": [
- "http://localhost:13747/"
- ],
+ "client_id": "mbs-stg",
+ "client_secret": "{{ mbs_stg_oidc_client_secret }}",
+ "redirect_uris": [],
"token_uri": "https://id.stg.fedoraproject.org/openidc/Token",
- "token_introspection_uri": "https://id.stg.fedoraproject.org/openidc/TokenInfo"
+ "token_introspection_uri": "https://id.stg.fedoraproject.org/openidc/TokenInfo",
+ "userinfo_uri": "https://id.stg.fedoraproject.org/openidc/UserInfo"
}
}
diff --git a/roles/mbs/common/templates/config.py b/roles/mbs/common/templates/config.py
index d245d456d0..fb6bb81082 100644
--- a/roles/mbs/common/templates/config.py
+++ b/roles/mbs/common/templates/config.py
@@ -98,8 +98,10 @@ class ProdConfiguration(BaseConfiguration):
LOG_LEVEL = 'debug'
LOG_BACKEND = 'console'
+ # Yes, use tls.
PDC_INSECURE = False
- PDC_DEVELOP = False
+ # No, don't try to obtain a token (we just read. we don't write.)
+ PDC_DEVELOP = True
KOJI_CONFIG = path.join(confdir, 'koji.conf')
{% if env == 'staging' %}
diff --git a/roles/mbs/common/templates/mbs-fedmsg.py b/roles/mbs/common/templates/mbs-fedmsg.py
new file mode 100644
index 0000000000..1d8c824ba0
--- /dev/null
+++ b/roles/mbs/common/templates/mbs-fedmsg.py
@@ -0,0 +1,7 @@
+import socket
+
+config = {
+ # So that the MBS can find it's cert in /etc/fedmsg.d/ssl.py
+ 'cert_prefix': 'mbs',
+ 'name': 'mbs.%s' % socket.gethostname().split('.', 1)[0],
+}
diff --git a/roles/mbs/frontend/tasks/main.yml b/roles/mbs/frontend/tasks/main.yml
index 1d24257955..761318c437 100644
--- a/roles/mbs/frontend/tasks/main.yml
+++ b/roles/mbs/frontend/tasks/main.yml
@@ -4,7 +4,7 @@
- name: disable the scheduler on the frontend
copy: >
src={{ item }} dest=/etc/fedmsg.d/{{ item }}
- owner=apache group=apache mode=0600
+ owner=fedmsg group=fedmsg mode=0644
with_items:
- mbs-scheduler.py
notify:
@@ -41,6 +41,7 @@
with_items:
- httpd_can_network_connect_db
- httpd_can_network_memcache
+ - httpd_can_network_connect
- httpd_can_sendmail
tags:
- mbs
diff --git a/roles/opendkim/files/opendkim.conf b/roles/opendkim/files/opendkim.conf
index 03353770c0..c50d1d1692 100644
--- a/roles/opendkim/files/opendkim.conf
+++ b/roles/opendkim/files/opendkim.conf
@@ -131,3 +131,7 @@ OversignHeaders From
## caching service. Useful if the nameserver being used by the filter is
## not local.
# QueryCache yes
+#
+# We need to increase the default header size because notifs adds a X-fedmsg header for
+# each message in a digest.
+MaximumHeaders 262144
diff --git a/roles/openqa/dispatcher/tasks/main.yml b/roles/openqa/dispatcher/tasks/main.yml
index da87403149..1a3fc0de22 100644
--- a/roles/openqa/dispatcher/tasks/main.yml
+++ b/roles/openqa/dispatcher/tasks/main.yml
@@ -16,8 +16,10 @@
# - wikitcms_password
## string - password for relval_user
# - deployment_type
-## string - Fedora Infrastructure thing; for this role, decides
-## whether and where to submit wiki results
+## string - Fedora Infrastructure thing; for this role, the
+## fedora_openqa config file will be set appropriately
+## for infra deployments if this is set, so don't set
+## it for private deployments
#
# When all of the above are set, a wikitcms 'credentials' file will
# be created and result submission to the wiki will be enabled. If
@@ -75,25 +77,29 @@
chdir: /root/openQA-python-client
when: "gitclient|changed or instclient.rc > 0"
-# We check this out to a different place from the 'server' task, because
-# otherwise it's hard to make sure we install each time it changes.
-- name: Check out openqa_fedora_tools
+- name: Remove old openqa_fedora_tools checkout
+ file: path=/root/openqa_fedora_tools-dispatcher state=absent
+
+- name: Remove old fedora-openqa-schedule script
+ file: path=/usr/bin/fedora-openqa-schedule state=absent
+
+- name: Check out fedora_openqa (scheduler / reporter tool)
git:
- repo: https://bitbucket.org/rajcze/openqa_fedora_tools
- dest: /root/openqa_fedora_tools-dispatcher
+ repo: https://pagure.io/fedora-qa/fedora_openqa.git
+ dest: /root/fedora_openqa
register: gittools
-- name: Check if openqa_fedora_tools has ever been installed
- stat: path=/usr/bin/fedora-openqa-schedule
+- name: Check if fedora_openqa has ever been installed
+ stat: path=/usr/bin/fedora-openqa
register: insttools
changed_when: "1 != 1"
failed_when: "1 != 1"
check_mode: no
-- name: Install openqa_fedora_tools
+- name: Install fedora_openqa
command: "python setup.py install"
args:
- chdir: /root/openqa_fedora_tools-dispatcher/scheduler
+ chdir: /root/fedora_openqa
when: "gittools|changed or not insttools.stat.exists"
notify:
- restart fedmsg-hub
@@ -103,7 +109,7 @@
tags:
- config
-- name: Create fedora-openqa-schedule config directory
+- name: Create fedora_openqa config directory
file: path=/etc/fedora-openqa state=directory owner=root group=root mode=0755
- name: Write schedule.conf
diff --git a/roles/openqa/dispatcher/templates/openqa_consumer.py.j2 b/roles/openqa/dispatcher/templates/openqa_consumer.py.j2
index 33219f72dc..d0b62d2282 100644
--- a/roles/openqa/dispatcher/templates/openqa_consumer.py.j2
+++ b/roles/openqa/dispatcher/templates/openqa_consumer.py.j2
@@ -1,13 +1,16 @@
config = {
{% if openqa_consumer %}
- 'fedora_openqa_schedule.consumer.enabled': True,
+ 'fedora_openqa.scheduler.prod.enabled': True,
{% endif %}
-{% if wikitcms_user is defined and wikitcms_password is defined %}
{% if deployment_type is defined and deployment_type == 'prod' %}
- 'fedora_openqa_schedule.wiki.consumer.prod.enabled': True,
- 'fedora_openqa_schedule.resultsdb.reporter.prod.enabled': True,
-{% elif deployment_type is defined and deployment_type == 'stg' %}
- 'fedora_openqa_schedule.wiki.consumer.stg.enabled': True,
+{% if wikitcms_user is defined and wikitcms_password is defined %}
+ 'fedora_openqa.reporter.wiki.prod.enabled': True,
{% endif %}
+ 'fedora_openqa.reporter.resultsdb.prod.enabled': True,
+{% elif deployment_type is defined and deployment_type == 'stg' %}
+{% if wikitcms_user is defined and wikitcms_password is defined %}
+ 'fedora_openqa.reporter.wiki.stg.enabled': True,
+{% endif %}
+ 'fedora_openqa.reporter.resultsdb.stg.enabled': True,
{% endif %}
}
diff --git a/roles/openqa/dispatcher/templates/schedule.conf.j2 b/roles/openqa/dispatcher/templates/schedule.conf.j2
index ab79ae819d..f599284ff0 100644
--- a/roles/openqa/dispatcher/templates/schedule.conf.j2
+++ b/roles/openqa/dispatcher/templates/schedule.conf.j2
@@ -1,14 +1,24 @@
[report]
-openqa_url: https://{{ external_hostname|default(ansible_nodename) }}
-{% if wikitcms_user is defined and wikitcms_password is defined %}
-submit_wiki: true
-submit_resultsdb: true
-{% else %}
-submit_wiki: false
-submit_resultsdb: false
-{% endif %}
{% if deployment_type is defined and deployment_type == 'prod' %}
resultsdb_url: http://resultsdb01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
+wiki_hostname: fedoraproject.org
+[consumers]
+prod_oqa_hostname: localhost
+prod_oqa_baseurl: https://{{ external_hostname|default(ansible_nodename) }}
+prod_wiki_hostname: fedoraproject.org
+prod_wiki_report: true
+prod_rdb_url: http://resultsdb01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
+prod_rdb_report: true
{% elif deployment_type is defined and deployment_type == 'stg' %}
resultsdb_url: http://resultsdb-stg01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
+wiki_hostname: stg.fedoraproject.org
+[consumers]
+# as we use the production scheduler
+prod_oqa_hostname: localhost
+stg_oqa_hostname: localhost
+stg_oqa_baseurl: https://{{ external_hostname|default(ansible_nodename) }}
+stg_wiki_hostname: stg.fedoraproject.org
+stg_wiki_report: true
+stg_rdb_url: http://resultsdb-stg01.qa.fedoraproject.org/resultsdb_api/api/v2.0/
+stg_rdb_report: true
{% endif %}
diff --git a/roles/openqa/server/files/createhdds b/roles/openqa/server/files/createhdds
index c76019d8b6..794d80450e 100644
--- a/roles/openqa/server/files/createhdds
+++ b/roles/openqa/server/files/createhdds
@@ -1,4 +1,4 @@
#!/bin/sh
cd /var/lib/openqa/share/factory/hdd/fixed
-LIBGUESTFS_BACKEND=direct withlock /var/lock/createhdds.lock /root/openqa_fedora_tools/tools/createhdds.py all --clean
+LIBGUESTFS_BACKEND=direct withlock /var/lock/createhdds.lock /root/createhdds/createhdds.py all --clean
diff --git a/roles/openqa/server/tasks/main.yml b/roles/openqa/server/tasks/main.yml
index 957290b6be..27dbe1a39c 100644
--- a/roles/openqa/server/tasks/main.yml
+++ b/roles/openqa/server/tasks/main.yml
@@ -120,11 +120,13 @@
become_user: geekotest
when: "(testsbranch.stdout.find('Not a git repository') != -1) or (testsbranch.stdout.find('On branch master') != -1)"
-- name: Check out openqa_fedora_tools
+- name: Remove old openqa_fedora_tools checkout
+ file: path=/root/openqa_fedora_tools state=absent
+
+- name: Check out createhdds
git:
- repo: https://bitbucket.org/rajcze/openqa_fedora_tools
- dest: /root/openqa_fedora_tools
- register: git_result
+ repo: https://pagure.io/fedora-qa/createhdds.git
+ dest: /root/createhdds
- name: Create asset directories
file: path={{ item }} state=directory owner=geekotest group=root mode=0755
@@ -139,7 +141,7 @@
copy: src=createhdds dest=/etc/cron.daily/createhdds owner=root group=root mode=0755
- name: Check if any hard disk images need (re)building
- command: "/root/openqa_fedora_tools/tools/createhdds.py check"
+ command: "/root/createhdds/createhdds.py check"
args:
chdir: /var/lib/openqa/share/factory/hdd/fixed
register: diskcheck
diff --git a/roles/pkgdb2/templates/pkgdb2.cfg b/roles/pkgdb2/templates/pkgdb2.cfg
index c033723ea9..bdb677d508 100644
--- a/roles/pkgdb2/templates/pkgdb2.cfg
+++ b/roles/pkgdb2/templates/pkgdb2.cfg
@@ -27,7 +27,9 @@ AUTOAPPROVE_PKGERS = [
#### FAS group for the pkgdb admins
{% if env == 'staging' %}
-ADMIN_GROUP = ['cvsadmin']
+# Factory 2 added to make it easier for us to create repos for the modularity
+# team. Only in staging. Feel free to remove!
+ADMIN_GROUP = ['cvsadmin', 'factory2']
{% else %}
ADMIN_GROUP = ['sysadmin-main', 'cvsadmin']
{% endif %}
diff --git a/roles/web-data-analysis/files/condense-hotspot.cron b/roles/web-data-analysis/files/condense-hotspot.cron
new file mode 100644
index 0000000000..b4fe6d07c1
--- /dev/null
+++ b/roles/web-data-analysis/files/condense-hotspot.cron
@@ -0,0 +1 @@
+0 07 * * * root /usr/local/bin/condense-hotspot.sh
diff --git a/roles/web-data-analysis/files/condense-hotspot.sh b/roles/web-data-analysis/files/condense-hotspot.sh
new file mode 100644
index 0000000000..1f5976c4b9
--- /dev/null
+++ b/roles/web-data-analysis/files/condense-hotspot.sh
@@ -0,0 +1,88 @@
+#!/bin/bash
+
+# This file is part of Fedora Project Infrastructure Ansible
+# Repository.
+#
+# Fedora Project Infrastructure Ansible Repository is free software:
+# you can redistribute it and/or modify it under the terms of the GNU
+# General Public License as published by the Free Software Foundation,
+# either version 3 of the License, or (at your option) any later
+# version.
+#
+# Fedora Project Infrastructure Ansible Repository is distributed in
+# the hope that it will be useful, but WITHOUT ANY WARRANTY; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fedora Project Infrastructure Ansible Repository. If
+# not, see .
+
+# There is a multiday delay involved in processing the logs. It
+# may take up to 4 days to get the logs to the main-server. It may
+# take a day to combine all the logs onto combined-httpd. So we assume
+# we are 5 days behind.
+
+let NUMDAYS=5
+let OLDDAYS=$(( $NUMDAYS+1 ))
+
+PROJECT=hotspot
+WEBLOG=fedoraproject.org
+
+# This is the year/month/day for a N days ago.
+YEAR=$(/bin/date -d "-${NUMDAYS} days" +%Y)
+MONTH=$(/bin/date -d "-${NUMDAYS} days" +%m)
+DAY=$(/bin/date -d "-${NUMDAYS} days" +%d)
+
+# And we have have to deal with year/month/day boundaries for our later grep.
+OLDDATE=$(/bin/date -d "-${OLDDAYS} days" +%Y-%m-%d)
+OLDYEAR=$(/bin/date -d "-${OLDDAYS} days" +%Y)
+
+NFSDIR=/mnt/fedora_stats/combined-http
+TARGET=${NFSDIR}/${YEAR}/${MONTH}/${DAY}
+
+LOGFILE=${TARGET}/${WEBLOG}-access.log
+
+WORKDIR=/mnt/fedora_stats/data/${PROJECT}
+WORKFILE=${WORKDIR}/${YEAR}/${MONTH}/raw-${DAY}
+
+WEBDIR=/var/www/html/csv-reports/${PROJECT}
+
+TEMPDIR=$( mktemp -d /tmp/web-data-analysis.XXXXXXXXX )
+
+LBIN=/usr/local/bin/
+LSHARE=/usr/local/share/web-data-analysis
+
+mkdir -p ${WORKDIR}/${YEAR}/${MONTH}
+if [[ ! -f ${WORKDIR}/${YEAR}/out-${YEAR}-${MONTH} ]]; then
+ touch ${WORKDIR}/${YEAR}/out-${YEAR}-${MONTH}
+fi
+
+if [[ ! -f ${WORKDIR}/out-${YEAR} ]]; then
+ touch ${WORKDIR}/out-${YEAR}
+fi
+
+if [[ ! -f ${LOGFILE} ]]; then
+ echo "No logfile found for ${YEAR}/${MONTH}/${DAY}. Please fix."
+else
+ awk -f ${LSHARE}/${PROJECT}.awk ${LOGFILE} > ${WORKFILE}
+fi
+
+# So the data isn't strictly across month boundries due to the end of
+# the logfiles being at 04:00 versus 23:59. Also log files might get
+# stuck and you end up with days or weeks of data in a single
+# file. Because the data is pretty small we can get away with adding up the data every day.
+
+find ${WORKDIR} -type f | grep raw- | xargs cat | sort -u | awk 'BEGIN{x=0; y=0}; {if (x != $1){ print x,y; x=$1; y=$2} else {y=y+$2}}' > ${WORKDIR}/worked-all
+
+
+awk -f ${LSHARE}/${PROJECT}-data.awk ${WORKDIR}/worked-all | grep -v "1970-01-01,0,0,0" | sort -u > ${WEBDIR}/${PROJECT}data-all.csv
+
+# Make the seven day moving average file
+/usr/local/bin/hotspot-moving_avg.py > ${WEBDIR}/${PROJECT}data-all-7day-ma.csv
+
+gnuplot ${LSHARE}/${PROJECT}.gp
+
+# cleanup the temp data
+rm -rf ${TEMPDIR}
diff --git a/roles/web-data-analysis/files/hotspot-data.awk b/roles/web-data-analysis/files/hotspot-data.awk
new file mode 100644
index 0000000000..93ec3d3d46
--- /dev/null
+++ b/roles/web-data-analysis/files/hotspot-data.awk
@@ -0,0 +1,33 @@
+BEGIN{
+ date=strftime("%F",86401);
+ count=1;
+ sum=0;
+ most=0;
+ least=0
+ print "1970-01-01,AVG,LEAST,MAX"
+}
+
+{
+ newdate=strftime("%F",$1); # convert this to a printable date
+ if (date != newdate){
+ print date "," int(sum/count) "," least "," most;
+ date=newdate;
+ count=1; # start count to 0. we should have 288 per day but logs are stupid
+ sum=$2; # start the sum
+ most=$2; # what is going to be our most per day
+ least=$2; # what is going to be our least per day
+ } else {
+ count=count+1;
+ sum=sum+$2;
+ if ($2 > most){
+ most=$2;
+ };
+ if ($2 < least) {
+ least=$2;
+ }
+ }
+}
+
+END{
+ print date "," int(sum/count) "," least "," most;
+}
diff --git a/roles/web-data-analysis/files/hotspot-moving_avg.py b/roles/web-data-analysis/files/hotspot-moving_avg.py
new file mode 100644
index 0000000000..13929fa481
--- /dev/null
+++ b/roles/web-data-analysis/files/hotspot-moving_avg.py
@@ -0,0 +1,50 @@
+#!/usr/bin/python
+
+# This file is part of Fedora Project Infrastructure Ansible
+# Repository.
+#
+# Fedora Project Infrastructure Ansible Repository is free software:
+# you can redistribute it and/or modify it under the terms of the GNU
+# General Public License as published by the Free Software Foundation,
+# either version 3 of the License, or (at your option) any later
+# version.
+#
+# Fedora Project Infrastructure Ansible Repository is distributed in
+# the hope that it will be useful, but WITHOUT ANY WARRANTY; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fedora Project Infrastructure Ansible Repository. If
+# not, see .
+
+# This is a complete horrible hack to get something done. Patches are
+# really welcome.
+
+import pandas
+#import matplotlib.pyplot as plt
+import math
+
+rolling = 7
+
+tree = {}
+
+df = pandas.read_csv("/var/www/html/csv-reports/hotspot/hotspotdata-all.csv")
+
+dates = df['1970-01-01']
+AVG = pandas.rolling_mean(df['AVG'],rolling)
+LEAST = pandas.rolling_mean(df['LEAST'],rolling)
+MAX = pandas.rolling_mean(df['MAX'],rolling)
+
+
+for i in xrange(0,len(dates)):
+ if math.isnan(MAX[i]):
+ csv_line = ",".join([dates[i],"0","0"])
+ else:
+ csv_line = ",".join([dates[i],
+ str(int(AVG[i])),
+ str(int(LEAST[i])),
+ str(int(MAX[i])),
+ ])
+ print csv_line
diff --git a/roles/web-data-analysis/files/hotspot.awk b/roles/web-data-analysis/files/hotspot.awk
new file mode 100644
index 0000000000..f47da6958e
--- /dev/null
+++ b/roles/web-data-analysis/files/hotspot.awk
@@ -0,0 +1,95 @@
+#
+# Take the apache log line
+# 83.163.161.147 - - [30/Sep/2012:13:54:19 +0000] "GET /static/hotspot.txt HTTP/1.1" 200 3 "-" "dnssec-trigger/0.11"
+# Convert to
+# 1349013000 1
+
+function convertdate(str) {
+ gsub(/\[/, "", str)
+ gsub(/\]/, "", str)
+ split(str,a,":");
+ split(a[1],b,"/");
+ temp="";
+ switch (b[2]) {
+ case "Jan":
+ temp="01"
+ break;
+ case "Feb":
+ temp="02"
+ break;
+ case "Mar":
+ temp="03"
+ break;
+ case "Apr":
+ temp="04"
+ break;
+ case "May":
+ temp="05"
+ break;
+ case "Jun":
+ temp="06"
+ break;
+ case "Jul":
+ temp="07"
+ break;
+ case "Aug":
+ temp="08"
+ break;
+ case "Sep":
+ temp="09"
+ break;
+ case "Oct":
+ temp="10"
+ break;
+ case "Nov":
+ temp="11"
+ break;
+ case "Dec":
+ temp="12"
+ break;
+ default:
+ temp="00"
+ break;
+ }
+ x=b[3]" "temp" "b[1]" "a[2]" "a[3] " "a[4]
+ y=int(mktime(x)/300) # 300 seconds make 5 minutes (I NEED A GLOBAL VAR)
+ return y
+}
+
+
+BEGIN{
+ timestamp=0;
+ num_ts = 0;
+ ts_hotspots=0;
+ total_hotsponts=0;
+}
+
+#
+# We assume that every 300 seconds a system will log in at least 1
+# time because the Networkmanager addon does so.
+# Convert our date stamp to the nearest 5 minute block and add data to
+# it. If the log file goes backwards or jumps etc this will mean
+# multiple outputs for a timestamp. A later process will need to deal
+# with that. All this will do is output how many it saw at that block
+# in the log file.
+#
+
+$7 ~/hotspot.txt/ && $6 ~/GET/ {
+ date = convertdate($4)
+ if (timestamp != date) {
+ num_ts = num_ts +1;
+ print (timestamp*300),ts_hotspots # GLOBAL VAR GOES HERE
+ timestamp = date;
+ ts_hotspots = 1;
+ } else {
+ ts_hotspots = ts_hotspots +1;
+ total_hotspots = total_hotspots +1;
+ }
+}
+
+END {
+ num_ts = num_ts +1;
+ print int(timestamp*300),ts_hotspots # LOOK GLOBAL VAR AGAIN
+}
+
+## END OF FILE
diff --git a/roles/web-data-analysis/files/hotspot.gp b/roles/web-data-analysis/files/hotspot.gp
new file mode 100644
index 0000000000..dc17471de5
--- /dev/null
+++ b/roles/web-data-analysis/files/hotspot.gp
@@ -0,0 +1,25 @@
+set grid
+set xdata time
+set format x "%Y-%m-%d"
+set timefmt "%Y-%m-%d"
+
+set datafile separator ","
+set term png size 1600,1200
+
+##
+set output "/var/www/html/csv-reports/images/hotspot-all.png"
+set title "IPs grabbing hotspot per day"
+plot ["2014-12-01":"2017-12-31"] \
+ '/var/www/html/csv-reports/hotspot/hotspotdata-all.csv' using 1:2 title 'Average every 5min' with lines lw 4, \
+ '/var/www/html/csv-reports/hotspot/hotspotdata-all.csv' using 1:3 title 'Least 5min' with lines lw 4, \
+ '/var/www/html/csv-reports/hotspot/hotspotdata-all.csv' using 1:4 title 'Max 5min' with lines lw 4
+unset output
+
+##
+set output "/var/www/html/csv-reports/images/hotspot-all-ma.png"
+set title "Moving Average of IPs grabbing hotspot"
+plot ["2014-12-01":"2017-12-31"] \
+ '/var/www/html/csv-reports/hotspot/hotspotdatadata-all-7day-ma.csv' using 1:2 title 'Average every 5min' with lines lw 4, \
+ '/var/www/html/csv-reports/hotspot/hotspotdatadata-all-7day-ma.csv' using 1:3 title 'Least 5min' with lines lw 4, \
+ '/var/www/html/csv-reports/hotspot/hotspotdatadata-all-7day-ma.csv' using 1:4 title 'Max 5min' with lines lw 4
+unset output
diff --git a/roles/web-data-analysis/tasks/main.yml b/roles/web-data-analysis/tasks/main.yml
index 81cf5b1e8a..8afb91f780 100644
--- a/roles/web-data-analysis/tasks/main.yml
+++ b/roles/web-data-analysis/tasks/main.yml
@@ -39,25 +39,25 @@
- name: scripts to condense data down for further processing
copy: src={{item}} dest=/usr/local/bin/ mode=0755
- with_items: [condense-mirrorlogs.sh, condense-getfedoralogs.sh, run-daily-awstats.sh]
+ with_items: [condense-mirrorlogs.sh, condense-getfedoralogs.sh, run-daily-awstats.sh, condense-hotspot.sh ]
tags:
- web-data
- name: python scripts to calculate various data
copy: src={{item}} dest=/usr/local/bin/ mode=0755
- with_items: [mirrorlist.py, mirrors-moving_avg.py]
+ with_items: [mirrorlist.py, mirrors-moving_avg.py, hotspot-moving_avg.py]
tags:
- web-data
- name: awk files for csv creation
copy: src={{item}} dest=/usr/local/share/web-data-analysis mode=0644
- with_items: [mirrors-data.awk, getfedora-data.awk, getfedora.awk]
+ with_items: [mirrors-data.awk, getfedora-data.awk, getfedora.awk, hotspot-data.awk, hotspot.awk ]
tags:
- web-data
- name: gnuplot file for image creation
copy: src={{item}} dest=/usr/local/share/web-data-analysis mode=0644
- with_items: [ mirrors-data.gp, getfedora-data.gp, mirrors-moving.gp ]
+ with_items: [ mirrors-data.gp, getfedora-data.gp, mirrors-moving.gp, hotspot.gp ]
tags:
- web-data