diff --git a/inventory/group_vars/hotness b/inventory/group_vars/hotness index 0cb42a9d74..8844ef65fd 100644 --- a/inventory/group_vars/hotness +++ b/inventory/group_vars/hotness @@ -15,7 +15,6 @@ sudoers: "{{ private }}/files/sudo/hotness-sudoers" # These people get told when something goes wrong. fedmsg_error_recipients: -- rbean@redhat.com - pingou@fedoraproject.org - phracek@redhat.com - thozza@redhat.com diff --git a/inventory/group_vars/hotness-stg b/inventory/group_vars/hotness-stg index a6938f282e..3e8ed20de2 100644 --- a/inventory/group_vars/hotness-stg +++ b/inventory/group_vars/hotness-stg @@ -15,7 +15,6 @@ sudoers: "{{ private }}/files/sudo/hotness-sudoers" # These people get told when something goes wrong. fedmsg_error_recipients: -- rbean@redhat.com - pingou@fedoraproject.org - phracek@redhat.com - thozza@redhat.com diff --git a/inventory/group_vars/mbs-frontend b/inventory/group_vars/mbs-frontend index 34e3981f0a..b84a047deb 100644 --- a/inventory/group_vars/mbs-frontend +++ b/inventory/group_vars/mbs-frontend @@ -24,8 +24,8 @@ fas_client_groups: sysadmin-noc,sysadmin-releng # These are consumed by a task in roles/fedmsg/base/main.yml fedmsg_certs: - service: mbs - owner: root - group: apache + owner: fedmsg + group: fedmsg can_send: - mbs.module.state.change # Only the backend sends this message.. diff --git a/inventory/group_vars/mbs-frontend-stg b/inventory/group_vars/mbs-frontend-stg index 1c104763c7..0b692933c7 100644 --- a/inventory/group_vars/mbs-frontend-stg +++ b/inventory/group_vars/mbs-frontend-stg @@ -24,8 +24,8 @@ fas_client_groups: sysadmin-noc,sysadmin-releng # These are consumed by a task in roles/fedmsg/base/main.yml fedmsg_certs: - service: mbs - owner: root - group: apache + owner: fedmsg + group: fedmsg can_send: - mbs.module.state.change # Only the backend sends this message.. diff --git a/playbooks/clear_memcached.yml b/playbooks/clear_memcached.yml index fe913c3656..eaae858dad 100644 --- a/playbooks/clear_memcached.yml +++ b/playbooks/clear_memcached.yml @@ -1,5 +1,5 @@ - name: clear memcache - hosts: memcached + hosts: memcached:memcached-stg serial: 1 tasks: diff --git a/roles/cgit/clean_lock_cron/tasks/main.yml b/roles/cgit/clean_lock_cron/tasks/main.yml index be680c416d..9a189f65c0 100644 --- a/roles/cgit/clean_lock_cron/tasks/main.yml +++ b/roles/cgit/clean_lock_cron/tasks/main.yml @@ -3,3 +3,4 @@ - name: install the cron file copy: src=clean-lock.cron dest=/etc/cron.d/cgit-clean-lock.cron mode=0644 + tags: cgit diff --git a/roles/cgit/make_pkgs_list/files/make-cgit-pkgs-list.sh b/roles/cgit/make_pkgs_list/files/make-cgit-pkgs-list.sh index bfc306186f..3e2b6f814a 100644 --- a/roles/cgit/make_pkgs_list/files/make-cgit-pkgs-list.sh +++ b/roles/cgit/make_pkgs_list/files/make-cgit-pkgs-list.sh @@ -8,14 +8,21 @@ newfile=`mktemp` target=/srv/git/repositories +# These are the pagure folders that we don't want to bother showing in cgit (it +# makes things too slow...) +blacklist='forks tickets docs requests' + for d in `ls $target`; do - if [ ! -L $target/$d ] && [ -d $target/$d ]; then + # If it's not a link, it is a directory, and it's not in the blacklist.. + if [ ! -L $target/$d ] && [ -d $target/$d ] && [[ ! $blacklist == *"$d"* ]]; then + # Then take every file inside and stuff it into our tmpfile. for f in `ls $target/$d/`; do echo "$d/$f" >> $newfile; done; fi; done; +# When we're done with everything in $target, make that avail to cgit. mv -Z $newfile /srv/git/pkgs-git-repos-list chown apache:apache /srv/git/pkgs-git-repos-list chmod 644 /srv/git/pkgs-git-repos-list diff --git a/roles/cgit/make_pkgs_list/tasks/main.yml b/roles/cgit/make_pkgs_list/tasks/main.yml index 061a9e1886..a7e0d832c5 100644 --- a/roles/cgit/make_pkgs_list/tasks/main.yml +++ b/roles/cgit/make_pkgs_list/tasks/main.yml @@ -3,20 +3,25 @@ - name: install semanage yum: pkg=/usr/sbin/semanage state=present + tags: cgit - name: create the git root directory (/srv/git) file: dest=/srv/git state=directory mode=0755 + tags: cgit - name: ensure the repo list file exists copy: content="" dest=/srv/git/pkgs-git-repos-list force=no owner=apache group=apache mode=0644 + tags: cgit - name: install the script copy: src=make-cgit-pkgs-list.sh dest=/usr/local/bin/make-cgit-pkgs-list.sh mode=0755 when: inventory_hostname.startswith('pkgs') + tags: cgit - name: install the script for fedorapeople copy: src=make-people-git.sh dest=/usr/local/bin/make-people-git.sh mode=0755 when: inventory_hostname.startswith('people') + tags: cgit # # For the batcave we just have a small static list @@ -24,10 +29,12 @@ - name: install the file for batcave copy: src=cgit-projects-batcave dest=/etc/cgit-projects-batcave mode=0644 when: inventory_hostname.startswith('batcave') + tags: cgit - name: install the file for infinote copy: src=cgit-projects-infinote dest=/etc/cgit-projects-infinote mode=0644 when: inventory_hostname.startswith('infinote') + tags: cgit - name: install the cron job cron: > @@ -36,6 +43,7 @@ user=root job="/usr/local/bin/lock-wrapper make-cgit-pkgs-list '/usr/local/bin/make-cgit-pkgs-list.sh | /usr/local/bin/nag-once make-cgit-pkgs-list 1d 2>&1'" when: inventory_hostname.startswith('pkgs') + tags: cgit - name: install the cron job for fedora people cgit cron: > @@ -44,6 +52,7 @@ user=root job="/usr/local/bin/lock-wrapper make-people-git '/usr/local/bin/make-people-git.sh | /usr/local/bin/nag-once make-people-git 1d 2>&1'" when: inventory_hostname.startswith('people') + tags: cgit - name: check the selinux context of the repo list command: matchpathcon /srv/git/pkgs-git-repos-list diff --git a/roles/check-compose/files/checkcomp_consumer.py b/roles/check-compose/files/checkcomp_consumer.py deleted file mode 100644 index 5d3068cffb..0000000000 --- a/roles/check-compose/files/checkcomp_consumer.py +++ /dev/null @@ -1,3 +0,0 @@ -config = { - 'checkcomp_consumer.prod.enabled': True, -} diff --git a/roles/check-compose/tasks/main.yml b/roles/check-compose/tasks/main.yml index 0d91c4833b..7c8e52b8bd 100644 --- a/roles/check-compose/tasks/main.yml +++ b/roles/check-compose/tasks/main.yml @@ -78,7 +78,7 @@ - restart fedmsg-hub - name: Enable fedmsg consumer - copy: src=checkcomp_consumer.py dest=/etc/fedmsg.d/checkcomp_consumer.py owner=root group=root mode=0644 + template: src=checkcomp_consumer.py.j2 dest=/etc/fedmsg.d/checkcomp_consumer.py owner=root group=root mode=0644 notify: - restart fedmsg-hub tags: diff --git a/roles/check-compose/templates/checkcomp_consumer.py.j2 b/roles/check-compose/templates/checkcomp_consumer.py.j2 new file mode 100644 index 0000000000..d5af64308a --- /dev/null +++ b/roles/check-compose/templates/checkcomp_consumer.py.j2 @@ -0,0 +1,9 @@ +config = { +{% if deployment_type is defined and deployment_type == 'prod' %} + 'checkcomp_consumer.prod.enabled': True, +{% elif deployment_type is defined and deployment_type == 'stg' %} + 'checkcomp_consumer.stg.enabled': True, +{% else %} + checkcomp_consumer.test.enabled': True, +{% endif %} +} diff --git a/roles/copr/backend/files/provision/files/mock/site-defaults.cfg b/roles/copr/backend/files/provision/files/mock/site-defaults.cfg index 399126e6fd..d8491025e5 100644 --- a/roles/copr/backend/files/provision/files/mock/site-defaults.cfg +++ b/roles/copr/backend/files/provision/files/mock/site-defaults.cfg @@ -14,14 +14,12 @@ config_opts['plugin_conf']['compress_logs_enable'] = True config_opts['plugin_conf']['compress_logs_opts'] = {} config_opts['plugin_conf']['compress_logs_opts']['command'] = "/usr/bin/gzip -9 --force" -{% if ansible_architecture == 'x86_64' %} config_opts['plugin_conf']['tmpfs_enable'] = True config_opts['plugin_conf']['tmpfs_opts'] = {} config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024 config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = '75g' config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755' config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = False -{% endif %} {% if ansible_distribution == 'Fedora' and ansible_distribution_major_version|int > 21 %} config_opts['yum_command'] = '/usr/bin/yum-deprecated' diff --git a/roles/copr/backend/files/provision/provision_builder_tasks_ppc64le.yml b/roles/copr/backend/files/provision/provision_builder_tasks_ppc64le.yml index 9effa0fbcb..1b2e89dc0d 100644 --- a/roles/copr/backend/files/provision/provision_builder_tasks_ppc64le.yml +++ b/roles/copr/backend/files/provision/provision_builder_tasks_ppc64le.yml @@ -4,9 +4,6 @@ - name: set bigger timeout for yum ini_file: dest=/etc/yum.conf section=main option=timeout value=1000 -- name: install distribution-gpg-keys which are right now not in fedora main - shell: dnf install -y https://kojipkgs.fedoraproject.org//packages/distribution-gpg-keys/1.5/1.fc24/noarch/distribution-gpg-keys-1.5-1.fc24.noarch.rpm - - name: install pkgs yum: state=present pkg={{ item }} with_items: @@ -63,6 +60,7 @@ - name: symlink F26 configs to rawhide ones shell: for cfg in /etc/mock/fedora-rawhide-*.cfg; do ln -s $cfg `echo $cfg|sed 's/rawhide/26/'`; done + when: prepare_base_image is defined # ansible doesn't support simultaneously usage of async and with_* options # it's not even planned for implementation, see https://github.com/ansible/ansible/issues/5841 diff --git a/roles/copr/backend/templates/provision/nova_cloud_vars_ppc64le.yml b/roles/copr/backend/templates/provision/nova_cloud_vars_ppc64le.yml index f7a7da1cfc..2a65936f9e 100644 --- a/roles/copr/backend/templates/provision/nova_cloud_vars_ppc64le.yml +++ b/roles/copr/backend/templates/provision/nova_cloud_vars_ppc64le.yml @@ -11,7 +11,7 @@ OS_PASSWORD_OLD: "{{ copr_nova_password|default('variable OS_PASSWORD_OLD is und OS_PASSWORD: "{{ copr_password|default('variable OS_PASSWORD is undefined')}}" -image_name: "builder-f24-ppc64le" +image_name: "builder-f24-ppc64le-swapmounted" flavor_name: "{{ copr_builder_flavor_name }}" network_name: "{{ copr_builder_network_name }}" key_name: "{{ copr_builder_key_name }}" diff --git a/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf b/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf index 149fd04d43..8fd2808ce4 100644 --- a/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf +++ b/roles/httpd/reverseproxy/templates/reversepassproxy.registry.conf @@ -3,9 +3,8 @@ RequestHeader set X-Scheme https early RequestHeader set X-Forwarded-Proto https early ProxyPreserveHost On -{% if env == "maybe-in-staging" %} -RewriteRule ^/v2/([a-zA-Z]*)/blobs/([a-zA-Z]*) https://ourregistry/v2/$1/blobs/$2 [R] -{% endif %} +RewriteCond %{HTTP:VIA} !cdn77 +RewriteRule ^/v2/(.*)/blobs/([a-zA-Z0-9:]*) https://cdn.registry.fedoraproject.org/v2/$1/blobs/$2 [R] # This is terible, but Docker. RewriteEngine on diff --git a/roles/mbs/common/tasks/main.yml b/roles/mbs/common/tasks/main.yml index 5a1126d3bb..7e46835ff8 100644 --- a/roles/mbs/common/tasks/main.yml +++ b/roles/mbs/common/tasks/main.yml @@ -15,6 +15,15 @@ - mbs - mbs/common +- name: kill development config + file: path=/etc/fedmsg.d/module_build_service.py state=absent + notify: + - restart apache + - restart fedmsg-hub + tags: + - mbs + - mbs/common + - name: copy app configuration template: > src=config.py dest=/etc/module-build-service/config.py @@ -26,10 +35,22 @@ - mbs - mbs/common -- name: copy client secrets that aren't really secret. +- name: copy fedmsg configuration + template: > + src=mbs-fedmsg.py dest=/etc/fedmsg.d/mbs-fedmsg.py + owner=root group=fedmsg mode=0644 + notify: + - restart apache + - restart fedmsg-hub + tags: + - mbs + - mbs/common + +- name: copy client secrets template: > src=client_secrets.json.{{env}} dest=/etc/module-build-service/client_secrets.json - owner=root group=root mode=0644 + owner=root group=fedmsg mode=0640 + when: inventory_hostname.startswith('mbs-frontend') notify: - restart apache tags: diff --git a/roles/mbs/common/templates/client_secrets.json.production b/roles/mbs/common/templates/client_secrets.json.production index e25c1b49a8..9b12f4cc78 100644 --- a/roles/mbs/common/templates/client_secrets.json.production +++ b/roles/mbs/common/templates/client_secrets.json.production @@ -1,12 +1,11 @@ { "web": { "auth_uri": "https://id.fedoraproject.org/openidc/Authorization", - "client_id": "mbs-authorizer", - "client_secret": "notsecret", - "redirect_uris": [ - "http://localhost:13747/" - ], + "client_id": "mbs-prod", + "client_secret": "{{ mbs_prod_oidc_client_secret }}", + "redirect_uris": [], "token_uri": "https://id.fedoraproject.org/openidc/Token", - "token_introspection_uri": "https://id.fedoraproject.org/openidc/TokenInfo" + "token_introspection_uri": "https://id.fedoraproject.org/openidc/TokenInfo", + "userinfo_uri": "https://id.fedoraproject.org/openidc/UserInfo" } } diff --git a/roles/mbs/common/templates/client_secrets.json.staging b/roles/mbs/common/templates/client_secrets.json.staging index 7fd5069dae..f78371caf8 100644 --- a/roles/mbs/common/templates/client_secrets.json.staging +++ b/roles/mbs/common/templates/client_secrets.json.staging @@ -1,12 +1,11 @@ { "web": { "auth_uri": "https://id.stg.fedoraproject.org/openidc/Authorization", - "client_id": "mbs-authorizer", - "client_secret": "notsecret", - "redirect_uris": [ - "http://localhost:13747/" - ], + "client_id": "mbs-stg", + "client_secret": "{{ mbs_stg_oidc_client_secret }}", + "redirect_uris": [], "token_uri": "https://id.stg.fedoraproject.org/openidc/Token", - "token_introspection_uri": "https://id.stg.fedoraproject.org/openidc/TokenInfo" + "token_introspection_uri": "https://id.stg.fedoraproject.org/openidc/TokenInfo", + "userinfo_uri": "https://id.stg.fedoraproject.org/openidc/UserInfo" } } diff --git a/roles/mbs/common/templates/config.py b/roles/mbs/common/templates/config.py index d245d456d0..fb6bb81082 100644 --- a/roles/mbs/common/templates/config.py +++ b/roles/mbs/common/templates/config.py @@ -98,8 +98,10 @@ class ProdConfiguration(BaseConfiguration): LOG_LEVEL = 'debug' LOG_BACKEND = 'console' + # Yes, use tls. PDC_INSECURE = False - PDC_DEVELOP = False + # No, don't try to obtain a token (we just read. we don't write.) + PDC_DEVELOP = True KOJI_CONFIG = path.join(confdir, 'koji.conf') {% if env == 'staging' %} diff --git a/roles/mbs/common/templates/mbs-fedmsg.py b/roles/mbs/common/templates/mbs-fedmsg.py new file mode 100644 index 0000000000..1d8c824ba0 --- /dev/null +++ b/roles/mbs/common/templates/mbs-fedmsg.py @@ -0,0 +1,7 @@ +import socket + +config = { + # So that the MBS can find it's cert in /etc/fedmsg.d/ssl.py + 'cert_prefix': 'mbs', + 'name': 'mbs.%s' % socket.gethostname().split('.', 1)[0], +} diff --git a/roles/mbs/frontend/tasks/main.yml b/roles/mbs/frontend/tasks/main.yml index 1d24257955..761318c437 100644 --- a/roles/mbs/frontend/tasks/main.yml +++ b/roles/mbs/frontend/tasks/main.yml @@ -4,7 +4,7 @@ - name: disable the scheduler on the frontend copy: > src={{ item }} dest=/etc/fedmsg.d/{{ item }} - owner=apache group=apache mode=0600 + owner=fedmsg group=fedmsg mode=0644 with_items: - mbs-scheduler.py notify: @@ -41,6 +41,7 @@ with_items: - httpd_can_network_connect_db - httpd_can_network_memcache + - httpd_can_network_connect - httpd_can_sendmail tags: - mbs diff --git a/roles/opendkim/files/opendkim.conf b/roles/opendkim/files/opendkim.conf index 03353770c0..c50d1d1692 100644 --- a/roles/opendkim/files/opendkim.conf +++ b/roles/opendkim/files/opendkim.conf @@ -131,3 +131,7 @@ OversignHeaders From ## caching service. Useful if the nameserver being used by the filter is ## not local. # QueryCache yes +# +# We need to increase the default header size because notifs adds a X-fedmsg header for +# each message in a digest. +MaximumHeaders 262144 diff --git a/roles/openqa/dispatcher/tasks/main.yml b/roles/openqa/dispatcher/tasks/main.yml index da87403149..1a3fc0de22 100644 --- a/roles/openqa/dispatcher/tasks/main.yml +++ b/roles/openqa/dispatcher/tasks/main.yml @@ -16,8 +16,10 @@ # - wikitcms_password ## string - password for relval_user # - deployment_type -## string - Fedora Infrastructure thing; for this role, decides -## whether and where to submit wiki results +## string - Fedora Infrastructure thing; for this role, the +## fedora_openqa config file will be set appropriately +## for infra deployments if this is set, so don't set +## it for private deployments # # When all of the above are set, a wikitcms 'credentials' file will # be created and result submission to the wiki will be enabled. If @@ -75,25 +77,29 @@ chdir: /root/openQA-python-client when: "gitclient|changed or instclient.rc > 0" -# We check this out to a different place from the 'server' task, because -# otherwise it's hard to make sure we install each time it changes. -- name: Check out openqa_fedora_tools +- name: Remove old openqa_fedora_tools checkout + file: path=/root/openqa_fedora_tools-dispatcher state=absent + +- name: Remove old fedora-openqa-schedule script + file: path=/usr/bin/fedora-openqa-schedule state=absent + +- name: Check out fedora_openqa (scheduler / reporter tool) git: - repo: https://bitbucket.org/rajcze/openqa_fedora_tools - dest: /root/openqa_fedora_tools-dispatcher + repo: https://pagure.io/fedora-qa/fedora_openqa.git + dest: /root/fedora_openqa register: gittools -- name: Check if openqa_fedora_tools has ever been installed - stat: path=/usr/bin/fedora-openqa-schedule +- name: Check if fedora_openqa has ever been installed + stat: path=/usr/bin/fedora-openqa register: insttools changed_when: "1 != 1" failed_when: "1 != 1" check_mode: no -- name: Install openqa_fedora_tools +- name: Install fedora_openqa command: "python setup.py install" args: - chdir: /root/openqa_fedora_tools-dispatcher/scheduler + chdir: /root/fedora_openqa when: "gittools|changed or not insttools.stat.exists" notify: - restart fedmsg-hub @@ -103,7 +109,7 @@ tags: - config -- name: Create fedora-openqa-schedule config directory +- name: Create fedora_openqa config directory file: path=/etc/fedora-openqa state=directory owner=root group=root mode=0755 - name: Write schedule.conf diff --git a/roles/openqa/dispatcher/templates/openqa_consumer.py.j2 b/roles/openqa/dispatcher/templates/openqa_consumer.py.j2 index 33219f72dc..d0b62d2282 100644 --- a/roles/openqa/dispatcher/templates/openqa_consumer.py.j2 +++ b/roles/openqa/dispatcher/templates/openqa_consumer.py.j2 @@ -1,13 +1,16 @@ config = { {% if openqa_consumer %} - 'fedora_openqa_schedule.consumer.enabled': True, + 'fedora_openqa.scheduler.prod.enabled': True, {% endif %} -{% if wikitcms_user is defined and wikitcms_password is defined %} {% if deployment_type is defined and deployment_type == 'prod' %} - 'fedora_openqa_schedule.wiki.consumer.prod.enabled': True, - 'fedora_openqa_schedule.resultsdb.reporter.prod.enabled': True, -{% elif deployment_type is defined and deployment_type == 'stg' %} - 'fedora_openqa_schedule.wiki.consumer.stg.enabled': True, +{% if wikitcms_user is defined and wikitcms_password is defined %} + 'fedora_openqa.reporter.wiki.prod.enabled': True, {% endif %} + 'fedora_openqa.reporter.resultsdb.prod.enabled': True, +{% elif deployment_type is defined and deployment_type == 'stg' %} +{% if wikitcms_user is defined and wikitcms_password is defined %} + 'fedora_openqa.reporter.wiki.stg.enabled': True, +{% endif %} + 'fedora_openqa.reporter.resultsdb.stg.enabled': True, {% endif %} } diff --git a/roles/openqa/dispatcher/templates/schedule.conf.j2 b/roles/openqa/dispatcher/templates/schedule.conf.j2 index ab79ae819d..f599284ff0 100644 --- a/roles/openqa/dispatcher/templates/schedule.conf.j2 +++ b/roles/openqa/dispatcher/templates/schedule.conf.j2 @@ -1,14 +1,24 @@ [report] -openqa_url: https://{{ external_hostname|default(ansible_nodename) }} -{% if wikitcms_user is defined and wikitcms_password is defined %} -submit_wiki: true -submit_resultsdb: true -{% else %} -submit_wiki: false -submit_resultsdb: false -{% endif %} {% if deployment_type is defined and deployment_type == 'prod' %} resultsdb_url: http://resultsdb01.qa.fedoraproject.org/resultsdb_api/api/v2.0/ +wiki_hostname: fedoraproject.org +[consumers] +prod_oqa_hostname: localhost +prod_oqa_baseurl: https://{{ external_hostname|default(ansible_nodename) }} +prod_wiki_hostname: fedoraproject.org +prod_wiki_report: true +prod_rdb_url: http://resultsdb01.qa.fedoraproject.org/resultsdb_api/api/v2.0/ +prod_rdb_report: true {% elif deployment_type is defined and deployment_type == 'stg' %} resultsdb_url: http://resultsdb-stg01.qa.fedoraproject.org/resultsdb_api/api/v2.0/ +wiki_hostname: stg.fedoraproject.org +[consumers] +# as we use the production scheduler +prod_oqa_hostname: localhost +stg_oqa_hostname: localhost +stg_oqa_baseurl: https://{{ external_hostname|default(ansible_nodename) }} +stg_wiki_hostname: stg.fedoraproject.org +stg_wiki_report: true +stg_rdb_url: http://resultsdb-stg01.qa.fedoraproject.org/resultsdb_api/api/v2.0/ +stg_rdb_report: true {% endif %} diff --git a/roles/openqa/server/files/createhdds b/roles/openqa/server/files/createhdds index c76019d8b6..794d80450e 100644 --- a/roles/openqa/server/files/createhdds +++ b/roles/openqa/server/files/createhdds @@ -1,4 +1,4 @@ #!/bin/sh cd /var/lib/openqa/share/factory/hdd/fixed -LIBGUESTFS_BACKEND=direct withlock /var/lock/createhdds.lock /root/openqa_fedora_tools/tools/createhdds.py all --clean +LIBGUESTFS_BACKEND=direct withlock /var/lock/createhdds.lock /root/createhdds/createhdds.py all --clean diff --git a/roles/openqa/server/tasks/main.yml b/roles/openqa/server/tasks/main.yml index 957290b6be..27dbe1a39c 100644 --- a/roles/openqa/server/tasks/main.yml +++ b/roles/openqa/server/tasks/main.yml @@ -120,11 +120,13 @@ become_user: geekotest when: "(testsbranch.stdout.find('Not a git repository') != -1) or (testsbranch.stdout.find('On branch master') != -1)" -- name: Check out openqa_fedora_tools +- name: Remove old openqa_fedora_tools checkout + file: path=/root/openqa_fedora_tools state=absent + +- name: Check out createhdds git: - repo: https://bitbucket.org/rajcze/openqa_fedora_tools - dest: /root/openqa_fedora_tools - register: git_result + repo: https://pagure.io/fedora-qa/createhdds.git + dest: /root/createhdds - name: Create asset directories file: path={{ item }} state=directory owner=geekotest group=root mode=0755 @@ -139,7 +141,7 @@ copy: src=createhdds dest=/etc/cron.daily/createhdds owner=root group=root mode=0755 - name: Check if any hard disk images need (re)building - command: "/root/openqa_fedora_tools/tools/createhdds.py check" + command: "/root/createhdds/createhdds.py check" args: chdir: /var/lib/openqa/share/factory/hdd/fixed register: diskcheck diff --git a/roles/pkgdb2/templates/pkgdb2.cfg b/roles/pkgdb2/templates/pkgdb2.cfg index c033723ea9..bdb677d508 100644 --- a/roles/pkgdb2/templates/pkgdb2.cfg +++ b/roles/pkgdb2/templates/pkgdb2.cfg @@ -27,7 +27,9 @@ AUTOAPPROVE_PKGERS = [ #### FAS group for the pkgdb admins {% if env == 'staging' %} -ADMIN_GROUP = ['cvsadmin'] +# Factory 2 added to make it easier for us to create repos for the modularity +# team. Only in staging. Feel free to remove! +ADMIN_GROUP = ['cvsadmin', 'factory2'] {% else %} ADMIN_GROUP = ['sysadmin-main', 'cvsadmin'] {% endif %} diff --git a/roles/web-data-analysis/files/condense-hotspot.cron b/roles/web-data-analysis/files/condense-hotspot.cron new file mode 100644 index 0000000000..b4fe6d07c1 --- /dev/null +++ b/roles/web-data-analysis/files/condense-hotspot.cron @@ -0,0 +1 @@ +0 07 * * * root /usr/local/bin/condense-hotspot.sh diff --git a/roles/web-data-analysis/files/condense-hotspot.sh b/roles/web-data-analysis/files/condense-hotspot.sh new file mode 100644 index 0000000000..1f5976c4b9 --- /dev/null +++ b/roles/web-data-analysis/files/condense-hotspot.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +# This file is part of Fedora Project Infrastructure Ansible +# Repository. +# +# Fedora Project Infrastructure Ansible Repository is free software: +# you can redistribute it and/or modify it under the terms of the GNU +# General Public License as published by the Free Software Foundation, +# either version 3 of the License, or (at your option) any later +# version. +# +# Fedora Project Infrastructure Ansible Repository is distributed in +# the hope that it will be useful, but WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License +# along with Fedora Project Infrastructure Ansible Repository. If +# not, see . + +# There is a multiday delay involved in processing the logs. It +# may take up to 4 days to get the logs to the main-server. It may +# take a day to combine all the logs onto combined-httpd. So we assume +# we are 5 days behind. + +let NUMDAYS=5 +let OLDDAYS=$(( $NUMDAYS+1 )) + +PROJECT=hotspot +WEBLOG=fedoraproject.org + +# This is the year/month/day for a N days ago. +YEAR=$(/bin/date -d "-${NUMDAYS} days" +%Y) +MONTH=$(/bin/date -d "-${NUMDAYS} days" +%m) +DAY=$(/bin/date -d "-${NUMDAYS} days" +%d) + +# And we have have to deal with year/month/day boundaries for our later grep. +OLDDATE=$(/bin/date -d "-${OLDDAYS} days" +%Y-%m-%d) +OLDYEAR=$(/bin/date -d "-${OLDDAYS} days" +%Y) + +NFSDIR=/mnt/fedora_stats/combined-http +TARGET=${NFSDIR}/${YEAR}/${MONTH}/${DAY} + +LOGFILE=${TARGET}/${WEBLOG}-access.log + +WORKDIR=/mnt/fedora_stats/data/${PROJECT} +WORKFILE=${WORKDIR}/${YEAR}/${MONTH}/raw-${DAY} + +WEBDIR=/var/www/html/csv-reports/${PROJECT} + +TEMPDIR=$( mktemp -d /tmp/web-data-analysis.XXXXXXXXX ) + +LBIN=/usr/local/bin/ +LSHARE=/usr/local/share/web-data-analysis + +mkdir -p ${WORKDIR}/${YEAR}/${MONTH} +if [[ ! -f ${WORKDIR}/${YEAR}/out-${YEAR}-${MONTH} ]]; then + touch ${WORKDIR}/${YEAR}/out-${YEAR}-${MONTH} +fi + +if [[ ! -f ${WORKDIR}/out-${YEAR} ]]; then + touch ${WORKDIR}/out-${YEAR} +fi + +if [[ ! -f ${LOGFILE} ]]; then + echo "No logfile found for ${YEAR}/${MONTH}/${DAY}. Please fix." +else + awk -f ${LSHARE}/${PROJECT}.awk ${LOGFILE} > ${WORKFILE} +fi + +# So the data isn't strictly across month boundries due to the end of +# the logfiles being at 04:00 versus 23:59. Also log files might get +# stuck and you end up with days or weeks of data in a single +# file. Because the data is pretty small we can get away with adding up the data every day. + +find ${WORKDIR} -type f | grep raw- | xargs cat | sort -u | awk 'BEGIN{x=0; y=0}; {if (x != $1){ print x,y; x=$1; y=$2} else {y=y+$2}}' > ${WORKDIR}/worked-all + + +awk -f ${LSHARE}/${PROJECT}-data.awk ${WORKDIR}/worked-all | grep -v "1970-01-01,0,0,0" | sort -u > ${WEBDIR}/${PROJECT}data-all.csv + +# Make the seven day moving average file +/usr/local/bin/hotspot-moving_avg.py > ${WEBDIR}/${PROJECT}data-all-7day-ma.csv + +gnuplot ${LSHARE}/${PROJECT}.gp + +# cleanup the temp data +rm -rf ${TEMPDIR} diff --git a/roles/web-data-analysis/files/hotspot-data.awk b/roles/web-data-analysis/files/hotspot-data.awk new file mode 100644 index 0000000000..93ec3d3d46 --- /dev/null +++ b/roles/web-data-analysis/files/hotspot-data.awk @@ -0,0 +1,33 @@ +BEGIN{ + date=strftime("%F",86401); + count=1; + sum=0; + most=0; + least=0 + print "1970-01-01,AVG,LEAST,MAX" +} + +{ + newdate=strftime("%F",$1); # convert this to a printable date + if (date != newdate){ + print date "," int(sum/count) "," least "," most; + date=newdate; + count=1; # start count to 0. we should have 288 per day but logs are stupid + sum=$2; # start the sum + most=$2; # what is going to be our most per day + least=$2; # what is going to be our least per day + } else { + count=count+1; + sum=sum+$2; + if ($2 > most){ + most=$2; + }; + if ($2 < least) { + least=$2; + } + } +} + +END{ + print date "," int(sum/count) "," least "," most; +} diff --git a/roles/web-data-analysis/files/hotspot-moving_avg.py b/roles/web-data-analysis/files/hotspot-moving_avg.py new file mode 100644 index 0000000000..13929fa481 --- /dev/null +++ b/roles/web-data-analysis/files/hotspot-moving_avg.py @@ -0,0 +1,50 @@ +#!/usr/bin/python + +# This file is part of Fedora Project Infrastructure Ansible +# Repository. +# +# Fedora Project Infrastructure Ansible Repository is free software: +# you can redistribute it and/or modify it under the terms of the GNU +# General Public License as published by the Free Software Foundation, +# either version 3 of the License, or (at your option) any later +# version. +# +# Fedora Project Infrastructure Ansible Repository is distributed in +# the hope that it will be useful, but WITHOUT ANY WARRANTY; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License +# along with Fedora Project Infrastructure Ansible Repository. If +# not, see . + +# This is a complete horrible hack to get something done. Patches are +# really welcome. + +import pandas +#import matplotlib.pyplot as plt +import math + +rolling = 7 + +tree = {} + +df = pandas.read_csv("/var/www/html/csv-reports/hotspot/hotspotdata-all.csv") + +dates = df['1970-01-01'] +AVG = pandas.rolling_mean(df['AVG'],rolling) +LEAST = pandas.rolling_mean(df['LEAST'],rolling) +MAX = pandas.rolling_mean(df['MAX'],rolling) + + +for i in xrange(0,len(dates)): + if math.isnan(MAX[i]): + csv_line = ",".join([dates[i],"0","0"]) + else: + csv_line = ",".join([dates[i], + str(int(AVG[i])), + str(int(LEAST[i])), + str(int(MAX[i])), + ]) + print csv_line diff --git a/roles/web-data-analysis/files/hotspot.awk b/roles/web-data-analysis/files/hotspot.awk new file mode 100644 index 0000000000..f47da6958e --- /dev/null +++ b/roles/web-data-analysis/files/hotspot.awk @@ -0,0 +1,95 @@ +# +# Take the apache log line +# 83.163.161.147 - - [30/Sep/2012:13:54:19 +0000] "GET /static/hotspot.txt HTTP/1.1" 200 3 "-" "dnssec-trigger/0.11" +# Convert to +# 1349013000 1 + +function convertdate(str) { + gsub(/\[/, "", str) + gsub(/\]/, "", str) + split(str,a,":"); + split(a[1],b,"/"); + temp=""; + switch (b[2]) { + case "Jan": + temp="01" + break; + case "Feb": + temp="02" + break; + case "Mar": + temp="03" + break; + case "Apr": + temp="04" + break; + case "May": + temp="05" + break; + case "Jun": + temp="06" + break; + case "Jul": + temp="07" + break; + case "Aug": + temp="08" + break; + case "Sep": + temp="09" + break; + case "Oct": + temp="10" + break; + case "Nov": + temp="11" + break; + case "Dec": + temp="12" + break; + default: + temp="00" + break; + } + x=b[3]" "temp" "b[1]" "a[2]" "a[3] " "a[4] + y=int(mktime(x)/300) # 300 seconds make 5 minutes (I NEED A GLOBAL VAR) + return y +} + + +BEGIN{ + timestamp=0; + num_ts = 0; + ts_hotspots=0; + total_hotsponts=0; +} + +# +# We assume that every 300 seconds a system will log in at least 1 +# time because the Networkmanager addon does so. +# Convert our date stamp to the nearest 5 minute block and add data to +# it. If the log file goes backwards or jumps etc this will mean +# multiple outputs for a timestamp. A later process will need to deal +# with that. All this will do is output how many it saw at that block +# in the log file. +# + +$7 ~/hotspot.txt/ && $6 ~/GET/ { + date = convertdate($4) + if (timestamp != date) { + num_ts = num_ts +1; + print (timestamp*300),ts_hotspots # GLOBAL VAR GOES HERE + timestamp = date; + ts_hotspots = 1; + } else { + ts_hotspots = ts_hotspots +1; + total_hotspots = total_hotspots +1; + } +} + +END { + num_ts = num_ts +1; + print int(timestamp*300),ts_hotspots # LOOK GLOBAL VAR AGAIN +} + +## END OF FILE diff --git a/roles/web-data-analysis/files/hotspot.gp b/roles/web-data-analysis/files/hotspot.gp new file mode 100644 index 0000000000..dc17471de5 --- /dev/null +++ b/roles/web-data-analysis/files/hotspot.gp @@ -0,0 +1,25 @@ +set grid +set xdata time +set format x "%Y-%m-%d" +set timefmt "%Y-%m-%d" + +set datafile separator "," +set term png size 1600,1200 + +## +set output "/var/www/html/csv-reports/images/hotspot-all.png" +set title "IPs grabbing hotspot per day" +plot ["2014-12-01":"2017-12-31"] \ + '/var/www/html/csv-reports/hotspot/hotspotdata-all.csv' using 1:2 title 'Average every 5min' with lines lw 4, \ + '/var/www/html/csv-reports/hotspot/hotspotdata-all.csv' using 1:3 title 'Least 5min' with lines lw 4, \ + '/var/www/html/csv-reports/hotspot/hotspotdata-all.csv' using 1:4 title 'Max 5min' with lines lw 4 +unset output + +## +set output "/var/www/html/csv-reports/images/hotspot-all-ma.png" +set title "Moving Average of IPs grabbing hotspot" +plot ["2014-12-01":"2017-12-31"] \ + '/var/www/html/csv-reports/hotspot/hotspotdatadata-all-7day-ma.csv' using 1:2 title 'Average every 5min' with lines lw 4, \ + '/var/www/html/csv-reports/hotspot/hotspotdatadata-all-7day-ma.csv' using 1:3 title 'Least 5min' with lines lw 4, \ + '/var/www/html/csv-reports/hotspot/hotspotdatadata-all-7day-ma.csv' using 1:4 title 'Max 5min' with lines lw 4 +unset output diff --git a/roles/web-data-analysis/tasks/main.yml b/roles/web-data-analysis/tasks/main.yml index 81cf5b1e8a..8afb91f780 100644 --- a/roles/web-data-analysis/tasks/main.yml +++ b/roles/web-data-analysis/tasks/main.yml @@ -39,25 +39,25 @@ - name: scripts to condense data down for further processing copy: src={{item}} dest=/usr/local/bin/ mode=0755 - with_items: [condense-mirrorlogs.sh, condense-getfedoralogs.sh, run-daily-awstats.sh] + with_items: [condense-mirrorlogs.sh, condense-getfedoralogs.sh, run-daily-awstats.sh, condense-hotspot.sh ] tags: - web-data - name: python scripts to calculate various data copy: src={{item}} dest=/usr/local/bin/ mode=0755 - with_items: [mirrorlist.py, mirrors-moving_avg.py] + with_items: [mirrorlist.py, mirrors-moving_avg.py, hotspot-moving_avg.py] tags: - web-data - name: awk files for csv creation copy: src={{item}} dest=/usr/local/share/web-data-analysis mode=0644 - with_items: [mirrors-data.awk, getfedora-data.awk, getfedora.awk] + with_items: [mirrors-data.awk, getfedora-data.awk, getfedora.awk, hotspot-data.awk, hotspot.awk ] tags: - web-data - name: gnuplot file for image creation copy: src={{item}} dest=/usr/local/share/web-data-analysis mode=0644 - with_items: [ mirrors-data.gp, getfedora-data.gp, mirrors-moving.gp ] + with_items: [ mirrors-data.gp, getfedora-data.gp, mirrors-moving.gp, hotspot.gp ] tags: - web-data