infrastructure/ansible
1
0
Fork 0
Code Pull requests 7 Activity Actions

Wipe everything that is to do with pdc-backend from our ansible repo

Browse source 
Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>
This commit is contained in:
Pierre-Yves Chibon 2020-08-04 18:13:11 +02:00 • committed by kevin
parent 8fa4144415
commit f91a80046b
10 changed files with 2 additions and 262 deletions
Download patch file Download diff file Expand all files Collapse all files

22
playbooks/groups/pdc.yml
View file

@ -1,9 +1,9 @@
# PDC servers (both frontend and backend)
- import_playbook: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=pdc_web:pdc_web_stg:pdc_backend:pdc_backend_stg"
- import_playbook: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=pdc_web:pdc_web_stg"
- name: dole out the generic configuration
hosts: pdc_web:pdc_web_stg:pdc_backend:pdc_backend_stg
hosts: pdc_web:pdc_web_stg
user: root
gather_facts: True
vars_files:
@ -52,21 +52,3 @@
username: "pdc{{ env_suffix }}"
when: inventory_hostname.startswith(('pdc-web01','pdc-web01.stg'))
- pdc/frontend
- name: stuff just for the backend nodes
hosts: pdc_backend:pdc_backend_stg
user: root
gather_facts: True
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"
roles:
- fedmsg/base
- fedmsg/hub
- pdc/backend
- role: collectd/fedmsg-service
process: fedmsg-hub

22
playbooks/manual/staging-sync/pdc.yml
View file

@ -12,17 +12,6 @@
tasks:
- service: name=httpd state=stopped
- name: bring staging services down (fedmsg-hub)
hosts: pdc-backend-stg
user: root
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
tasks:
- service: name=fedmsg-hub state=stopped
# Here's the meaty part in the middle
- name: drop and re-create the staging db entirely
hosts: db01.stg.phx2.fedoraproject.org
@ -59,14 +48,3 @@
tasks:
- service: name=httpd state=started
- name: bring staging services up (fedmsg-hub)
hosts: pdc-backend-stg
user: root
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
tasks:
- service: name=fedmsg-hub state=started

48
playbooks/manual/upgrade/pdc.yml
View file

@ -1,33 +1,3 @@
- name: verify the backend, and stop it
hosts: pdc_backend:pdc_backend_stg
user: root
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"
pre_tasks:
- name: clean all metadata
command: yum clean all
check_mode: no
- name: update PDC packages
package:
name: ["pdc-client", "pdc-updater", "libmodulemd", "python2-productmd"]
state: latest
roles:
- pdc/backend
post_tasks:
- name: tell nagios to shush w.r.t. the backend
nagios: action=downtime minutes=15 service=host host={{ inventory_hostname_short }}{{ env_suffix }}
delegate_to: noc01.iad2.fedoraproject.org
ignore_errors: true
- name: Stop the pdc-updater backend
service: name="fedmsg-hub" state=stopped
- name: verify the frontend, stop it, upgrade the db, and restart it
hosts: pdc_web:pdc_web_stg
user: root
@ -67,21 +37,3 @@
nagios: action=unsilence service=host host={{ inventory_hostname_short }}{{ env_suffix }}
delegate_to: noc01.iad2.fedoraproject.org
ignore_errors: true
- name: restart the backend
hosts: pdc_backend:pdc_backend_stg
user: root
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"
tasks:
- name: And... start the backend again
service: name="fedmsg-hub" state=started
- name: tell nagios to unshush w.r.t. the backend
nagios: action=unsilence service=host host={{ inventory_hostname_short }}{{ env_suffix }}
delegate_to: noc01.iad2.fedoraproject.org
ignore_errors: true

4
roles/haproxy/templates/haproxy.cfg
View file

@ -215,10 +215,6 @@ backend openqa-backend
frontend pdc-frontend
bind 0.0.0.0:10045
default_backend pdc-backend
backend pdc-backend
balance hdr(appserver)
{% if env != "staging" %}
# Set session persistence with a cookie.

3
roles/nagios_client/templates/check_fedmsg_consumers.cfg.j2
View file

@ -15,7 +15,6 @@ command[check_fedmsg_cp_bugzilla2fedmsg]={{libdir}}/nagios/plugins/check_fedmsg_
command[check_fedmsg_cp_fedimg_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FedimgConsumer MonitoringProducer
command[check_fedmsg_cp_hotness_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub BugzillaTicketFiler MonitoringProducer
command[check_fedmsg_cp_packages_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub CacheInvalidator MonitoringProducer
command[check_fedmsg_cp_pdc_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub PDCUpdater MonitoringProducer
command[check_fedmsg_cp_mbs_backend]={{libdir}}/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub MBSConsumer MBSProducer MonitoringProducer
command[check_fedmsg_cexceptions_busgateway_hub]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub Nommer 1 10
@ -34,7 +33,6 @@ command[check_fedmsg_cexceptions_bugzilla2fedmsg]={{libdir}}/nagios/plugins/chec
command[check_fedmsg_cexceptions_fedimg_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub FedimgConsumer 1 10
command[check_fedmsg_cexceptions_hotness_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub BugzillaTicketFiler 1 10
command[check_fedmsg_cexceptions_packages_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub CacheInvalidator 1 10
command[check_fedmsg_cexceptions_pdc_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub PDCUpdater 1 10
command[check_fedmsg_cexceptions_mbs_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub MBSConsumer 1 10
command[check_fedmsg_cbacklog_busgateway_hub]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub Nommer 500 1000
@ -53,7 +51,6 @@ command[check_fedmsg_cbacklog_bugzilla2fedmsg]={{libdir}}/nagios/plugins/check_f
command[check_fedmsg_cbacklog_fedimg_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub FedimgConsumer 2000 5000
command[check_fedmsg_cbacklog_hotness_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub BugzillaTicketFiler 1000 5000
command[check_fedmsg_cbacklog_packages_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub CacheInvalidator 30000 40000
command[check_fedmsg_cbacklog_pdc_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub PDCUpdater 10000 20000
command[check_fedmsg_cbacklog_mbs_backend]={{libdir}}/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub MBSConsumer 10000 20000
command[check_fedmsg_fmn_digest_last_ran]={{libdir}}/nagios/plugins/check_fedmsg_producer_last_ran.py fedmsg-hub DigestProducer 90 600

3
roles/nagios_server/templates/nrpe/nrpe.cfg.j2
View file

@ -391,7 +391,6 @@ command[check_fedmsg_cp_bugzilla2fedmsg]=/usr/lib64/nagios/plugins/check_fedmsg_
command[check_fedmsg_cp_fedimg_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub FedimgConsumer MonitoringProducer
command[check_fedmsg_cp_hotness_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub BugzillaTicketFiler MonitoringProducer
command[check_fedmsg_cp_packages_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub CacheInvalidator MonitoringProducer
command[check_fedmsg_cp_pdc_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub PDCUpdater MonitoringProducer
command[check_fedmsg_cp_mbs_backend]=/usr/lib64/nagios/plugins/check_fedmsg_producers_consumers.py fedmsg-hub MBSConsumer MBSProducer MonitoringProducer
command[check_fedmsg_cexceptions_busgateway_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub Nommer 1 10
@ -405,7 +404,6 @@ command[check_fedmsg_cexceptions_bugzilla2fedmsg]=/usr/lib64/nagios/plugins/chec
command[check_fedmsg_cexceptions_fedimg_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub FedimgConsumer 1 10
command[check_fedmsg_cexceptions_hotness_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub BugzillaTicketFiler 1 10
command[check_fedmsg_cexceptions_packages_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub CacheInvalidator 1 10
command[check_fedmsg_cexceptions_pdc_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub PDCUpdater 1 10
command[check_fedmsg_cexceptions_mbs_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_exceptions.py fedmsg-hub MBSConsumer 1 10
command[check_fedmsg_cbacklog_busgateway_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub Nommer 500 1000
@ -419,7 +417,6 @@ command[check_fedmsg_cbacklog_bugzilla2fedmsg]=/usr/lib64/nagios/plugins/check_f
command[check_fedmsg_cbacklog_fedimg_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub FedimgConsumer 2000 5000
command[check_fedmsg_cbacklog_hotness_backend]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub BugzillaTicketFiler 100 500
command[check_fedmsg_cbacklog_packages_backend_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub CacheInvalidator 30000 40000
command[check_fedmsg_cbacklog_pdc_backend_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub PDCUpdater 10000 20000
command[check_fedmsg_cbacklog_mbs_backend_hub]=/usr/lib64/nagios/plugins/check_fedmsg_consumer_backlog.py fedmsg-hub MBSConsumer 1000 2000
command[check_fedmsg_fmn_digest_last_ran]=/usr/lib64/nagios/plugins/check_fedmsg_producer_last_ran.py fedmsg-hub DigestProducer 90 600

3
roles/pdc/backend/files/pdc-audit.cron
View file

@ -1,3 +0,0 @@
# run once a day and report
MAILTO=releng-cron@lists.fedoraproject.org
0 18 * * * root /usr/local/bin/lock-wrapper pdc-audit /usr/bin/pdc-updater-audit

39
roles/pdc/backend/tasks/main.yml
View file

@ -1,39 +0,0 @@
---
# Configuration for the pdc-updater backend consumer
- name: remove the old pdc-updater repo
file: dest=/etc/yum.repos.d/ralph-pdc-updater-epel-7.repo state=absent
tags: pdc
- name: install needed packages
package: name={{ item }} state=present
with_items:
- pdc-updater
tags: pdc
- name: copy database configuration
template: >
src={{ item }} dest=/etc/fedmsg.d/{{ item }}
owner=fedmsg group=fedmsg mode=0600
with_items:
- pdcupdater.py
notify: restart fedmsg-hub
tags: pdc
- name: make sure /etc/pdc.d exists
file: path=/dev/pdc.d state=directory owner=root group=root
tags: pdc
- name: copy client tool configuration
template: >
src={{ item }} dest=/etc/pdc.d/{{item}}
owner=fedmsg group=fedmsg mode=0600
with_items:
- fedora.json
tags: pdc
- name: setup cron for a daily audit script
copy: src=pdc-audit.cron dest=/etc/cron.d/pdc-audit.cron mode=0644
tags:
- pdc
- cron

14
roles/pdc/backend/templates/fedora.json
View file

@ -1,14 +0,0 @@
{
"fedora": {
"develop": false,
"insecure": false,
{% if env == 'staging' %}
"host": "https://pdc.stg.fedoraproject.org/rest_api/v1/",
"token": "{{pdc_updater_api_token_stg }}"
{% else %}
"host": "https://pdc.fedoraproject.org/rest_api/v1/",
"token": "{{pdc_updater_api_token_prod }}"
{% endif %}
}
}

106
roles/pdc/backend/templates/pdcupdater.py
View file

@ -1,106 +0,0 @@
# Configuration for the pdc-updater backend.
config = {
# Should we turn on the realtime updater?
'pdcupdater.enabled': True,
{% if inventory_hostname.startswith('pdc-backend01') %}
# Use only one thread at a time to handle messages. If we have more than
# one, then we can end up POSTing multiple enormous JSON blobs to the PDC
# web frontend, and we'll send it into OOM death. One at a time, people..
"moksha.workers_per_consumer": 1,
{% else %}
"moksha.workers_per_consumer": 8,
{% endif %}
# Credentials to talk to PDC
'pdcupdater.pdc': {
{% if env == 'staging' %}
'server': 'http://pdc-web01.stg.iad2.fedoraproject.org/rest_api/v1/',
'token': '{{pdc_updater_api_token_stg }}',
{% else %}
'server': 'http://pdc-web01.iad2.fedoraproject.org/rest_api/v1/',
'token': '{{pdc_updater_api_token_prod }}',
{% endif %}
},
# Credentials to talk to FAS
'pdcupdater.fas': {
{% if env == 'staging' %}
'base_url': 'https://admin.stg.fedoraproject.org/accounts',
{% else %}
'base_url': 'https://admin.fedoraproject.org/accounts',
{% endif %}
'username': '{{ fedoraDummyUser }}',
'password': '{{ fedoraDummyUserPassword }}',
},
# MBS details
{% if env == 'staging' %}
'pdcupdater.mbs_url': 'https://mbs.stg.fedoraproject.org/module-build-service/2/module-builds/',
{% else %}
'pdcupdater.mbs_url': 'https://mbs.fedoraproject.org/module-build-service/2/module-builds/',
{% endif %}
# dead.package details
{% if env == 'staging' %}
'pdcupdater.file_check_url': 'https://src.stg.fedoraproject.org/%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s',
{% else %}
'pdcupdater.file_check_url': 'https://src.fedoraproject.org/%(namespace)s/%(repo)s/blob/%(branch)s/f/%(file)s',
{% endif %}
# Koji details
{% if env == 'staging' %}
'pdcupdater.koji_url': 'https://koji.stg.fedoraproject.org/kojihub',
{% else %}
'pdcupdater.koji_url': 'http://koji01.iad2.fedoraproject.org/kojihub',
{% endif %}
# Use 8 threads to talk to koji in parallel.
'pdcupdater.koji_io_threads': 8,
# Where to find composes
{% if env == 'staging' %}
'pdcupdater.old_composes_url': 'https://kojipkgs.stg.fedoraproject.org/compose/',
{% else %}
'pdcupdater.old_composes_url': 'https://kojipkgs.fedoraproject.org/compose/',
{% endif %}
# Where to find the fedora-atomic json definitions.
'pdcupdater.fedora_atomic_git_url': 'https://pagure.io/fedora-atomic/raw/master/f/',
# We have an explicit list of these in the config so we can turn them on
# and off individually in production if one is causing an issue.
'pdcupdater.handlers': [
# See CSI information on host for info on the different handlers.
{% if inventory_hostname.startswith('pdc-backend01') %}
'pdcupdater.handlers.compose:NewComposeHandler',
{% elif inventory_hostname.startswith('pdc-backend02') %}
# For MBS https://fedoraproject.org/wiki/Changes/ModuleBuildService
'pdcupdater.handlers.modules:ModuleStateChangeHandler',
# For https://fedoraproject.org/wiki/Changes/ArbitraryBranching
'pdcupdater.handlers.retirement:RetireComponentHandler',
{% elif inventory_hostname.startswith('pdc-backend03') %}
## https://fedoraproject.org/wiki/User:Ralph/Drafts/Infrastructure/Factory2/ModellingDeps
#'pdcupdater.handlers.depchain.rpms:NewRPMBuildTimeDepChainHandler',
#'pdcupdater.handlers.depchain.rpms:NewRPMRunTimeDepChainHandler',
'pdcupdater.handlers.depchain.containers:ContainerRPMInclusionDepChainHandler',
{% endif %}
# This just isn't used for now.. but we can re-enable it if we need it.
#'pdcupdater.handlers.atomic:AtomicComponentGroupHandler',
],
'logging': dict(
loggers=dict(
pdcupdater={
"level": "DEBUG",
"propagate": False,
"handlers": ["console"],
},
requests={
"level": "INFO",
"propagate": False,
"handlers": ["console"],
},
)
)
}