diff --git a/inventory/group_vars/pdc-backend b/inventory/group_vars/pdc-backend index 147784ba65..76e743f864 100644 --- a/inventory/group_vars/pdc-backend +++ b/inventory/group_vars/pdc-backend @@ -20,3 +20,20 @@ fedmsg_error_recipients: - mprahl@redhat.com - jkaluza@redhat.com - fvalder@redhat.com + +csi_security_category: Moderate +csi_primary_contact: Ralph Bean +csi_purpose: fedmsg-hub daemon that ferries data from fedmsg to PDC. +csi_relationship: | + NOTICE - pdc-backend01 and pdc-backend02 do *different* things. They both run + a fedmsg-hub daemon that loads the pdc-updater consumer plugin. However, the + pdc-updater plugin is configured to do different things in each place. + + On pdc-updater01, the compose handler is enabled which listens for new pungi + composes, and stores them in PDC. Fedora QE uses this data. The consumer + has only a single thread enabled to avoid OOMing itself with more than one + compose at a time. + + On pdc-updater02, the dep chain and modularity handlers are enabled which + listen for koji and MBS activity, and store that in PDC. Multiple threads + are enabled so that it can work more efficiently on these smaller tasks. diff --git a/inventory/group_vars/pdc-backend-stg b/inventory/group_vars/pdc-backend-stg index 1eb49dc544..edf319d156 100644 --- a/inventory/group_vars/pdc-backend-stg +++ b/inventory/group_vars/pdc-backend-stg @@ -20,3 +20,20 @@ fedmsg_error_recipients: - mprahl@redhat.com - jkaluza@redhat.com - fvalder@redhat.com + +csi_security_category: Moderate +csi_primary_contact: Ralph Bean +csi_purpose: fedmsg-hub daemon that ferries data from fedmsg to PDC. +csi_relationship: | + NOTICE - pdc-backend01 and pdc-backend02 do *different* things. They both run + a fedmsg-hub daemon that loads the pdc-updater consumer plugin. However, the + pdc-updater plugin is configured to do different things in each place. + + On pdc-updater01, the compose handler is enabled which listens for new pungi + composes, and stores them in PDC. Fedora QE uses this data. The consumer + has only a single thread enabled to avoid OOMing itself with more than one + compose at a time. + + On pdc-updater02, the dep chain and modularity handlers are enabled which + listen for koji and MBS activity, and store that in PDC. Multiple threads + are enabled so that it can work more efficiently on these smaller tasks. diff --git a/inventory/host_vars/pdc-backend02.phx2.fedoraproject.org b/inventory/host_vars/pdc-backend02.phx2.fedoraproject.org new file mode 100644 index 0000000000..e8e34b3184 --- /dev/null +++ b/inventory/host_vars/pdc-backend02.phx2.fedoraproject.org @@ -0,0 +1,13 @@ +--- +nm: 255.255.255.0 +gw: 10.5.126.254 +dns: 10.5.126.21 + +ks_url: http://10.5.126.23/repo/rhel/ks/kvm-rhel-7 +ks_repo: http://10.5.126.23/repo/rhel/RHEL7-x86_64/ + +eth0_ip: 10.5.126.239 + +volgroup: /dev/vg_guests +vmhost: virthost22.phx2.fedoraproject.org +datacenter: phx2 diff --git a/inventory/host_vars/pdc-backend02.stg.phx2.fedoraproject.org b/inventory/host_vars/pdc-backend02.stg.phx2.fedoraproject.org new file mode 100644 index 0000000000..ffd2fca32d --- /dev/null +++ b/inventory/host_vars/pdc-backend02.stg.phx2.fedoraproject.org @@ -0,0 +1,13 @@ +--- +nm: 255.255.255.0 +gw: 10.5.126.254 +dns: 10.5.126.21 + +ks_url: http://10.5.126.23/repo/rhel/ks/kvm-rhel-7 +ks_repo: http://10.5.126.23/repo/rhel/RHEL7-x86_64/ + +eth0_ip: 10.5.126.240 + +volgroup: /dev/vg_guests +vmhost: virthost20.phx2.fedoraproject.org +datacenter: phx2 diff --git a/inventory/inventory b/inventory/inventory index 27f026b1dd..9c180b1421 100644 --- a/inventory/inventory +++ b/inventory/inventory @@ -757,6 +757,7 @@ nuancier02.stg.phx2.fedoraproject.org packages03.stg.phx2.fedoraproject.org paste01.stg.phx2.fedoraproject.org pdc-backend01.stg.phx2.fedoraproject.org +pdc-backend02.stg.phx2.fedoraproject.org pdc-web01.stg.phx2.fedoraproject.org pgbdr01.stg.phx2.fedoraproject.org pgbdr02.stg.phx2.fedoraproject.org @@ -1089,9 +1090,11 @@ pdc-web01.stg.phx2.fedoraproject.org [pdc-backend] pdc-backend01.phx2.fedoraproject.org +pdc-backend02.phx2.fedoraproject.org [pdc-backend-stg] pdc-backend01.stg.phx2.fedoraproject.org +pdc-backend02.stg.phx2.fedoraproject.org [piwik-stg] #piwik01.stg.phx2.fedoraproject.org diff --git a/roles/pdc/backend/templates/pdcupdater.py b/roles/pdc/backend/templates/pdcupdater.py index 82a7c6f200..3680020027 100644 --- a/roles/pdc/backend/templates/pdcupdater.py +++ b/roles/pdc/backend/templates/pdcupdater.py @@ -4,10 +4,14 @@ config = { # Should we turn on the realtime updater? 'pdcupdater.enabled': True, + {% if inventory_hostname.startswith('pdc-backend01') %} # Use only one thread at a time to handle messages. If we have more than # one, then we can end up POSTing multiple enormous JSON blobs to the PDC # web frontend, and we'll send it into OOM death. One at a time, people.. "moksha.workers_per_consumer": 1, + {% else %} + "moksha.workers_per_consumer": 8, + {% endif %} # Credentials to talk to PDC 'pdcupdater.pdc': { @@ -62,23 +66,19 @@ config = { # We have an explicit list of these in the config so we can turn them on # and off individually in production if one is causing an issue. 'pdcupdater.handlers': [ + # See CSI information on host for info on the different handlers. + {% if inventory_hostname.startswith('pdc-backend01') %} 'pdcupdater.handlers.compose:NewComposeHandler', - 'pdcupdater.handlers.atomic:AtomicComponentGroupHandler', - + {% elif inventory_hostname.startswith('pdc-backend02') %} # https://fedoraproject.org/wiki/User:Ralph/Drafts/Infrastructure/Factory2/ModellingDeps 'pdcupdater.handlers.depchain.rpms:NewRPMBuildTimeDepChainHandler', 'pdcupdater.handlers.depchain.rpms:NewRPMRunTimeDepChainHandler', - # The new version of PDC broke all of these. There are different - # endpoints, etc. - ##'pdcupdater.handlers.pkgdb:NewPackageHandler', - ##'pdcupdater.handlers.pkgdb:NewPackageBranchHandler', - ##'pdcupdater.handlers.rpms:NewRPMHandler', - ##'pdcupdater.handlers.persons:NewPersonHandler', - - {% if env == 'staging' %} # For MBS https://fedoraproject.org/wiki/Changes/ModuleBuildService 'pdcupdater.handlers.modules:ModuleStateChangeHandler', + + # This just isn't used for now.. but we can re-enable it if we need it. + #'pdcupdater.handlers.atomic:AtomicComponentGroupHandler', {% endif %} ],