From fa46e8bc0e5daeffb772527d45efe6a6ba1ea6fe Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Sun, 29 Dec 2019 15:33:01 +0000 Subject: [PATCH] koji_hub: switch to event workers in apache and bump limits way up. By default apache uses prefork and a limit of 250. It's possible that this limit was the thing causing us issues over the last week. This moves to the event mpm and ups limits a lot. It also needs to up limits on db connections or the increased workers will just cause the db server to overload. With this setup, builders are no longer dropping out, but it's not clear if it's solved all the issues we have been seeing. Signed-off-by: Kevin Fenzi --- roles/koji_hub/files/00-mpm.conf | 18 ++++++++++++++++++ roles/koji_hub/tasks/main.yml | 8 ++++++++ roles/koji_hub/templates/kojiweb.conf.j2 | 5 +++++ .../templates/postgresql.conf | 2 +- 4 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 roles/koji_hub/files/00-mpm.conf diff --git a/roles/koji_hub/files/00-mpm.conf b/roles/koji_hub/files/00-mpm.conf new file mode 100644 index 0000000000..f349d67269 --- /dev/null +++ b/roles/koji_hub/files/00-mpm.conf @@ -0,0 +1,18 @@ +# Select the MPM module which should be used by uncommenting exactly +# one of the following LoadModule lines: + +# prefork MPM: Implements a non-threaded, pre-forking web server +# See: http://httpd.apache.org/docs/2.4/mod/prefork.html +#LoadModule mpm_prefork_module modules/mod_mpm_prefork.so + +# worker MPM: Multi-Processing Module implementing a hybrid +# multi-threaded multi-process web server +# See: http://httpd.apache.org/docs/2.4/mod/worker.html +# +#LoadModule mpm_worker_module modules/mod_mpm_worker.so + +# event MPM: A variant of the worker MPM with the goal of consuming +# threads only for connections with active processing +# See: http://httpd.apache.org/docs/2.4/mod/event.html +# +LoadModule mpm_event_module modules/mod_mpm_event.so diff --git a/roles/koji_hub/tasks/main.yml b/roles/koji_hub/tasks/main.yml index e20947083e..c80f5ef0f6 100644 --- a/roles/koji_hub/tasks/main.yml +++ b/roles/koji_hub/tasks/main.yml @@ -43,6 +43,14 @@ # https://lists.fedoraproject.org/pipermail/buildsys/2015-April/004636.html when: env == 'staging' +- name: set the apache mpm to use event MPM + copy: src=00-mpm.conf dest=/etc/httpd/conf.modules.d/00-mpm.conf + notify: + - reload proxyhttpd + tags: + - config + - koji_hub + - name: hub config template: src=hub.conf.j2 dest=/etc/koji-hub/hub.conf owner=apache group=apache mode=600 tags: diff --git a/roles/koji_hub/templates/kojiweb.conf.j2 b/roles/koji_hub/templates/kojiweb.conf.j2 index f43efd4b6e..cdbfde9561 100644 --- a/roles/koji_hub/templates/kojiweb.conf.j2 +++ b/roles/koji_hub/templates/kojiweb.conf.j2 @@ -2,6 +2,11 @@ RewriteEngine On RewriteRule ^/$ /koji/ [R,L] KeepAlive On +StartServers 150 +ServerLimit 900 +MaxRequestWorkers 900 +MaxRequestsPerChild 10000 + #We use wsgi by default Alias /koji "/usr/share/koji-web/scripts/wsgi_publisher.py" #(configuration goes in /etc/kojiweb/web.conf) diff --git a/roles/postgresql_server/templates/postgresql.conf b/roles/postgresql_server/templates/postgresql.conf index 4b26ec6cea..bfc9727ad9 100644 --- a/roles/postgresql_server/templates/postgresql.conf +++ b/roles/postgresql_server/templates/postgresql.conf @@ -65,7 +65,7 @@ listen_addresses = '*' # what IP address(es) to listen on; # defaults to 'localhost', '*' = all # (change requires restart) port = 5432 # (change requires restart) -max_connections = 750 # (change requires restart) +max_connections = 2000 # (change requires restart) # Note: Increasing max_connections costs ~400 bytes of shared memory per # connection slot, plus lock space (see max_locks_per_transaction). You might # also need to raise shared_buffers to support more connections.