From 6722cab41003f4ddd60849703715a710c3214f8d Mon Sep 17 00:00:00 2001 From: Adrian Reber Date: Thu, 15 Oct 2015 14:03:02 +0000 Subject: [PATCH] Increase the number of possible child processes The mirrorlist-server is the process which has the mirrorlist data loaded and which is accessed by the public facing mirrorlist_client.wsgi. The mirrorlist-server uses the ForkingUnixStreamServer which has a default of max_children = 40. (https://hg.python.org/cpython/file/2.7/Lib/SocketServer.py#l516) Looking at the code of ForkingUnixStreamServer it says at https://hg.python.org/cpython/file/2.7/Lib/SocketServer.py#l523 # If we're above the max number of children, wait and reap them until # we go back below threshold. Note that we use waitpid(-1) below to be # able to collect children in size() syscalls instead # of size(): the downside is that this might reap children # which we didn't spawn, which is why we only resort to this when we're # above max_children. As we are running the wsgi with processes=45 this sounds like it can lead to situation where it might just hang. This increases max_children to 80 and processes to 60. Signed-off-by: Adrian Reber --- files/hotfix/mirrorlist/mirrorlist_server.py | 1 + inventory/group_vars/mirrorlist2 | 2 +- inventory/group_vars/mirrorlist2-stg | 2 +- roles/mirrormanager/mirrorlist2/tasks/main.yml | 9 +++++++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/files/hotfix/mirrorlist/mirrorlist_server.py b/files/hotfix/mirrorlist/mirrorlist_server.py index 2d98fa0fdb..2b8191257e 100644 --- a/files/hotfix/mirrorlist/mirrorlist_server.py +++ b/files/hotfix/mirrorlist/mirrorlist_server.py @@ -938,6 +938,7 @@ def sigterm_handler(signum, frame): class ForkingUnixStreamServer(ForkingMixIn, UnixStreamServer): request_queue_size = 300 + max_children = 80 def finish_request(self, request, client_address): signal.signal(signal.SIGHUP, signal.SIG_IGN) BaseServer.finish_request(self, request, client_address) diff --git a/inventory/group_vars/mirrorlist2 b/inventory/group_vars/mirrorlist2 index 42138a399c..f0d5655172 100644 --- a/inventory/group_vars/mirrorlist2 +++ b/inventory/group_vars/mirrorlist2 @@ -23,7 +23,7 @@ fas_client_groups: sysadmin-noc,fi-apprentice nrpe_procs_warn: 500 nrpe_procs_crit: 600 # By default run 45 wsgi procs -mirrorlist_procs: 45 +mirrorlist_procs: 60 # Set this to get the vpn postfix setup postfix_group: vpn diff --git a/inventory/group_vars/mirrorlist2-stg b/inventory/group_vars/mirrorlist2-stg index 8e42b4f5e8..257013a545 100644 --- a/inventory/group_vars/mirrorlist2-stg +++ b/inventory/group_vars/mirrorlist2-stg @@ -14,7 +14,7 @@ fas_client_groups: sysadmin-noc,fi-apprentice,sysadmin-web nrpe_procs_warn: 500 nrpe_procs_crit: 600 # By default run 45 wsgi procs -mirrorlist_procs: 45 +mirrorlist_procs: 60 # Set this to get the vpn postfix setup postfix_group: vpn diff --git a/roles/mirrormanager/mirrorlist2/tasks/main.yml b/roles/mirrormanager/mirrorlist2/tasks/main.yml index 786432f936..2e5f5ee054 100644 --- a/roles/mirrormanager/mirrorlist2/tasks/main.yml +++ b/roles/mirrormanager/mirrorlist2/tasks/main.yml @@ -73,3 +73,12 @@ # tags: # - mirrorlist2 # - selinux + + +- name: HOTFIX - increase the number of possible child processes + copy: src="{{ files }}/hotfix/mirrorlist/mirrorlist_server.py" dest=/usr/share/mirrormanager2/mirrorlist_server.py + owner=root group=root mode=0755 + tags: + - hotfix + notify: + - restart mirrorlist-server