From fc3cd53252c641293ef43de70dce50f0331646cd Mon Sep 17 00:00:00 2001 From: Matt Domsch Date: Sat, 15 Jun 2013 03:18:00 +0000 Subject: [PATCH] mirrorlist_server: reduce memory usage considerably Rather than loading in the entire 500k entries of the global netblock list, only load in the ~30k entries that match ASNs that our mirrors have explicitly claimed as being relevant to them, either as the Host.asn or in the HostPeerASN list. This has the effect of reducing memory size by about half for the overall program, which, should improve fork() time and reduce failure to insufficient RAM available. --- files/mirrorlist/mirrorlist_server.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/files/mirrorlist/mirrorlist_server.py b/files/mirrorlist/mirrorlist_server.py index eb328606da..7a73b6aaa3 100755 --- a/files/mirrorlist/mirrorlist_server.py +++ b/files/mirrorlist/mirrorlist_server.py @@ -684,7 +684,7 @@ def setup_cache_tree(cache, field): node.data[field] = v return tree -def setup_netblocks(netblocks_file): +def setup_netblocks(netblocks_file, asns_wanted=None): tree = radix.Radix() if netblocks_file is not None: try: @@ -698,12 +698,12 @@ def setup_netblocks(netblocks_file): mask = int(mask) if mask == 0: continue asn = int(s[1]) - node = tree.add(s[0]) - node.data['asn'] = asn + if asns_wanted is None or asn in asns_wanted: + node = tree.add(s[0]) + node.data['asn'] = asn except: pass f.close() - return tree def read_caches(): @@ -769,7 +769,7 @@ def read_caches(): global netblock_country_tree internet2_tree = setup_netblocks(internet2_netblocks_file) - global_tree = setup_netblocks(global_netblocks_file) + global_tree = setup_netblocks(global_netblocks_file, asn_host_cache) # host_netblocks_tree key is a netblock, value is a list of host IDs host_netblocks_tree = setup_cache_tree(host_netblock_cache, 'hosts') # netblock_country_tree key is a netblock, value is a single country string