Infrastructure/ansible
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

mirrorlist_server: reduce memory usage considerably

Browse source 
Rather than loading in the entire 500k entries of the global netblock
    list, only load in the ~30k entries that match ASNs that our mirrors
    have explicitly claimed as being relevant to them, either as the
    Host.asn or in the HostPeerASN list.

    This has the effect of reducing memory size by about half for the
    overall program, which, should improve fork() time and reduce failure
    to insufficient RAM available.
This commit is contained in:
Matt Domsch 2013-06-15 03:18:00 +00:00
parent b10428ccc9
commit fc3cd53252
1 changed files with 5 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

10
files/mirrorlist/mirrorlist_server.py
View file

@ -684,7 +684,7 @@ def setup_cache_tree(cache, field):
node.data[field] = v
return tree
def setup_netblocks(netblocks_file):
def setup_netblocks(netblocks_file, asns_wanted=None):
tree = radix.Radix()
if netblocks_file is not None:
try:
@ -698,12 +698,12 @@ def setup_netblocks(netblocks_file):
mask = int(mask)
if mask == 0: continue
asn = int(s[1])
node = tree.add(s[0])
node.data['asn'] = asn
if asns_wanted is None or asn in asns_wanted:
node = tree.add(s[0])
node.data['asn'] = asn
except:
pass
f.close()
return tree
def read_caches():
@ -769,7 +769,7 @@ def read_caches():
global netblock_country_tree
internet2_tree = setup_netblocks(internet2_netblocks_file)
global_tree = setup_netblocks(global_netblocks_file)
global_tree = setup_netblocks(global_netblocks_file, asn_host_cache)
# host_netblocks_tree key is a netblock, value is a list of host IDs
host_netblocks_tree = setup_cache_tree(host_netblock_cache, 'hosts')
# netblock_country_tree key is a netblock, value is a single country string