From 726ccd2929d01bce2bf371da22cb2dd117d1bcfe Mon Sep 17 00:00:00 2001 From: Matt Domsch Date: Mon, 17 Jun 2013 19:16:13 +0000 Subject: [PATCH] remove MM hotfixes, included in 1.4.1 --- files/mirrorlist/mirrorlist_server.py | 1044 ------------------------- tasks/mirrorlist.yml | 6 - 2 files changed, 1050 deletions(-) delete mode 100755 files/mirrorlist/mirrorlist_server.py diff --git a/files/mirrorlist/mirrorlist_server.py b/files/mirrorlist/mirrorlist_server.py deleted file mode 100755 index 7a73b6aaa3..0000000000 --- a/files/mirrorlist/mirrorlist_server.py +++ /dev/null @@ -1,1044 +0,0 @@ -#!/usr/bin/python -# -# Copyright (c) 2007-2013 Dell, Inc. -# by Matt Domsch -# Licensed under the MIT/X11 license - -# standard library modules in alphabetical order -import datetime -import getopt -import logging -import logging.handlers -import os -import random -import cPickle as pickle -import select -import signal -import socket -from SocketServer import StreamRequestHandler, ForkingMixIn, UnixStreamServer, BaseServer -import sys -from string import zfill, atoi -import time -import traceback - -try: - import threading -except ImportError: - import dummy_threading as threading - -# not-so-standard library modules that this program needs -from IPy import IP -import GeoIP -import radix -from weighted_shuffle import weighted_shuffle - -# can be overridden on the command line -pidfile = '/var/run/mirrormanager/mirrorlist_server.pid' -socketfile = '/var/run/mirrormanager/mirrorlist_server.sock' -cachefile = '/var/lib/mirrormanager/mirrorlist_cache.pkl' -internet2_netblocks_file = '/var/lib/mirrormanager/i2_netblocks.txt' -global_netblocks_file = '/var/lib/mirrormanager/global_netblocks.txt' -logfile = None -debug = False -must_die = False -# at a point in time when we're no longer serving content for versions -# that don't use yum prioritymethod=fallback -# (e.g. after Fedora 7 is past end-of-life) -# then we can set this value to True -# this only affects results requested using path=... -# for dirs which aren't repositories (such as iso/) -# because we don't know the Version associated with that dir here. -default_ordered_mirrorlist = False - -gipv4 = None -gipv6 = None - -# key is strings in tuple (repo.prefix, arch) -mirrorlist_cache = {} - -# key is directory.name, returns keys for mirrorlist_cache -directory_name_to_mirrorlist = {} - -# key is an IPy.IP structure, value is list of host ids -host_netblock_cache = {} - -# key is hostid, value is list of countries to allow -host_country_allowed_cache = {} - -repo_arch_to_directoryname = {} - -# redirect from a repo with one name to a repo with another -repo_redirect = {} -country_continent_redirect_cache = {} - -# our own private copy of country_continents to be edited -country_continents = GeoIP.country_continents - -disabled_repositories = {} -host_bandwidth_cache = {} -host_country_cache = {} -host_max_connections_cache = {} -file_details_cache = {} -hcurl_cache = {} -asn_host_cache = {} -internet2_tree = radix.Radix() -global_tree = radix.Radix() -host_netblocks_tree = radix.Radix() -netblock_country_tree = radix.Radix() -location_cache = {} -netblock_country_cache = {} - -## Set up our syslog data. -syslogger = logging.getLogger('mirrormanager') -syslogger.setLevel(logging.INFO) -handler = logging.handlers.SysLogHandler(address='/dev/log', facility=logging.handlers.SysLogHandler.LOG_LOCAL4) -syslogger.addHandler(handler) - -def lookup_ip_asn(tree, ip): - """ @t is a radix tree - @ip is an IPy.IP object which may be contained in an entry in l - """ - node = tree.search_best(ip.strNormal()) - if node is None: - return None - return node.data['asn'] - - -def uniqueify(seq, idfun=None): - # order preserving - if idfun is None: - def idfun(x): return x - seen = {} - result = [] - for item in seq: - marker = idfun(item) - # in old Python versions: - # if seen.has_key(marker) - # but in new ones: - if marker in seen: continue - seen[marker] = 1 - result.append(item) - return result - -##### Metalink Support ##### - -def metalink_header(): - # fixme add alternate format pubdate when specified - pubdate = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT") - doc = '' - doc += '\n' - doc += '\n' - doc += indent(2) + '\n' % (file) - y = detailslist[0] - - def details(y, indentlevel=2): - doc = '' - if y['timestamp'] is not None: - doc += indent(indentlevel+1) + '%s\n' % y['timestamp'] - if y['size'] is not None: - doc += indent(indentlevel+1) + '%s\n' % y['size'] - doc += indent(indentlevel+1) + '\n' - hashes = ('md5', 'sha1', 'sha256', 'sha512') - for h in hashes: - if y[h] is not None: - doc += indent(indentlevel+2) + '%s\n' % (h, y[h]) - doc += indent(indentlevel+1) + '\n' - return doc - - doc += details(y, 2) - # there can be multiple files - if len(detailslist) > 1: - doc += indent(3) + '\n' - for y in detailslist[1:]: - doc += indent(4) + '\n' - doc += details(y,5) - doc += indent(4) + '\n' - doc += indent(3) + '\n' - - doc += indent(3) + '\n' - for (hostid, hcurls) in hosts_and_urls: - private = '' - if hostid not in cache['global']: - private = 'mm0:private="True"' - for url in hcurls: - protocol = url.split(':')[0] - # FIXME January 2010 - # adding protocol= here is not part of the Metalink 3.0 spec, - # but MirrorManager 1.2.6 used it accidentally, as did yum 3.2.20-3 as released - # in Fedora 8, 9, and 10. After those three are EOL (~January 2010), the - # extra protocol= can be removed. - doc += indent(4) + '' % (protocol, protocol, host_country_cache[hostid].upper(), preference, private) - doc += url - doc += '\n' - preference = max(preference-1, 1) - doc += indent(3) + '\n' - doc += indent(2) + '\n' - doc += indent(1) + '\n' - doc += '\n' - return ('metalink', 200, doc) - -def tree_lookup(tree, ip, field, maxResults=None): - # fast lookup in the tree; if present, find all the matching values by deleting the found one and searching again - # this is safe w/o copying the tree again only because this is the only place the tree is used, and - # we'll get a new copy of the tree from our parent the next time it fork()s. - # returns a list of tuples (prefix, data) - result = [] - len_data = 0 - if ip is None: - return result - node = tree.search_best(ip.strNormal()) - while node is not None: - prefix = node.prefix - if type(node.data[field]) == list: - len_data += len(node.data[field]) - else: - len_data += 1 - t = (prefix, node.data[field],) - result.append(t) - if maxResults is None or len_data < maxResults: - tree.delete(prefix) - node = tree.search_best(ip.strNormal()) - else: - break - return result - -def trim_by_client_country(s, clientCountry): - if clientCountry is None: - return s - r = s.copy() - for hostid in s: - if hostid in host_country_allowed_cache and \ - clientCountry not in host_country_allowed_cache[hostid]: - r.remove(hostid) - return r - -def shuffle(s): - l = [] - for hostid in s: - item = (host_bandwidth_cache[hostid], hostid) - l.append(item) - newlist = weighted_shuffle(l) - results = [] - for (bandwidth, hostid) in newlist: - results.append(hostid) - return results - -continents = {} - -def handle_country_continent_redirect(): - new_country_continents = GeoIP.country_continents - for country, continent in country_continent_redirect_cache.iteritems(): - new_country_continents[country] = continent - global country_continents - country_continents = new_country_continents - -def setup_continents(): - new_continents = {} - handle_country_continent_redirect() - for c in country_continents.keys(): - continent = country_continents[c] - if continent not in new_continents: - new_continents[continent] = [c] - else: - new_continents[continent].append(c) - global continents - continents = new_continents - -def do_global(kwargs, cache, clientCountry, header): - c = trim_by_client_country(cache['global'], clientCountry) - header += 'country = global ' - return (header, c) - -def do_countrylist(kwargs, cache, clientCountry, requested_countries, header): - - def collapse(d): - """ collapses a dict {key:set(hostids)} into a set of hostids """ - s = set() - for country, hostids in d.iteritems(): - for hostid in hostids: - s.add(hostid) - return s - - country_cache = {} - for c in requested_countries: - if c in cache['byCountry']: - country_cache[c] = cache['byCountry'][c] - header += 'country = %s ' % c - s = collapse(country_cache) - s = trim_by_client_country(s, clientCountry) - return (header, s) - -def get_same_continent_countries(clientCountry, requested_countries): - result = [] - for r in requested_countries: - if r in country_continents: - requestedCountries = [c.upper() for c in continents[country_continents[r]] \ - if c != clientCountry ] - result.extend(requestedCountries) - uniqueify(result) - return result - -def do_continent(kwargs, cache, clientCountry, requested_countries, header): - if len(requested_countries) > 0: - rc = requested_countries - else: - rc = [clientCountry] - clist = get_same_continent_countries(clientCountry, rc) - return do_countrylist(kwargs, cache, clientCountry, clist, header) - -def do_country(kwargs, cache, clientCountry, requested_countries, header): - if 'GLOBAL' in requested_countries: - return do_global(kwargs, cache, clientCountry, header) - return do_countrylist(kwargs, cache, clientCountry, requested_countries, header) - -def do_netblocks(kwargs, cache, header): - hostresults = set() - if not kwargs.has_key('netblock') or kwargs['netblock'] == "1": - tree_results = tree_lookup(host_netblocks_tree, kwargs['IP'], 'hosts') - for (prefix, hostids) in tree_results: - for hostid in hostids: - if hostid in cache['byHostId']: - hostresults.add((prefix, hostid,)) - header += 'Using preferred netblock ' - return (header, hostresults) - -def do_internet2(kwargs, cache, clientCountry, header): - hostresults = set() - ip = kwargs['IP'] - if ip is None: - return (header, hostresults) - asn = lookup_ip_asn(internet2_tree, ip) - if asn is not None: - header += 'Using Internet2 ' - if clientCountry is not None and clientCountry in cache['byCountryInternet2']: - hostresults = cache['byCountryInternet2'][clientCountry] - hostresults = trim_by_client_country(hostresults, clientCountry) - return (header, hostresults) - -def do_asn(kwargs, cache, header): - hostresults = set() - ip = kwargs['IP'] - if ip is None: - return (header, hostresults) - asn = lookup_ip_asn(global_tree, ip) - if asn is not None and asn in asn_host_cache: - for hostid in asn_host_cache[asn]: - if hostid in cache['byHostId']: - hostresults.add(hostid) - header += 'Using ASN %s ' % asn - return (header, hostresults) - -def do_geoip(kwargs, cache, clientCountry, header): - hostresults = set() - if clientCountry is not None and clientCountry in cache['byCountry']: - hostresults = cache['byCountry'][clientCountry] - header += 'country = %s ' % clientCountry - hostresults = trim_by_client_country(hostresults, clientCountry) - return (header, hostresults) - -def do_location(kwargs, header): - hostresults = set() - if 'location' in kwargs and kwargs['location'] in location_cache: - hostresults = set(location_cache[kwargs['location']]) - header += "Using location %s " % kwargs['location'] - return (header, hostresults) - -def add_host_to_cache(cache, hostid, hcurl): - if hostid not in cache: - cache[hostid] = [hcurl] - else: - cache[hostid].append(hcurl) - return cache - -def append_path(hosts, cache, file, pathIsDirectory=False): - """ given a list of hosts, return a list of objects: - [(hostid, [hcurls]), ... ] - in the same order, appending file if it's not None""" - subpath = None - results = [] - if 'subpath' in cache: - subpath = cache['subpath'] - for hostid in hosts: - hcurls = [] - for hcurl_id in cache['byHostId'][hostid]: - s = hcurl_cache[hcurl_id] - if subpath is not None: - s += "/" + subpath - if file is None and pathIsDirectory: - s += "/" - if file is not None: - if not s.endswith('/'): - s += "/" - s += file - hcurls.append(s) - results.append((hostid, hcurls)) - return results - -def trim_to_preferred_protocols(hosts_and_urls): - """ remove all but http and ftp URLs, - and if both http and ftp are offered, - leave only http. Return [(hostid, url), ...] """ - results = [] - try_protocols = ('https', 'http', 'ftp') - for (hostid, hcurls) in hosts_and_urls: - protocols = {} - url = None - for hcurl in hcurls: - for p in try_protocols: - if hcurl.startswith(p+':'): - protocols[p] = hcurl - - for p in try_protocols: - if p in protocols: - url = protocols[p] - break - - if url is not None: - results.append((hostid, url)) - return results - -def client_ip_to_country(ip): - clientCountry = None - if ip is None: - return None - - # lookup in the cache first - tree_results = tree_lookup(netblock_country_tree, ip, 'country', maxResults=1) - if len(tree_results) > 0: - (prefix, clientCountry) = tree_results[0] - return clientCountry - - # attempt IPv6, then IPv6 6to4 as IPv4, then Teredo, then IPv4 - try: - if ip.version() == 6: - if gipv6 is not None: - clientCountry = gipv6.country_code_by_addr_v6(ip.strNormal()) - if clientCountry is None: - # Try the IPv6-to-IPv4 translation schemes - for scheme in (convert_6to4_v4, convert_teredo_v4): - result = scheme(ip) - if result is not None: - ip = result - break - if ip.version() == 4 and gipv4 is not None: - clientCountry = gipv4.country_code_by_addr(ip.strNormal()) - except: - pass - return clientCountry - -def do_mirrorlist(kwargs): - global debug - global logfile - - def return_error(kwargs, message='', returncode=200): - d = dict(returncode=returncode, message=message, resulttype='mirrorlist', results=[]) - if 'metalink' in kwargs and kwargs['metalink']: - d['resulttype'] = 'metalink' - d['results'] = metalink_failuredoc(message) - return d - - if not (kwargs.has_key('repo') and kwargs.has_key('arch')) and not kwargs.has_key('path'): - return return_error(kwargs, message='# either path=, or repo= and arch= must be specified') - - file = None - cache = None - pathIsDirectory = False - if kwargs.has_key('path'): - path = kwargs['path'].strip('/') - - # Strip duplicate "//" from the path - path = path.replace('//', '/') - - header = "# path = %s " % (path) - - sdir = path.split('/') - try: - # path was to a directory - cache = mirrorlist_cache['/'.join(sdir)] - pathIsDirectory=True - except KeyError: - # path was to a file, try its directory - file = sdir[-1] - sdir = sdir[:-1] - try: - cache = mirrorlist_cache['/'.join(sdir)] - except KeyError: - return return_error(kwargs, message=header + 'error: invalid path') - dir = '/'.join(sdir) - else: - if u'source' in kwargs['repo']: - kwargs['arch'] = u'source' - repo = repo_redirect.get(kwargs['repo'], kwargs['repo']) - arch = kwargs['arch'] - header = "# repo = %s arch = %s " % (repo, arch) - - if repo in disabled_repositories: - return return_error(kwargs, message=header + 'repo disabled') - try: - dir = repo_arch_to_directoryname[(repo, arch)] - if 'metalink' in kwargs and kwargs['metalink']: - dir += '/repodata' - file = 'repomd.xml' - else: - pathIsDirectory=True - cache = mirrorlist_cache[dir] - except KeyError: - repos = repo_arch_to_directoryname.keys() - repos.sort() - repo_information = header + "error: invalid repo or arch\n" - repo_information += "# following repositories are available:\n" - for i in repos: - if i[0] is not None and i[1] is not None: - repo_information += "# repo=%s&arch=%s\n" % i - return return_error(kwargs, message=repo_information) - - # set kwargs['IP'] exactly once - try: - kwargs['IP'] = IP(kwargs['client_ip']) - except: - kwargs['IP'] = None - - ordered_mirrorlist = cache.get('ordered_mirrorlist', default_ordered_mirrorlist) - done = 0 - location_results = set() - netblock_results = set() - asn_results = set() - internet2_results = set() - country_results = set() - geoip_results = set() - continent_results = set() - global_results = set() - - header, location_results = do_location(kwargs, header) - - requested_countries = [] - if kwargs.has_key('country'): - requested_countries = uniqueify([c.upper() for c in kwargs['country'].split(',') ]) - - # if they specify a country, don't use netblocks or ASN - if not 'country' in kwargs: - header, netblock_results = do_netblocks(kwargs, cache, header) - if len(netblock_results) > 0: - if not ordered_mirrorlist: - done=1 - - if not done: - header, asn_results = do_asn(kwargs, cache, header) - if len(asn_results) + len(netblock_results) >= 3: - if not ordered_mirrorlist: - done = 1 - - clientCountry = client_ip_to_country(kwargs['IP']) - - if clientCountry is None: - print_client_country = "N/A" - else: - print_client_country = clientCountry - - if debug and kwargs.has_key('repo') and kwargs.has_key('arch'): - msg = "IP: %s; DATE: %s; COUNTRY: %s; REPO: %s; ARCH: %s\n" % ( - (kwargs['IP'] or 'None'), time.strftime("%Y-%m-%d"), - print_client_country, kwargs['repo'], kwargs['arch']) - - sys.stdout.write(msg) - sys.stdout.flush() - - if logfile is not None: - logfile.write(msg) - logfile.flush() - - if not done: - header, internet2_results = do_internet2(kwargs, cache, clientCountry, header) - if len(internet2_results) + len(netblock_results) + len(asn_results) >= 3: - if not ordered_mirrorlist: - done = 1 - - if not done and 'country' in kwargs: - header, country_results = do_country(kwargs, cache, clientCountry, requested_countries, header) - if len(country_results) == 0: - header, continent_results = do_continent(kwargs, cache, clientCountry, requested_countries, header) - done = 1 - - if not done: - header, geoip_results = do_geoip(kwargs, cache, clientCountry, header) - if len(geoip_results) >= 3: - if not ordered_mirrorlist: - done = 1 - - if not done: - header, continent_results = do_continent(kwargs, cache, clientCountry, [], header) - if len(geoip_results) + len(continent_results) >= 3: - done = 1 - - if not done: - header, global_results = do_global(kwargs, cache, clientCountry, header) - - def _random_shuffle(s): - l = list(s) - random.shuffle(l) - return l - - def _ordered_netblocks(s): - def ipy_len(t): - (prefix, hostid) = t - return IP(prefix).len() - v4_netblocks = [] - v6_netblocks = [] - for (prefix, hostid) in s: - ip = IP(prefix) - if ip.version() == 4: - v4_netblocks.append((prefix, hostid)) - elif ip.version() == 6: - v6_netblocks.append((prefix, hostid)) - # mix up the order, as sort will preserve same-key ordering - random.shuffle(v4_netblocks) - v4_netblocks.sort(key=ipy_len) - random.shuffle(v6_netblocks) - v6_netblocks.sort(key=ipy_len) - v4_netblocks = [t[1] for t in v4_netblocks] - v6_netblocks = [t[1] for t in v6_netblocks] - return v6_netblocks + v4_netblocks - - def whereismymirror(result_sets): - return_string = 'None' - allhosts = [] - found = False - for (l,s,f) in result_sets: - if len(l) > 0: - allhosts.extend(f(l)) - if not found: - return_string = s - found = True - - allhosts = uniqueify(allhosts) - return allhosts, return_string - - result_sets = [ - (location_results, "location", _random_shuffle), - (netblock_results, "netblocks", _ordered_netblocks), - (asn_results, "asn", _random_shuffle), - (internet2_results, "I2", _random_shuffle), - (country_results, "country", shuffle), - (geoip_results, "geoip", shuffle), - (continent_results, "continent", shuffle), - (global_results, "global", shuffle), - ] - - allhosts, where_string = whereismymirror(result_sets) - try: - ip_str = kwargs['IP'].strNormal() - except: - ip_str = 'Unknown IP' - log_string = "mirrorlist: %s found its best mirror from %s" % (ip_str, where_string) - syslogger.info(log_string) - - hosts_and_urls = append_path(allhosts, cache, file, pathIsDirectory=pathIsDirectory) - - if 'metalink' in kwargs and kwargs['metalink']: - (resulttype, returncode, results)=metalink(cache, dir, file, hosts_and_urls) - d = dict(message=None, resulttype=resulttype, returncode=returncode, results=results) - return d - - else: - host_url_list = trim_to_preferred_protocols(hosts_and_urls) - d = dict(message=header, resulttype='mirrorlist', returncode=200, results=host_url_list) - return d - -def setup_cache_tree(cache, field): - tree = radix.Radix() - for k, v in cache.iteritems(): - node = tree.add(k.strNormal()) - node.data[field] = v - return tree - -def setup_netblocks(netblocks_file, asns_wanted=None): - tree = radix.Radix() - if netblocks_file is not None: - try: - f = open(netblocks_file, 'r') - except: - return tree - for l in f: - try: - s = l.split() - start, mask = s[0].split('/') - mask = int(mask) - if mask == 0: continue - asn = int(s[1]) - if asns_wanted is None or asn in asns_wanted: - node = tree.add(s[0]) - node.data['asn'] = asn - except: - pass - f.close() - return tree - -def read_caches(): - global mirrorlist_cache - global host_netblock_cache - global host_country_allowed_cache - global host_max_connections_cache - global repo_arch_to_directoryname - global repo_redirect - global country_continent_redirect_cache - global disabled_repositories - global host_bandwidth_cache - global host_country_cache - global file_details_cache - global hcurl_cache - global asn_host_cache - global location_cache - global netblock_country_cache - - data = {} - try: - f = open(cachefile, 'r') - data = pickle.load(f) - f.close() - except: - pass - - if 'mirrorlist_cache' in data: - mirrorlist_cache = data['mirrorlist_cache'] - if 'host_netblock_cache' in data: - host_netblock_cache = data['host_netblock_cache'] - if 'host_country_allowed_cache' in data: - host_country_allowed_cache = data['host_country_allowed_cache'] - if 'repo_arch_to_directoryname' in data: - repo_arch_to_directoryname = data['repo_arch_to_directoryname'] - if 'repo_redirect_cache' in data: - repo_redirect = data['repo_redirect_cache'] - if 'country_continent_redirect_cache' in data: - country_continent_redirect_cache = data['country_continent_redirect_cache'] - if 'disabled_repositories' in data: - disabled_repositories = data['disabled_repositories'] - if 'host_bandwidth_cache' in data: - host_bandwidth_cache = data['host_bandwidth_cache'] - if 'host_country_cache' in data: - host_country_cache = data['host_country_cache'] - if 'file_details_cache' in data: - file_details_cache = data['file_details_cache'] - if 'hcurl_cache' in data: - hcurl_cache = data['hcurl_cache'] - if 'asn_host_cache' in data: - asn_host_cache = data['asn_host_cache'] - if 'location_cache' in data: - location_cache = data['location_cache'] - if 'netblock_country_cache' in data: - netblock_country_cache = data['netblock_country_cache'] - if 'host_max_connections_cache' in data: - host_max_connections_cache = data['host_max_connections_cache'] - - setup_continents() - global internet2_tree - global global_tree - global host_netblocks_tree - global netblock_country_tree - - internet2_tree = setup_netblocks(internet2_netblocks_file) - global_tree = setup_netblocks(global_netblocks_file, asn_host_cache) - # host_netblocks_tree key is a netblock, value is a list of host IDs - host_netblocks_tree = setup_cache_tree(host_netblock_cache, 'hosts') - # netblock_country_tree key is a netblock, value is a single country string - netblock_country_tree = setup_cache_tree(netblock_country_cache, 'country') - -def errordoc(metalink, message): - if metalink: - doc = metalink_failuredoc(message) - else: - doc = message - return doc - -class MirrorlistHandler(StreamRequestHandler): - def handle(self): - signal.signal(signal.SIGHUP, signal.SIG_IGN) - random.seed() - try: - # read size of incoming pickle - readlen = 0 - size = '' - while readlen < 10: - size += self.rfile.read(10 - readlen) - readlen = len(size) - size = atoi(size) - - # read the pickle - readlen = 0 - p = '' - while readlen < size: - p += self.rfile.read(size - readlen) - readlen = len(p) - d = pickle.loads(p) - self.connection.shutdown(socket.SHUT_RD) - except: - pass - - try: - try: - r = do_mirrorlist(d) - except: - raise - message = r['message'] - results = r['results'] - resulttype = r['resulttype'] - returncode = r['returncode'] - except Exception, e: - message=u'# Bad Request %s\n# %s' % (e, d) - exception_msg = traceback.format_exc(e) - sys.stderr.write(message+'\n') - sys.stderr.write(exception_msg) - sys.stderr.flush() - returncode = 400 - results = [] - resulttype = 'mirrorlist' - if d['metalink']: - resulttype = 'metalink' - results = errordoc(d['metalink'], message) - - try: - p = pickle.dumps({'message':message, 'resulttype':resulttype, 'results':results, 'returncode':returncode}) - self.connection.sendall(zfill('%s' % len(p), 10)) - - self.connection.sendall(p) - self.connection.shutdown(socket.SHUT_WR) - except: - pass - -def sighup_handler(signum, frame): - global logfile - if logfile is not None: - name = logfile.name - logfile.close() - logfile = open(name, 'a') - - # put this in a separate thread so it doesn't block clients - if threading.active_count() < 2: - thread = threading.Thread(target=load_databases_and_caches) - thread.daemon = False - try: - thread.start() - except KeyError: - # bug fix for handing an exception when unable to delete from _limbo even though it's not in limbo - # https://code.google.com/p/googleappengine/source/browse/trunk/python/google/appengine/dist27/threading.py?r=327 - pass - -def sigterm_handler(signum, frame): - global must_die - signal.signal(signal.SIGHUP, signal.SIG_IGN) - signal.signal(signal.SIGTERM, signal.SIG_IGN) - if signum == signal.SIGTERM: - must_die = True - -class ForkingUnixStreamServer(ForkingMixIn, UnixStreamServer): - request_queue_size = 300 - def finish_request(self, request, client_address): - signal.signal(signal.SIGHUP, signal.SIG_IGN) - BaseServer.finish_request(self, request, client_address) - -def parse_args(): - global cachefile - global socketfile - global internet2_netblocks_file - global global_netblocks_file - global debug - global logfile - global pidfile - opts, args = getopt.getopt(sys.argv[1:], "c:i:g:p:s:dl:", - ["cache", "internet2_netblocks", "global_netblocks", "pidfile", "socket", "debug", "log="]) - for option, argument in opts: - if option in ("-c", "--cache"): - cachefile = argument - if option in ("-i", "--internet2_netblocks"): - internet2_netblocks_file = argument - if option in ("-g", "--global_netblocks"): - global_netblocks_file = argument - if option in ("-s", "--socket"): - socketfile = argument - if option in ("-p", "--pidfile"): - pidfile = argument - if option in ("-l", "--log"): - try: - logfile = open(argument, 'a') - except: - logfile = None - if option in ("-d", "--debug"): - debug = True - -def open_geoip_databases(): - global gipv4 - global gipv6 - try: - gipv4 = GeoIP.open("/usr/share/GeoIP/GeoIP.dat", GeoIP.GEOIP_STANDARD) - except: - gipv4=None - try: - gipv6 = GeoIP.open("/usr/share/GeoIP/GeoIPv6.dat", GeoIP.GEOIP_STANDARD) - except: - gipv6=None - -def convert_6to4_v4(ip): - all_6to4 = IP('2002::/16') - if ip.version() != 6 or ip not in all_6to4: - return None - parts=ip.strNormal().split(':') - - ab = int(parts[1],16) - a = (ab >> 8) & 0xFF - b = ab & 0xFF - cd = int(parts[2],16) - c = (cd >> 8) & 0xFF - d = cd & 0xFF - - v4addr = '%d.%d.%d.%d' % (a,b,c,d) - return IP(v4addr) - -def convert_teredo_v4(ip): - teredo_std = IP('2001::/32') - teredo_xp = IP('3FFE:831F::/32') - if ip.version() != 6 or (ip not in teredo_std and ip not in teredo_xp): - return None - parts=ip.strNormal().split(':') - - ab = int(parts[6],16) - a = ((ab >> 8) & 0xFF) ^ 0xFF - b = (ab & 0xFF) ^ 0xFF - cd = int(parts[7],16) - c = ((cd >> 8) & 0xFF) ^ 0xFF - d = (cd & 0xFF) ^ 0xFF - - v4addr = '%d.%d.%d.%d' % (a,b,c,d) - return IP(v4addr) - -def load_databases_and_caches(*args, **kwargs): - sys.stderr.write("load_databases_and_caches...") - sys.stderr.flush() - open_geoip_databases() - read_caches() - sys.stderr.write("done.\n") - sys.stderr.flush() - -def remove_pidfile(pidfile): - os.unlink(pidfile) - -def create_pidfile_dir(pidfile): - piddir = os.path.dirname(pidfile) - try: - os.makedirs(piddir, mode=0755) - except OSError, err: - if err.errno == 17: # File exists - pass - else: - raise - except: - raise - -def write_pidfile(pidfile, pid): - create_pidfile_dir(pidfile) - f = open(pidfile, 'w') - f.write(str(pid)) - f.close() - return 0 - -def manage_pidfile(pidfile): - """returns 1 if another process is running that is named in pidfile, - otherwise creates/writes pidfile and returns 0.""" - pid = os.getpid() - try: - f = open(pidfile, 'r') - except IOError, err: - if err.errno == 2: # No such file or directory - return write_pidfile(pidfile, pid) - return 1 - - oldpid=f.read() - f.close() - - # is the oldpid process still running? - try: - os.kill(int(oldpid), 0) - except ValueError: # malformed oldpid - return write_pidfile(pidfile, pid) - except OSError, err: - if err.errno == 3: # No such process - return write_pidfile(pidfile, pid) - return 1 - - -def main(): - global logfile - global pidfile - signal.signal(signal.SIGHUP, signal.SIG_IGN) - parse_args() - manage_pidfile(pidfile) - - oldumask = os.umask(0) - try: - os.unlink(socketfile) - except: - pass - - load_databases_and_caches() - signal.signal(signal.SIGHUP, sighup_handler) - # restart interrupted syscalls like select - signal.siginterrupt(signal.SIGHUP, False) - ss = ForkingUnixStreamServer(socketfile, MirrorlistHandler) - - while not must_die: - try: - ss.serve_forever() - except select.error: - pass - - try: - os.unlink(socketfile) - except: - pass - - if logfile is not None: - try: - logfile.close() - except: - pass - - remove_pidfile(pidfile) - return 0 - - -if __name__ == "__main__": - try: - sys.exit(main()) - except KeyboardInterrupt: - sys.exit(-1) diff --git a/tasks/mirrorlist.yml b/tasks/mirrorlist.yml index 7bc26135fe..0bc4cf81e5 100644 --- a/tasks/mirrorlist.yml +++ b/tasks/mirrorlist.yml @@ -37,12 +37,6 @@ # selinux policy - mirrormanager - put in place - for the sockfile # -# install mirrorlist_server.py hotfix -- name: hotfix mirrorlist_server.py - copy: src=$files/mirrorlist/mirrorlist_server.py dest=/usr/share/mirrormanager/mirrorlist-server/mirrorlist_server.py mode=0755 - notify: - - restart supervisord - # setup and configure supervisord - name: /etc/supervisord.conf copy: src=$files/mirrorlist/supervisord.conf dest=/etc/supervisord.conf mode=0644