Add proxy-syncd and example squid.conf

scripts/proxy-mirror/README

@@ -0,0 +1 @@
+These files are scripts and examples to help you to configure your own proxy mirror.

scripts/proxy-mirror/example-squid-config/squid.conf.diff

@@ -0,0 +1,47 @@
+--- /tmp/squid.conf	2007-07-09 14:00:14.000000000 -0400
++++ squid.conf	2007-07-08 16:22:49.000000000 -0400
+@@ -86,7 +86,8 @@
+ #	visible on the internal address.
+ #
+ # Squid normally listens to port 3128
+-http_port 3128
++http_port 80 accel defaultsite=download.fedora.redhat.com
++cache_peer 209.132.176.220 parent 80 0 no-query originserver
+ 
+ #  TAG: https_port
+ #	Usage:  [ip:]port cert=certificate.pem [key=key.pem] [options...]
+@@ -759,7 +760,7 @@
+ #	objects.
+ #
+ #Default:
+-# cache_mem 8 MB
++ cache_mem 384 MB
+ 
+ #  TAG: cache_swap_low	(percent, 0-100)
+ #  TAG: cache_swap_high	(percent, 0-100)
+@@ -792,7 +793,7 @@
+ #	See replacement_policy below for a discussion of this policy.
+ #
+ #Default:
+-# maximum_object_size 4096 KB
++ maximum_object_size 2000000 KB
+ 
+ #  TAG: minimum_object_size	(bytes)
+ #	Objects smaller than this size will NOT be saved on disk.  The
+@@ -1014,7 +1015,7 @@
+ #	(hard coded at 1 MB).
+ #
+ #Default:
+-# cache_dir ufs /var/spool/squid 100 16 256
++ cache_dir ufs /var/spool/squid 53000 16 256
+ 
+ #  TAG: logformat
+ #	Usage:
+@@ -2541,6 +2542,7 @@
+ #http_access deny to_localhost
+ #
+ # INSERT YOUR OWN RULE(S) HERE TO ALLOW ACCESS FROM YOUR CLIENTS
++http_access allow all
+ 
+ # Example rule allowing access from your local networks. Adapt
+ # to list your (internal) IP networks from where browsing should

scripts/proxy-mirror/proxy-syncd/README

@@ -0,0 +1,2 @@
+	proxy-syncd.py
+This script monitors repomd.xml of all configured yum repositories on the originating HTTP server.  If it changes, then it forces a "Pragma: no-cache" refresh of all repodata files on the proxy mirror.  This ensures that all repodata pulled from the proxy mirror is self-consistent (i.e. filelists.sqlite.bz2 matches the repomd.xml) and guards against failure conditions of yum clients.

scripts/proxy-mirror/proxy-syncd/proxy-syncd.py

@@ -0,0 +1,103 @@
+#!/usr/bin/python
+import sys
+import time
+import sha
+from urlgrabber.grabber import URLGrabber
+from urlgrabber.grabber import URLGrabError
+
+# original address
+BASE1='http://gromit.redhat.com/pub/fedora/linux'
+# proxy address
+BASE2='http://download.boston.redhat.com/pub/fedora/linux'
+# individual repos
+DIRS="""
+/updates/7/i386
+/updates/testing/7/i386
+/updates/7/x86_64
+/updates/testing/7/x86_64
+/updates/7/ppc
+/updates/testing/7/x86_64
+/development/i386/os
+/development/x86_64/os
+/development/ppc/os
+/core/updates/6/i386
+/core/updates/6/x86_64
+/core/updates/6/ppc
+"""
+# All repodata files
+REPOFILES=['repomd.xml','filelists.sqlite.bz2','filelists.xml.gz','other.sqlite.bz2','other.xml.gz','primary.sqlite.bz2','primary.xml.gz','updateinfo.xml.gz']
+# Log File
+LOGFILE='~/repodata-syncd.log'
+
+DEBUG=False
+
+# Hash URL, return hex sha1sum
+# http_headers = (('Pragma', 'no-cache'),)
+def hash_url(url):
+    retval = ''
+    try:
+        f = g.urlopen(url)
+        so = sha.new()
+        so.update(f.read())
+        f.close()
+        retval = so.hexdigest()
+    except URLGrabError:
+        retval = 'ERROR: Try again later.'
+    return retval
+    
+# Print Debug Messages
+def debug(msg):
+    if DEBUG == True:
+        print "    DEBUG: %s" % msg
+
+# Get Hashes of All repomd.xml
+def hash_all_urls():
+    for path in DIRDICT.keys():
+        url = BASE1 + path + '/repodata/repomd.xml'
+        hash = hash_url(url)
+        DIRDICT[path] = hash
+        print("%s %s" % (url, hash))
+
+# Refresh Repodata
+def refresh_repodata(path):
+    url = BASE2 + path + '/repodata/'
+    for file in REPOFILES:
+        debug("Grabbing %s" % url + file)
+        try:
+            r.urlread(url + file)
+        except URLGrabError:
+            pass
+
+### Main()
+# Setup Variables
+DIRLIST = DIRS.split()
+tuples = []
+for x in DIRLIST:
+    if x.startswith('#') == False:
+        tuples.append((x,0))
+DIRDICT = dict(tuples)
+g = URLGrabber(keepalive=0)
+r = URLGrabber(keepalive=0,http_headers = (('Pragma', 'no-cache'),))
+
+# Get Initial Hashes
+hash_all_urls()
+serial = 0
+
+# Loop Forever
+while True:
+    print "serial=%d" % serial
+    # Check each repodata directory
+    for path in DIRDICT.keys():
+        url  = BASE1 + path + '/repodata/repomd.xml'
+        hash = hash_url(url)
+        if hash != DIRDICT[path]:
+            print "CHANGE %s" % url
+            print "       %s" % DIRDICT[path]
+            print "       %s" % hash
+            print 'REFRESHING ' + BASE2 + path
+            # if hash changes, refresh repodata on proxy server
+            refresh_repodata(path)
+            # update dictionary entry to new hash value
+            DIRDICT[path]=hash
+    time.sleep(120)
+    serial += 1