and we have a simple script to try and make updates faster

files/download/download-sync.cron

@@ -1,4 +1,4 @@
 # run twice daily rsync of download. but lock it
-MAILTO=smooge@gmail.com
+MAILTO=smooge@gmail.com,root@fedoraproject.org
 00 11,23 * * * root /usr/local/bin/lock-wrapper sync-up-downloads "/usr/local/bin/sync-up-downloads"
-00 23 * * * root /usr/local/bin/lock-wrapper sync-up-archives "/usr/local/bin/sync-up-archives"
+

files/download/download-sync.cron.ib01

@@ -0,0 +1,4 @@
+# run twice daily rsync of download. but lock it
+MAILTO=smooge@gmail.com,root@fedoraproject.org
+00 11,23 * * * root /usr/local/bin/lock-wrapper sync-up-downloads "/usr/local/bin/sync-up-downloads"
+00 11,23 * * * root /usr/local/bin/lock-wrapper sync-up-other "/usr/local/bin/sync-up-other"

files/download/last-sync

@@ -42,112 +42,121 @@ delta = 86400
 # if no sync happened 1 is returned
 quiet = False
 secondary = False
+rawtime = False
 
 def usage():
-	print
+        print
-	print "last-sync queries the Fedora Message Bus if new data is available on the public servers"
+        print "last-sync queries the Fedora Message Bus if new data is available on the public servers"
-	print
+        print
-	print "Usage: last-sync [options]"
+        print "Usage: last-sync [options]"
-	print
+        print
-	print "Options:"
+        print "Options:"
-	print "  -a, --all                 query all possible releases (default)"
+        print "  -a, --all                 query all possible releases (default)"
-	print "                            (fedora, epel, branched, rawhide)"
+        print "                            (fedora, epel, branched, rawhide)"
-	print "  -f, --fedora              only query if fedora has been updated during <delta>"
+        print "  -f, --fedora              only query if fedora has been updated during <delta>"
-	print "  -e, --epel                only query if epel has been updated"
+        print "  -e, --epel                only query if epel has been updated"
-	print "  -b, --branched            only query if the branched off release"
+        print "  -b, --branched            only query if the branched off release"
-	print "                            has been updated"
+        print "                            has been updated"
-	print "  -r, --rawhide             only query if rawhide has been updated"
+        print "  -r, --rawhide             only query if rawhide has been updated"
-	print "  -q, --quiet               do not print out any informations"
+        print "  -q, --quiet               do not print out any informations"
-	print "  -d DELTA, --delta=DELTA   specify the time interval which should be used"
+        print "  -t, --time                print date in seconds since 1970-01-01"
-	print "                            for the query (default: 86400)"
+        print "  -d DELTA, --delta=DELTA   specify the time interval which should be used"
+        print "                            for the query (default: 86400)"
 
 
 # -a -f -e -b -r -s -q -d
 def parse_args():
-	global topics
+        global topics
-	global delta
+        global delta
-	global quiet
+        global quiet
-	global secondary
+        global secondary
-	try:
+        global rawtime
-		opts, args = getopt.getopt(sys.argv[1:], "afhebrsqd:", ["all", "fedora", "epel", "rawhide", "branched", "secondary", "quiet", "delta="])
+        try:
-	except getopt.GetoptError as err:
+                opts, args = getopt.getopt(sys.argv[1:], "afhebrsqtd:", ["all", "fedora", "epel", "rawhide", "branched", "secondary", "quiet", "time", "delta="])
-		print str(err)
+        except getopt.GetoptError as err:
-		usage()
+                print str(err)
-		sys.exit(2)
+                usage()
+                sys.exit(2)
 
-	for option, argument in opts:
+        for option, argument in opts:
-		if option in ("-a", "--all"):
+                if option in ("-a", "--all"):
-			topics =  [ fedora, epel, branched, rawhide ]
+                        topics =  [ fedora, epel, branched, rawhide ]
-			secondary = True
+                        secondary = True
-		if option in ("-f", "--fedora"):
+                if option in ("-f", "--fedora"):
-			topics.append(fedora)
+                        topics.append(fedora)
-		if option in ("-e", "--epel"):
+                if option in ("-e", "--epel"):
-			topics.append(epel)
+                        topics.append(epel)
-		if option in ("-r", "--rawhide"):
+                if option in ("-r", "--rawhide"):
-			topics.append(rawhide)
+                        topics.append(rawhide)
-		if option in ("-b", "--branched"):
+                if option in ("-b", "--branched"):
-			topics.append(branched)
+                        topics.append(branched)
-		if option in ("-s", "--secondary"):
+                if option in ("-s", "--secondary"):
-			topics.append(rawhide)
+                        topics.append(rawhide)
-			secondary = True
+                        secondary = True
-		if option in ("-q", "--quiet"):
+                if option in ("-q", "--quiet"):
-			quiet = True
+                        quiet = True
-		if option in ("-d", "--delta"):
+                if option in ("-t", "--time"):
-			delta = argument
+                        rawtime = True
-		if option in ("-h"):
+                if option in ("-d", "--delta"):
-			usage();
+                        delta = argument
-			sys.exit(0)
+                if option in ("-h"):
+                        usage();
+                        sys.exit(0)
 
 
 
 def getKey(item):
-	return item[1]
+        return item[1]
 
 def create_url(url, topics, delta):
-	topic = ""
+        topic = ""
-	for i in topics:
+        for i in topics:
-		topic += "&topic=%s" % i
+                topic += "&topic=%s" % i
-	return '%s?delta=%s%s' % (url, delta, topic)
+        return '%s?delta=%s%s' % (url, delta, topic)
 
 parse_args()
 
 if topics == []:
-	topics =  [ fedora, epel, branched, rawhide ]
+        topics =  [ fedora, epel, branched, rawhide ]
-	secondary = True
+        secondary = True
 
 i = 0
 data = None
 while i < 5:
-	try:
+        try:
-		data = requests.get(create_url(base_url, topics, delta), timeout=1).json()
+                data = requests.get(create_url(base_url, topics, delta), timeout=1).json()
-		break
+                break
-	except:
+        except:
-		pass
+                pass
 
 if not data:
-	sys.exit(1)
+        sys.exit(1)
 
 repos = []
 
 for i in range(0, data['count']):
-	try:
+        try:
-		repo = "%s-%s" % (data['raw_messages'][i]['msg']['repo'], data['raw_messages'][i]['msg']['release'])
+                repo = "%s-%s" % (data['raw_messages'][i]['msg']['repo'], data['raw_messages'][i]['msg']['release'])
-	except:
+        except:
-		# the rawhide and branch sync message has no repo information
+                # the rawhide and branch sync message has no repo information
-		arch = data['raw_messages'][i]['msg']['arch']
+                arch = data['raw_messages'][i]['msg']['arch']
-		if arch == '':
+                if arch == '':
-			arch = 'primary'
+                        arch = 'primary'
-		elif not secondary:
+                elif not secondary:
-			continue
+                        continue
-		repo = "%s-%s" % (data['raw_messages'][i]['msg']['branch'], arch)
+                repo = "%s-%s" % (data['raw_messages'][i]['msg']['branch'], arch)
 
-	repos.append([repo, data['raw_messages'][i]['timestamp']])
+        repos.append([repo, data['raw_messages'][i]['timestamp']])
 
 if quiet == False:
-	for repo, timestamp in sorted(repos, key=getKey):
+        for repo, timestamp in sorted(repos, key=getKey):
-		print "%s: %s" % (repo, time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(timestamp)))
+                if rawtime == True:
+                   # this is useful if you want to compare the timestamp in seconds versus string
+                   print "%s: %s" % (repo, timestamp)
+                else:
+                   print "%s: %s" % (repo, time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(timestamp)))
 
 if data['count'] > 0:
-	sys.exit(0)
+        sys.exit(0)
 else:
-	sys.exit(1)
+        sys.exit(1)

files/download/sync-up-downloads.sh.ib01

@@ -12,21 +12,57 @@ ALT_EXCLUDES=""
 EPL_EXCLUDES=""
 FED_EXCLUDES=""
 
+DATE_EPEL='/root/last-epel-sync'
+DATE_FED='/root/last-fed-sync'
+
+touch ${DATE_EPEL}
+touch ${DATE_FED}
+
 LAST_SYNC='/usr/local/bin/last-sync'
 
 SERVER=dl.fedoraproject.org
 
-# The Buffet.
+## Do the EPEL download if we have had a sync since the last update.
-#${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-buffet0/  /srv/pub/ | tail -n2 | logger -p local0.notice -t rsync_updates_alt_stg
+CURDATE=`date +%s`
+if [[ -s ${DATE_EPEL} ]]; then
+    LASTRUN=$( cat DATE_EPEL | awk '{print int($NF)}' )
+else
+    LASTRUN=$( date +%s --date="Jan 1 00:00:00 UTC 2007" )
+fi
+
+DELTA=`echo ${CURDATE}-${LASTRUN} | bc`
+
+/path/to/last-sync -d ${DELTA} -q
+
+if [ "$?" -eq "0" ]; then
+    # EPEL
+    ${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-epel0/  /srv/pub/epel/  | tail -n2 | logger -p local0.notice -t rsync_epel
+    echo ${CURDATE} > ${DATE_EPEL}
+else
+    logger -p local0.notice -t rsync_epel "No change found. Not syncing"
+fi
+
+## Do the FEDORA download if we have had a sync since the last update.
+CURDATE=`date +%s`
+if [[ -s ${DATE_FED} ]]; then
+    LASTRUN=$( cat DATE_FED | awk '{print int($NF)}' )
+else
+    LASTRUN=$( date +%s --date="Jan 1 00:00:00 UTC 2007" )
+fi
+
+DELTA=`echo ${CURDATE}-${LASTRUN} | bc`
+
+/path/to/last-sync -d ${DELTA} -q
+
+if [ "$?" -eq "0" ]; then
+    # Enchilada
+    ${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-enchilada0/  /srv/pub/fedora/ | tail -n2 | logger -p local0.notice -t rsync_fedora
+    echo ${CURDATE} > ${DATE_FED}
+else
+    logger -p local0.notice -t rsync_fedora "No change found. Not syncing"
+fi
+
 
-# Alt
-${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-alt0/  /srv/pub/alt/ | tail -n2 | logger -p local0.notice -t rsync_alt
-# EPEL
-${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-epel0/  /srv/pub/epel/  | tail -n2 | logger -p local0.notice -t rsync_epel
-# Enchilada
-${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-enchilada0/  /srv/pub/fedora/ | tail -n2 | logger -p local0.notice -t rsync_fedora
-# Secondary
-${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-secondary/  /srv/pub/fedora-secondary/  | tail -n2 | logger -p local0.notice -t rsync_2nd
 
 # Let MM know I'm all up to date
 #/usr/bin/report_mirror

files/download/sync-up-other.sh.ib01

@@ -16,13 +16,13 @@ LAST_SYNC='/usr/local/bin/last-sync'
 
 SERVER=dl.fedoraproject.org
 
-# The Buffet.
+# Alt
-#${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-buffet0/  /srv/pub/ | tail -n2 | logger -p local0.notice -t rsync_updates_alt_stg
+${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-alt0/  /srv/pub/alt/ | tail -n2 | logger -p local0.notice -t rsync_alt
-
+# Secondary
+${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-secondary/  /srv/pub/fedora-secondary/  | tail -n2 | logger -p local0.notice -t rsync_2nd
 # Archives
 ${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-archive/  /srv/pub/archive/  | tail -n2 | logger -p local0.notice -t rsync_archive
 
 
-
 # Let MM know I'm all up to date
 #/usr/bin/report_mirror

playbooks/groups/download.yml

@@ -50,15 +50,18 @@
   - name: put in script for syncing on download-ib02
     copy: src="{{ files }}/download/sync-up-downloads.sh"  dest=/usr/local/bin/sync-up-downloads owner=root group=root mode=755
     when: inventory_hostname == 'download-ib02.fedoraproject.org'
+  - name: put in cron job for syncing
+    copy: src="{{ files }}/download/download-sync.cron"  dest=/etc/cron.d/download-sync owner=root group=root mode=644
+    when: inventory_hostname == 'download-ib02.fedoraproject.org'
   - name: put in script for syncing on download-ib01
     copy: src="{{ files }}/download/sync-up-downloads.sh.ib01" dest=/usr/local/bin/sync-up-downloads owner=root group=root mode=755
     when: inventory_hostname == 'download-ib01.fedoraproject.org'
   - name: put in script for syncing on download-ib01
-    copy: src="{{ files }}/download/sync-up-archives.sh.ib01" dest=/usr/local/bin/sync-up-archives owner=root group=root mode=755
+    copy: src="{{ files }}/download/sync-up-other.sh.ib01" dest=/usr/local/bin/sync-up-other owner=root group=root mode=755
     when: inventory_hostname == 'download-ib01.fedoraproject.org'
   - name: put in cron job for syncing
-    copy: src="{{ files }}/download/download-sync.cron"  dest=/etc/cron.d/download-sync owner=root group=root mode=644
+    copy: src="{{ files }}/download/download-sync.cron.ib01"  dest=/etc/cron.d/download-sync owner=root group=root mode=644
-    when: datacenter == 'ibiblio'
+    when: inventory_hostname == 'download-ib01.fedoraproject.org'
   - name: put in last sync scrypt for download-ib01
     copy: src="{{ files}}/download/last-sync" dest=/usr/local/bin/last-sync mode=0755
     when: inventory_hostname == 'download-ib01.fedoraproject.org'