From bdbc8dd6009fb71c6d035bb7d45837741be2fef7 Mon Sep 17 00:00:00 2001
From: Adrian Reber <adrian@lisas.de>
Date: Thu, 2 Jul 2015 20:32:34 +0000
Subject: [PATCH] Also switch to fedmsg based umdl crawl for Secondary

---
 .../mirrormanager/backend/files/backend.cron  |  7 +++---
 roles/mirrormanager/backend/files/last-sync   | 25 ++++++++++++++++---
 .../mirrormanager/backend/files/umdl-required | 13 +++++++---
 3 files changed, 34 insertions(+), 11 deletions(-)

diff --git a/roles/mirrormanager/backend/files/backend.cron b/roles/mirrormanager/backend/files/backend.cron
index cc72bfbe45..f3b867f652 100644
--- a/roles/mirrormanager/backend/files/backend.cron
+++ b/roles/mirrormanager/backend/files/backend.cron
@@ -9,11 +9,12 @@ MAILTO=root
 # check if category Fedora EPEL needs updating every 30 minutes
 0,30 * * * * mirrormanager /usr/local/bin/umdl-required EPEL /var/log/mirrormanager/umdl-required.log
 # check if category Fedora Linux needs updating every 30 minutes
-0,30 * * * * mirrormanager /usr/local/bin/umdl-required Linux /var/log/mirrormanager/umdl-required.log
+10,40 * * * * mirrormanager /usr/local/bin/umdl-required Linux /var/log/mirrormanager/umdl-required.log
+# check if category Fedora Secondary Arches needs updating every 30 minutes
+20,50 * * * * mirrormanager /usr/local/bin/umdl-required Secondary /var/log/mirrormanager/umdl-required.log
 # the remaining categories are updated once a day
 15 0 * * * mirrormanager /usr/bin/mm2_update-master-directory-list --category="Fedora Archive"
-15 8 * * * mirrormanager /usr/bin/mm2_update-master-directory-list --category="Fedora Other"
-15 16 * * * mirrormanager /usr/bin/mm2_update-master-directory-list --category="Fedora Secondary Arches"
+15 12  * * * mirrormanager /usr/bin/mm2_update-master-directory-list --category="Fedora Other"
 
 # Sync netblocks list once a day
 30 0 * * * mirrormanager cd /usr/share/mirrormanager2 && /usr/bin/mm2_get_global_netblocks /var/lib/mirrormanager/global_netblocks.txt
diff --git a/roles/mirrormanager/backend/files/last-sync b/roles/mirrormanager/backend/files/last-sync
index 8c3904bb92..332356a09c 100644
--- a/roles/mirrormanager/backend/files/last-sync
+++ b/roles/mirrormanager/backend/files/last-sync
@@ -41,6 +41,7 @@ delta = 86400
 # return 0 and no output if a sync happened during <delta>
 # if no sync happened 1 is returned
 quiet = False
+secondary = False
 
 def usage():
 	print
@@ -61,13 +62,14 @@ def usage():
 	print "                            for the query (default: 86400)"
 
 
-# -a -f -e -b -r -q -d
+# -a -f -e -b -r -s -q -d
 def parse_args():
 	global topics
 	global delta
 	global quiet
+	global secondary
 	try:
-		opts, args = getopt.getopt(sys.argv[1:], "afhebrqd:", ["all", "fedora", "epel", "rawhide", "branched", "quiet", "delta="])
+		opts, args = getopt.getopt(sys.argv[1:], "afhebrsqd:", ["all", "fedora", "epel", "rawhide", "branched", "secondary", "quiet", "delta="])
 	except getopt.GetoptError as err:
 		print str(err)
 		usage()
@@ -76,6 +78,7 @@ def parse_args():
 	for option, argument in opts:
 		if option in ("-a", "--all"):
 			topics =  [ fedora, epel, branched, rawhide ]
+			secondary = True
 		if option in ("-f", "--fedora"):
 			topics.append(fedora)
 		if option in ("-e", "--epel"):
@@ -84,6 +87,9 @@ def parse_args():
 			topics.append(rawhide)
 		if option in ("-b", "--branched"):
 			topics.append(branched)
+		if option in ("-s", "--secondary"):
+			topics.append(rawhide)
+			secondary = True
 		if option in ("-q", "--quiet"):
 			quiet = True
 		if option in ("-d", "--delta"):
@@ -107,8 +113,19 @@ parse_args()
 
 if topics == []:
 	topics =  [ fedora, epel, branched, rawhide ]
+	secondary = True
 
-data = requests.get(create_url(base_url, topics, delta)).json()
+i = 0
+data = None
+while i < 5:
+	try:
+		data = requests.get(create_url(base_url, topics, delta), timeout=1).json()
+		break
+	except:
+		pass
+
+if not data:
+	sys.exit(1)
 
 repos = []
 
@@ -120,7 +137,7 @@ for i in range(0, data['count']):
 		arch = data['raw_messages'][i]['msg']['arch']
 		if arch == '':
 			arch = 'primary'
-		else:
+		elif not secondary:
 			continue
 		repo = "%s-%s" % (data['raw_messages'][i]['msg']['branch'], arch)
 
diff --git a/roles/mirrormanager/backend/files/umdl-required b/roles/mirrormanager/backend/files/umdl-required
index af9bd10f28..4075a7f60c 100644
--- a/roles/mirrormanager/backend/files/umdl-required
+++ b/roles/mirrormanager/backend/files/umdl-required
@@ -13,8 +13,13 @@ LAST_SYNC_ARG=""
 
 if [ "${1}" ==  "Linux" ]; then
 	LAST_SYNC_ARG="-f -r"
+	CATEGORY="${1}"
 elif [ "${1}" ==  "EPEL" ]; then
 	LAST_SYNC_ARG="-e"
+	CATEGORY="${1}"
+elif [ "${1}" ==  "Secondary" ]; then
+	LAST_SYNC_ARG="-s"
+	CATEGORY="Secondary Arches"
 fi
 
 if [ "${LAST_SYNC_ARG}" == "" ]; then
@@ -39,18 +44,18 @@ if [ "$?" -ne "0" ]; then
 	exit 0
 fi
 
-echo -n "Last sync more than ${DELTA} seconds ago. Running umdl for Fedora ${1} at "
+echo -n "Last sync more than ${DELTA} seconds ago. Running umdl for Fedora ${CATEGORY} at "
 date
 
-/usr/local/bin/lock-wrapper umdl-${1} "/usr/bin/mm2_update-master-directory-list --category \"Fedora ${1}\""
+/usr/local/bin/lock-wrapper umdl-${1} "/usr/bin/mm2_update-master-directory-list --category \"Fedora ${CATEGORY}\""
 
 if [ "$?" -eq "0" ]; then
 	# success! remember the date of this run
 	echo "LASTRUN=${CURDATE}" > /var/run/mirrormanager/umdl-${1}
-	echo -n "Finished umdl successfully at "
+	echo -n "Finished umdl for Fedora ${CATEGORY} successfully at "
 	date
 	exit 0
 fi
 
-echo -n "umdl returned not zero. Something failed. Please check umdl.log. "
+echo -n "umdl for Fedora ${CATEGORY} returned non-zero. Something failed. Please check umdl.log. "
 date