Only run umdl if the data has actually changed

/usr/bin/mm2_update-master-directory-list is now wrapped with a
script which queries the fedmsg bus if there is actually new data
on the master mirror. So now it is wrapped with a lock wrapper
and fedmsg query wrapper.

roles/mirrormanager/backend/files/backend.cron

@@ -3,9 +3,10 @@ MAILTO=root
 # refresh the mirrorlist cache at the top of the hour
 55 * * * * mirrormanager /usr/bin/mm2_update-mirrorlist-server
 
-# update master directory list
+# update master directory list every hour
+# fedora message bus is queried before every run if there are actually changes
 # logs sent to /var/log/mirrormanager/umdl.log by default
-0 */2 * * * mirrormanager /usr/local/bin/lock-wrapper umdl /usr/bin/mm2_update-master-directory-list
+0 * * * * mirrormanager /usr/local/bin/umdl-required /var/log/mirrormanager/umdl-required.log
 
 # Sync netblocks list once a day
 30 0 * * * mirrormanager cd /usr/share/mirrormanager2 && /usr/bin/mm2_get_global_netblocks /var/lib/mirrormanager/global_netblocks.txt

roles/mirrormanager/backend/files/last-sync

@@ -0,0 +1,134 @@
+#!/usr/bin/python
+
+# Copyright (C) 2014 by Adrian Reber
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import requests
+import time
+import sys
+import getopt
+
+fedora = 'org.fedoraproject.prod.bodhi.updates.fedora.sync'
+epel = 'org.fedoraproject.prod.bodhi.updates.epel.sync'
+
+branched = 'org.fedoraproject.prod.compose.branched.rsync.complete'
+rawhide = 'org.fedoraproject.prod.compose.rawhide.rsync.complete'
+
+base_url = 'https://apps.fedoraproject.org/datagrepper/raw'
+
+
+topics =  []
+# default time interval to query for syncs: 1 day
+delta = 86400
+# return 0 and no output if a sync happened during <delta>
+# if no sync happened 1 is returned
+quiet = False
+
+def usage():
+	print
+	print "last-sync queries the Fedora Message Bus if new data is available on the public servers"
+	print
+	print "Usage: last-sync [options]"
+	print
+	print "Options:"
+	print "  -a, --all                 query all possible releases (default)"
+	print "                            (fedora, epel, branched, rawhide)"
+	print "  -f, --fedora              only query if fedora has been updated during <delta>"
+	print "  -e, --epel                only query if epel has been updated"
+	print "  -b, --branched            only query if the branched off release"
+	print "                            has been updated"
+	print "  -r, --rawhide             only query if rawhide has been updated"
+	print "  -q, --quiet               do not print out any informations"
+	print "  -d DELTA, --delta=DELTA   specify the time interval which should be used"
+	print "                            for the query (default: 86400)"
+
+
+# -a -f -e -b -r -q -d
+def parse_args():
+	global topics
+	global delta
+	global quiet
+	try:
+		opts, args = getopt.getopt(sys.argv[1:], "afhebrqd:", ["all", "fedora", "epel", "rawhide", "branched", "quiet", "delta="])
+	except getopt.GetoptError as err:
+		print str(err)
+		usage()
+		sys.exit(2)
+
+	for option, argument in opts:
+		if option in ("-a", "--all"):
+			topics =  [ fedora, epel, branched, rawhide ]
+		if option in ("-f", "--fedora"):
+			topics.append(fedora)
+		if option in ("-e", "--epel"):
+			topics.append(epel)
+		if option in ("-r", "--rawhide"):
+			topics.append(rawhide)
+		if option in ("-b", "--branched"):
+			topics.append(branched)
+		if option in ("-q", "--quiet"):
+			quiet = True
+		if option in ("-d", "--delta"):
+			delta = argument
+		if option in ("-h"):
+			usage();
+			sys.exit(0)
+
+
+
+def getKey(item):
+	return item[1]
+
+def create_url(url, topics, delta):
+	topic = ""
+	for i in topics:
+		topic += "&topic=%s" % i
+	return '%s?delta=%s%s' % (url, delta, topic)
+
+parse_args()
+
+if topics == []:
+	topics =  [ fedora, epel, branched, rawhide ]
+
+data = requests.get(create_url(base_url, topics, delta)).json()
+
+repos = []
+
+for i in range(0, data['count']):
+	try:
+		repo = "%s-%s" % (data['raw_messages'][i]['msg']['repo'], data['raw_messages'][i]['msg']['release'])
+	except:
+		# the rawhide and branch sync message has no repo information
+		arch = data['raw_messages'][i]['msg']['arch']
+		if arch == '':
+			arch = 'primary'
+		repo = "%s-%s" % (data['raw_messages'][i]['msg']['branch'], arch)
+
+	repos.append([repo, data['raw_messages'][i]['timestamp']])
+
+if quiet == False:
+	for repo, timestamp in sorted(repos, key=getKey):
+		print "%s: %s" % (repo, time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(timestamp)))
+
+if data['count'] > 0:
+	sys.exit(0)
+else:
+	sys.exit(1)

roles/mirrormanager/backend/files/mm2_umdl-required.logrotate

@@ -0,0 +1,9 @@
+/var/log/mirrormanager/umdl-required.log {
+    missingok
+    notifempty
+    daily
+    dateext
+    rotate 15
+    postrotate
+    endscript
+}

roles/mirrormanager/backend/files/umdl-required

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+if [ $# -ne 1 ]; then
+	echo "Exactly one parameter needed. /path/to/logfile"
+	exit 1
+fi
+
+exec >> $1
+exec 2>&1
+
+CURDATE=`date +%s`
+
+if [ -e /var/run/mirrormanager/umdl ]; then
+	. /var/run/mirrormanager/umdl
+else
+	# 48 hours -> 172800 seconds
+	let LASTRUN=CURDATE-172800
+fi
+
+let DELTA=CURDATE-LASTRUN
+
+date
+echo "Last sync more than ${DELTA} seconds ago. Running umdl"
+
+/usr/local/bin/last-sync -d ${DELTA} -q
+
+if [ "$?" -ne "0" ]; then
+	# no changes on the master mirror
+	# abort
+	exit 0
+fi
+
+/usr/local/bin/lock-wrapper umdl /usr/bin/mm2_update-master-directory-list
+
+if [ "$?" -eq "0" ]; then
+	# success! remember the date of this run
+	CURDATE=`date +%s`
+	echo "LASTRUN=${CURDATE}" > /var/run/mirrormanager/umdl
+	date
+	echo "Finished umdl successfully"
+	exit 0
+fi
+
+date
+echo "umdl returned not zero. Something failed. Please check umdl.log"

roles/mirrormanager/backend/tasks/main.yml

@@ -34,3 +34,12 @@
   copy: src=backend.cron dest=/etc/cron.d/mm2_backend.cron
   tags:
   - config
+
+- name: install the fedmsg query script
+  copy: src=last-sync dest=/usr/local/bin/last-sync mode=0755
+
+- name: install the umdl-required script
+  copy: src=last-sync dest=/usr/local/bin/umdl-required mode=0755
+
+- name: install the umdl-required logrotate file
+  copy: src=mm2_umdl-required.logrotate dest=/etc/logrotate.d/mm2_umdl-required mode=644