From cb983c1f1a4cf9b7733c2a9415b97cd245511d9d Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Wed, 6 Apr 2016 16:00:30 +0000 Subject: [PATCH] and we have a simple script to try and make updates faster --- files/download/download-sync.cron | 4 +- files/download/download-sync.cron.ib01 | 4 + files/download/last-sync | 161 +++++++++--------- files/download/sync-up-downloads.sh.ib01 | 56 ++++-- ...archives.sh.ib01 => sync-up-other.sh.ib01} | 8 +- playbooks/groups/download.yml | 9 +- 6 files changed, 147 insertions(+), 95 deletions(-) create mode 100644 files/download/download-sync.cron.ib01 rename files/download/{sync-up-archives.sh.ib01 => sync-up-other.sh.ib01} (64%) diff --git a/files/download/download-sync.cron b/files/download/download-sync.cron index 805aa97c56..90e771163b 100644 --- a/files/download/download-sync.cron +++ b/files/download/download-sync.cron @@ -1,4 +1,4 @@ # run twice daily rsync of download. but lock it -MAILTO=smooge@gmail.com +MAILTO=smooge@gmail.com,root@fedoraproject.org 00 11,23 * * * root /usr/local/bin/lock-wrapper sync-up-downloads "/usr/local/bin/sync-up-downloads" -00 23 * * * root /usr/local/bin/lock-wrapper sync-up-archives "/usr/local/bin/sync-up-archives" + diff --git a/files/download/download-sync.cron.ib01 b/files/download/download-sync.cron.ib01 new file mode 100644 index 0000000000..7ece2d8ac4 --- /dev/null +++ b/files/download/download-sync.cron.ib01 @@ -0,0 +1,4 @@ +# run twice daily rsync of download. but lock it +MAILTO=smooge@gmail.com,root@fedoraproject.org +00 11,23 * * * root /usr/local/bin/lock-wrapper sync-up-downloads "/usr/local/bin/sync-up-downloads" +00 11,23 * * * root /usr/local/bin/lock-wrapper sync-up-other "/usr/local/bin/sync-up-other" diff --git a/files/download/last-sync b/files/download/last-sync index 332356a09c..fd43dc77af 100644 --- a/files/download/last-sync +++ b/files/download/last-sync @@ -42,112 +42,121 @@ delta = 86400 # if no sync happened 1 is returned quiet = False secondary = False +rawtime = False def usage(): - print - print "last-sync queries the Fedora Message Bus if new data is available on the public servers" - print - print "Usage: last-sync [options]" - print - print "Options:" - print " -a, --all query all possible releases (default)" - print " (fedora, epel, branched, rawhide)" - print " -f, --fedora only query if fedora has been updated during " - print " -e, --epel only query if epel has been updated" - print " -b, --branched only query if the branched off release" - print " has been updated" - print " -r, --rawhide only query if rawhide has been updated" - print " -q, --quiet do not print out any informations" - print " -d DELTA, --delta=DELTA specify the time interval which should be used" - print " for the query (default: 86400)" + print + print "last-sync queries the Fedora Message Bus if new data is available on the public servers" + print + print "Usage: last-sync [options]" + print + print "Options:" + print " -a, --all query all possible releases (default)" + print " (fedora, epel, branched, rawhide)" + print " -f, --fedora only query if fedora has been updated during " + print " -e, --epel only query if epel has been updated" + print " -b, --branched only query if the branched off release" + print " has been updated" + print " -r, --rawhide only query if rawhide has been updated" + print " -q, --quiet do not print out any informations" + print " -t, --time print date in seconds since 1970-01-01" + print " -d DELTA, --delta=DELTA specify the time interval which should be used" + print " for the query (default: 86400)" # -a -f -e -b -r -s -q -d def parse_args(): - global topics - global delta - global quiet - global secondary - try: - opts, args = getopt.getopt(sys.argv[1:], "afhebrsqd:", ["all", "fedora", "epel", "rawhide", "branched", "secondary", "quiet", "delta="]) - except getopt.GetoptError as err: - print str(err) - usage() - sys.exit(2) + global topics + global delta + global quiet + global secondary + global rawtime + try: + opts, args = getopt.getopt(sys.argv[1:], "afhebrsqtd:", ["all", "fedora", "epel", "rawhide", "branched", "secondary", "quiet", "time", "delta="]) + except getopt.GetoptError as err: + print str(err) + usage() + sys.exit(2) - for option, argument in opts: - if option in ("-a", "--all"): - topics = [ fedora, epel, branched, rawhide ] - secondary = True - if option in ("-f", "--fedora"): - topics.append(fedora) - if option in ("-e", "--epel"): - topics.append(epel) - if option in ("-r", "--rawhide"): - topics.append(rawhide) - if option in ("-b", "--branched"): - topics.append(branched) - if option in ("-s", "--secondary"): - topics.append(rawhide) - secondary = True - if option in ("-q", "--quiet"): - quiet = True - if option in ("-d", "--delta"): - delta = argument - if option in ("-h"): - usage(); - sys.exit(0) + for option, argument in opts: + if option in ("-a", "--all"): + topics = [ fedora, epel, branched, rawhide ] + secondary = True + if option in ("-f", "--fedora"): + topics.append(fedora) + if option in ("-e", "--epel"): + topics.append(epel) + if option in ("-r", "--rawhide"): + topics.append(rawhide) + if option in ("-b", "--branched"): + topics.append(branched) + if option in ("-s", "--secondary"): + topics.append(rawhide) + secondary = True + if option in ("-q", "--quiet"): + quiet = True + if option in ("-t", "--time"): + rawtime = True + if option in ("-d", "--delta"): + delta = argument + if option in ("-h"): + usage(); + sys.exit(0) def getKey(item): - return item[1] + return item[1] def create_url(url, topics, delta): - topic = "" - for i in topics: - topic += "&topic=%s" % i - return '%s?delta=%s%s' % (url, delta, topic) + topic = "" + for i in topics: + topic += "&topic=%s" % i + return '%s?delta=%s%s' % (url, delta, topic) parse_args() if topics == []: - topics = [ fedora, epel, branched, rawhide ] - secondary = True + topics = [ fedora, epel, branched, rawhide ] + secondary = True i = 0 data = None while i < 5: - try: - data = requests.get(create_url(base_url, topics, delta), timeout=1).json() - break - except: - pass + try: + data = requests.get(create_url(base_url, topics, delta), timeout=1).json() + break + except: + pass if not data: - sys.exit(1) + sys.exit(1) repos = [] for i in range(0, data['count']): - try: - repo = "%s-%s" % (data['raw_messages'][i]['msg']['repo'], data['raw_messages'][i]['msg']['release']) - except: - # the rawhide and branch sync message has no repo information - arch = data['raw_messages'][i]['msg']['arch'] - if arch == '': - arch = 'primary' - elif not secondary: - continue - repo = "%s-%s" % (data['raw_messages'][i]['msg']['branch'], arch) + try: + repo = "%s-%s" % (data['raw_messages'][i]['msg']['repo'], data['raw_messages'][i]['msg']['release']) + except: + # the rawhide and branch sync message has no repo information + arch = data['raw_messages'][i]['msg']['arch'] + if arch == '': + arch = 'primary' + elif not secondary: + continue + repo = "%s-%s" % (data['raw_messages'][i]['msg']['branch'], arch) - repos.append([repo, data['raw_messages'][i]['timestamp']]) + repos.append([repo, data['raw_messages'][i]['timestamp']]) if quiet == False: - for repo, timestamp in sorted(repos, key=getKey): - print "%s: %s" % (repo, time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(timestamp))) + for repo, timestamp in sorted(repos, key=getKey): + if rawtime == True: + # this is useful if you want to compare the timestamp in seconds versus string + print "%s: %s" % (repo, timestamp) + else: + print "%s: %s" % (repo, time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(timestamp))) if data['count'] > 0: - sys.exit(0) + sys.exit(0) else: - sys.exit(1) + sys.exit(1) diff --git a/files/download/sync-up-downloads.sh.ib01 b/files/download/sync-up-downloads.sh.ib01 index 4edc7d8c18..361e7a3795 100755 --- a/files/download/sync-up-downloads.sh.ib01 +++ b/files/download/sync-up-downloads.sh.ib01 @@ -12,21 +12,57 @@ ALT_EXCLUDES="" EPL_EXCLUDES="" FED_EXCLUDES="" +DATE_EPEL='/root/last-epel-sync' +DATE_FED='/root/last-fed-sync' + +touch ${DATE_EPEL} +touch ${DATE_FED} + LAST_SYNC='/usr/local/bin/last-sync' SERVER=dl.fedoraproject.org -# The Buffet. -#${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-buffet0/ /srv/pub/ | tail -n2 | logger -p local0.notice -t rsync_updates_alt_stg +## Do the EPEL download if we have had a sync since the last update. +CURDATE=`date +%s` +if [[ -s ${DATE_EPEL} ]]; then + LASTRUN=$( cat DATE_EPEL | awk '{print int($NF)}' ) +else + LASTRUN=$( date +%s --date="Jan 1 00:00:00 UTC 2007" ) +fi + +DELTA=`echo ${CURDATE}-${LASTRUN} | bc` + +/path/to/last-sync -d ${DELTA} -q + +if [ "$?" -eq "0" ]; then + # EPEL + ${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-epel0/ /srv/pub/epel/ | tail -n2 | logger -p local0.notice -t rsync_epel + echo ${CURDATE} > ${DATE_EPEL} +else + logger -p local0.notice -t rsync_epel "No change found. Not syncing" +fi + +## Do the FEDORA download if we have had a sync since the last update. +CURDATE=`date +%s` +if [[ -s ${DATE_FED} ]]; then + LASTRUN=$( cat DATE_FED | awk '{print int($NF)}' ) +else + LASTRUN=$( date +%s --date="Jan 1 00:00:00 UTC 2007" ) +fi + +DELTA=`echo ${CURDATE}-${LASTRUN} | bc` + +/path/to/last-sync -d ${DELTA} -q + +if [ "$?" -eq "0" ]; then + # Enchilada + ${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-enchilada0/ /srv/pub/fedora/ | tail -n2 | logger -p local0.notice -t rsync_fedora + echo ${CURDATE} > ${DATE_FED} +else + logger -p local0.notice -t rsync_fedora "No change found. Not syncing" +fi + -# Alt -${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-alt0/ /srv/pub/alt/ | tail -n2 | logger -p local0.notice -t rsync_alt -# EPEL -${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-epel0/ /srv/pub/epel/ | tail -n2 | logger -p local0.notice -t rsync_epel -# Enchilada -${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-enchilada0/ /srv/pub/fedora/ | tail -n2 | logger -p local0.notice -t rsync_fedora -# Secondary -${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-secondary/ /srv/pub/fedora-secondary/ | tail -n2 | logger -p local0.notice -t rsync_2nd # Let MM know I'm all up to date #/usr/bin/report_mirror diff --git a/files/download/sync-up-archives.sh.ib01 b/files/download/sync-up-other.sh.ib01 similarity index 64% rename from files/download/sync-up-archives.sh.ib01 rename to files/download/sync-up-other.sh.ib01 index 60e4ec3de8..5c47129839 100755 --- a/files/download/sync-up-archives.sh.ib01 +++ b/files/download/sync-up-other.sh.ib01 @@ -16,13 +16,13 @@ LAST_SYNC='/usr/local/bin/last-sync' SERVER=dl.fedoraproject.org -# The Buffet. -#${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-buffet0/ /srv/pub/ | tail -n2 | logger -p local0.notice -t rsync_updates_alt_stg - +# Alt +${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-alt0/ /srv/pub/alt/ | tail -n2 | logger -p local0.notice -t rsync_alt +# Secondary +${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-secondary/ /srv/pub/fedora-secondary/ | tail -n2 | logger -p local0.notice -t rsync_2nd # Archives ${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-archive/ /srv/pub/archive/ | tail -n2 | logger -p local0.notice -t rsync_archive - # Let MM know I'm all up to date #/usr/bin/report_mirror diff --git a/playbooks/groups/download.yml b/playbooks/groups/download.yml index 946f400057..b66fd31d0d 100644 --- a/playbooks/groups/download.yml +++ b/playbooks/groups/download.yml @@ -50,15 +50,18 @@ - name: put in script for syncing on download-ib02 copy: src="{{ files }}/download/sync-up-downloads.sh" dest=/usr/local/bin/sync-up-downloads owner=root group=root mode=755 when: inventory_hostname == 'download-ib02.fedoraproject.org' + - name: put in cron job for syncing + copy: src="{{ files }}/download/download-sync.cron" dest=/etc/cron.d/download-sync owner=root group=root mode=644 + when: inventory_hostname == 'download-ib02.fedoraproject.org' - name: put in script for syncing on download-ib01 copy: src="{{ files }}/download/sync-up-downloads.sh.ib01" dest=/usr/local/bin/sync-up-downloads owner=root group=root mode=755 when: inventory_hostname == 'download-ib01.fedoraproject.org' - name: put in script for syncing on download-ib01 - copy: src="{{ files }}/download/sync-up-archives.sh.ib01" dest=/usr/local/bin/sync-up-archives owner=root group=root mode=755 + copy: src="{{ files }}/download/sync-up-other.sh.ib01" dest=/usr/local/bin/sync-up-other owner=root group=root mode=755 when: inventory_hostname == 'download-ib01.fedoraproject.org' - name: put in cron job for syncing - copy: src="{{ files }}/download/download-sync.cron" dest=/etc/cron.d/download-sync owner=root group=root mode=644 - when: datacenter == 'ibiblio' + copy: src="{{ files }}/download/download-sync.cron.ib01" dest=/etc/cron.d/download-sync owner=root group=root mode=644 + when: inventory_hostname == 'download-ib01.fedoraproject.org' - name: put in last sync scrypt for download-ib01 copy: src="{{ files}}/download/last-sync" dest=/usr/local/bin/last-sync mode=0755 when: inventory_hostname == 'download-ib01.fedoraproject.org'