and we have a simple script to try and make updates faster
This commit is contained in:
Stephen Smoogen
parent
790ba6f9fd
commit
cb983c1f1a
6 changed files with 147 additions and 95 deletions
|
|
@ -1,4 +1,4 @@
|
|||
# run twice daily rsync of download. but lock it
|
||||
MAILTO=smooge@gmail.com
|
||||
MAILTO=smooge@gmail.com,root@fedoraproject.org
|
||||
00 11,23 * * * root /usr/local/bin/lock-wrapper sync-up-downloads "/usr/local/bin/sync-up-downloads"
|
||||
00 23 * * * root /usr/local/bin/lock-wrapper sync-up-archives "/usr/local/bin/sync-up-archives"
|
||||
|
||||
|
|
|
|||
4
files/download/download-sync.cron.ib01
Normal file
4
files/download/download-sync.cron.ib01
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
# run twice daily rsync of download. but lock it
|
||||
MAILTO=smooge@gmail.com,root@fedoraproject.org
|
||||
00 11,23 * * * root /usr/local/bin/lock-wrapper sync-up-downloads "/usr/local/bin/sync-up-downloads"
|
||||
00 11,23 * * * root /usr/local/bin/lock-wrapper sync-up-other "/usr/local/bin/sync-up-other"
|
||||
|
|
@ -42,6 +42,7 @@ delta = 86400
|
|||
# if no sync happened 1 is returned
|
||||
quiet = False
|
||||
secondary = False
|
||||
rawtime = False
|
||||
|
||||
def usage():
|
||||
print
|
||||
|
|
@ -58,6 +59,7 @@ def usage():
|
|||
print " has been updated"
|
||||
print " -r, --rawhide only query if rawhide has been updated"
|
||||
print " -q, --quiet do not print out any informations"
|
||||
print " -t, --time print date in seconds since 1970-01-01"
|
||||
print " -d DELTA, --delta=DELTA specify the time interval which should be used"
|
||||
print " for the query (default: 86400)"
|
||||
|
||||
|
|
@ -68,8 +70,9 @@ def parse_args():
|
|||
global delta
|
||||
global quiet
|
||||
global secondary
|
||||
global rawtime
|
||||
try:
|
||||
opts, args = getopt.getopt(sys.argv[1:], "afhebrsqd:", ["all", "fedora", "epel", "rawhide", "branched", "secondary", "quiet", "delta="])
|
||||
opts, args = getopt.getopt(sys.argv[1:], "afhebrsqtd:", ["all", "fedora", "epel", "rawhide", "branched", "secondary", "quiet", "time", "delta="])
|
||||
except getopt.GetoptError as err:
|
||||
print str(err)
|
||||
usage()
|
||||
|
|
@ -92,6 +95,8 @@ def parse_args():
|
|||
secondary = True
|
||||
if option in ("-q", "--quiet"):
|
||||
quiet = True
|
||||
if option in ("-t", "--time"):
|
||||
rawtime = True
|
||||
if option in ("-d", "--delta"):
|
||||
delta = argument
|
||||
if option in ("-h"):
|
||||
|
|
@ -145,6 +150,10 @@ for i in range(0, data['count']):
|
|||
|
||||
if quiet == False:
|
||||
for repo, timestamp in sorted(repos, key=getKey):
|
||||
if rawtime == True:
|
||||
# this is useful if you want to compare the timestamp in seconds versus string
|
||||
print "%s: %s" % (repo, timestamp)
|
||||
else:
|
||||
print "%s: %s" % (repo, time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(timestamp)))
|
||||
|
||||
if data['count'] > 0:
|
||||
|
|
|
|||
|
|
@ -12,21 +12,57 @@ ALT_EXCLUDES=""
|
|||
EPL_EXCLUDES=""
|
||||
FED_EXCLUDES=""
|
||||
|
||||
DATE_EPEL='/root/last-epel-sync'
|
||||
DATE_FED='/root/last-fed-sync'
|
||||
|
||||
touch ${DATE_EPEL}
|
||||
touch ${DATE_FED}
|
||||
|
||||
LAST_SYNC='/usr/local/bin/last-sync'
|
||||
|
||||
SERVER=dl.fedoraproject.org
|
||||
|
||||
# The Buffet.
|
||||
#${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-buffet0/ /srv/pub/ | tail -n2 | logger -p local0.notice -t rsync_updates_alt_stg
|
||||
## Do the EPEL download if we have had a sync since the last update.
|
||||
CURDATE=`date +%s`
|
||||
if [[ -s ${DATE_EPEL} ]]; then
|
||||
LASTRUN=$( cat DATE_EPEL | awk '{print int($NF)}' )
|
||||
else
|
||||
LASTRUN=$( date +%s --date="Jan 1 00:00:00 UTC 2007" )
|
||||
fi
|
||||
|
||||
# Alt
|
||||
${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-alt0/ /srv/pub/alt/ | tail -n2 | logger -p local0.notice -t rsync_alt
|
||||
DELTA=`echo ${CURDATE}-${LASTRUN} | bc`
|
||||
|
||||
/path/to/last-sync -d ${DELTA} -q
|
||||
|
||||
if [ "$?" -eq "0" ]; then
|
||||
# EPEL
|
||||
${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-epel0/ /srv/pub/epel/ | tail -n2 | logger -p local0.notice -t rsync_epel
|
||||
echo ${CURDATE} > ${DATE_EPEL}
|
||||
else
|
||||
logger -p local0.notice -t rsync_epel "No change found. Not syncing"
|
||||
fi
|
||||
|
||||
## Do the FEDORA download if we have had a sync since the last update.
|
||||
CURDATE=`date +%s`
|
||||
if [[ -s ${DATE_FED} ]]; then
|
||||
LASTRUN=$( cat DATE_FED | awk '{print int($NF)}' )
|
||||
else
|
||||
LASTRUN=$( date +%s --date="Jan 1 00:00:00 UTC 2007" )
|
||||
fi
|
||||
|
||||
DELTA=`echo ${CURDATE}-${LASTRUN} | bc`
|
||||
|
||||
/path/to/last-sync -d ${DELTA} -q
|
||||
|
||||
if [ "$?" -eq "0" ]; then
|
||||
# Enchilada
|
||||
${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-enchilada0/ /srv/pub/fedora/ | tail -n2 | logger -p local0.notice -t rsync_fedora
|
||||
# Secondary
|
||||
${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-secondary/ /srv/pub/fedora-secondary/ | tail -n2 | logger -p local0.notice -t rsync_2nd
|
||||
echo ${CURDATE} > ${DATE_FED}
|
||||
else
|
||||
logger -p local0.notice -t rsync_fedora "No change found. Not syncing"
|
||||
fi
|
||||
|
||||
|
||||
|
||||
# Let MM know I'm all up to date
|
||||
#/usr/bin/report_mirror
|
||||
|
|
|
|||
|
|
@ -16,13 +16,13 @@ LAST_SYNC='/usr/local/bin/last-sync'
|
|||
|
||||
SERVER=dl.fedoraproject.org
|
||||
|
||||
# The Buffet.
|
||||
#${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-buffet0/ /srv/pub/ | tail -n2 | logger -p local0.notice -t rsync_updates_alt_stg
|
||||
|
||||
# Alt
|
||||
${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-alt0/ /srv/pub/alt/ | tail -n2 | logger -p local0.notice -t rsync_alt
|
||||
# Secondary
|
||||
${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-secondary/ /srv/pub/fedora-secondary/ | tail -n2 | logger -p local0.notice -t rsync_2nd
|
||||
# Archives
|
||||
${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-archive/ /srv/pub/archive/ | tail -n2 | logger -p local0.notice -t rsync_archive
|
||||
|
||||
|
||||
|
||||
# Let MM know I'm all up to date
|
||||
#/usr/bin/report_mirror
|
||||
|
|
@ -50,15 +50,18 @@
|
|||
- name: put in script for syncing on download-ib02
|
||||
copy: src="{{ files }}/download/sync-up-downloads.sh" dest=/usr/local/bin/sync-up-downloads owner=root group=root mode=755
|
||||
when: inventory_hostname == 'download-ib02.fedoraproject.org'
|
||||
- name: put in cron job for syncing
|
||||
copy: src="{{ files }}/download/download-sync.cron" dest=/etc/cron.d/download-sync owner=root group=root mode=644
|
||||
when: inventory_hostname == 'download-ib02.fedoraproject.org'
|
||||
- name: put in script for syncing on download-ib01
|
||||
copy: src="{{ files }}/download/sync-up-downloads.sh.ib01" dest=/usr/local/bin/sync-up-downloads owner=root group=root mode=755
|
||||
when: inventory_hostname == 'download-ib01.fedoraproject.org'
|
||||
- name: put in script for syncing on download-ib01
|
||||
copy: src="{{ files }}/download/sync-up-archives.sh.ib01" dest=/usr/local/bin/sync-up-archives owner=root group=root mode=755
|
||||
copy: src="{{ files }}/download/sync-up-other.sh.ib01" dest=/usr/local/bin/sync-up-other owner=root group=root mode=755
|
||||
when: inventory_hostname == 'download-ib01.fedoraproject.org'
|
||||
- name: put in cron job for syncing
|
||||
copy: src="{{ files }}/download/download-sync.cron" dest=/etc/cron.d/download-sync owner=root group=root mode=644
|
||||
when: datacenter == 'ibiblio'
|
||||
copy: src="{{ files }}/download/download-sync.cron.ib01" dest=/etc/cron.d/download-sync owner=root group=root mode=644
|
||||
when: inventory_hostname == 'download-ib01.fedoraproject.org'
|
||||
- name: put in last sync scrypt for download-ib01
|
||||
copy: src="{{ files}}/download/last-sync" dest=/usr/local/bin/last-sync mode=0755
|
||||
when: inventory_hostname == 'download-ib01.fedoraproject.org'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue