Infrastructure/ansible
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

and we have a simple script to try and make updates faster

Browse source
This commit is contained in:
Stephen Smoogen 2016-04-06 16:00:30 +00:00
parent 790ba6f9fd
commit cb983c1f1a
6 changed files with 147 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

4
files/download/download-sync.cron
View file

@ -1,4 +1,4 @@
# run twice daily rsync of download. but lock it
MAILTO=smooge@gmail.com
MAILTO=smooge@gmail.com,root@fedoraproject.org
00 11,23 * * * root /usr/local/bin/lock-wrapper sync-up-downloads "/usr/local/bin/sync-up-downloads"
00 23 * * * root /usr/local/bin/lock-wrapper sync-up-archives "/usr/local/bin/sync-up-archives"

4
files/download/download-sync.cron.ib01 Normal file
View file

@ -0,0 +1,4 @@
# run twice daily rsync of download. but lock it
MAILTO=smooge@gmail.com,root@fedoraproject.org
00 11,23 * * * root /usr/local/bin/lock-wrapper sync-up-downloads "/usr/local/bin/sync-up-downloads"
00 11,23 * * * root /usr/local/bin/lock-wrapper sync-up-other "/usr/local/bin/sync-up-other"

161
files/download/last-sync
View file

@ -42,112 +42,121 @@ delta = 86400
# if no sync happened 1 is returned
quiet = False
secondary = False
rawtime = False
def usage():
print
print "last-sync queries the Fedora Message Bus if new data is available on the public servers"
print
print "Usage: last-sync [options]"
print
print "Options:"
print " -a, --all query all possible releases (default)"
print " (fedora, epel, branched, rawhide)"
print " -f, --fedora only query if fedora has been updated during <delta>"
print " -e, --epel only query if epel has been updated"
print " -b, --branched only query if the branched off release"
print " has been updated"
print " -r, --rawhide only query if rawhide has been updated"
print " -q, --quiet do not print out any informations"
print " -d DELTA, --delta=DELTA specify the time interval which should be used"
print " for the query (default: 86400)"
print
print "last-sync queries the Fedora Message Bus if new data is available on the public servers"
print
print "Usage: last-sync [options]"
print
print "Options:"
print " -a, --all query all possible releases (default)"
print " (fedora, epel, branched, rawhide)"
print " -f, --fedora only query if fedora has been updated during <delta>"
print " -e, --epel only query if epel has been updated"
print " -b, --branched only query if the branched off release"
print " has been updated"
print " -r, --rawhide only query if rawhide has been updated"
print " -q, --quiet do not print out any informations"
print " -t, --time print date in seconds since 1970-01-01"
print " -d DELTA, --delta=DELTA specify the time interval which should be used"
print " for the query (default: 86400)"
# -a -f -e -b -r -s -q -d
def parse_args():
global topics
global delta
global quiet
global secondary
try:
opts, args = getopt.getopt(sys.argv[1:], "afhebrsqd:", ["all", "fedora", "epel", "rawhide", "branched", "secondary", "quiet", "delta="])
except getopt.GetoptError as err:
print str(err)
usage()
sys.exit(2)
global topics
global delta
global quiet
global secondary
global rawtime
try:
opts, args = getopt.getopt(sys.argv[1:], "afhebrsqtd:", ["all", "fedora", "epel", "rawhide", "branched", "secondary", "quiet", "time", "delta="])
except getopt.GetoptError as err:
print str(err)
usage()
sys.exit(2)
for option, argument in opts:
if option in ("-a", "--all"):
topics = [ fedora, epel, branched, rawhide ]
secondary = True
if option in ("-f", "--fedora"):
topics.append(fedora)
if option in ("-e", "--epel"):
topics.append(epel)
if option in ("-r", "--rawhide"):
topics.append(rawhide)
if option in ("-b", "--branched"):
topics.append(branched)
if option in ("-s", "--secondary"):
topics.append(rawhide)
secondary = True
if option in ("-q", "--quiet"):
quiet = True
if option in ("-d", "--delta"):
delta = argument
if option in ("-h"):
usage();
sys.exit(0)
for option, argument in opts:
if option in ("-a", "--all"):
topics = [ fedora, epel, branched, rawhide ]
secondary = True
if option in ("-f", "--fedora"):
topics.append(fedora)
if option in ("-e", "--epel"):
topics.append(epel)
if option in ("-r", "--rawhide"):
topics.append(rawhide)
if option in ("-b", "--branched"):
topics.append(branched)
if option in ("-s", "--secondary"):
topics.append(rawhide)
secondary = True
if option in ("-q", "--quiet"):
quiet = True
if option in ("-t", "--time"):
rawtime = True
if option in ("-d", "--delta"):
delta = argument
if option in ("-h"):
usage();
sys.exit(0)
def getKey(item):
return item[1]
return item[1]
def create_url(url, topics, delta):
topic = ""
for i in topics:
topic += "&topic=%s" % i
return '%s?delta=%s%s' % (url, delta, topic)
topic = ""
for i in topics:
topic += "&topic=%s" % i
return '%s?delta=%s%s' % (url, delta, topic)
parse_args()
if topics == []:
topics = [ fedora, epel, branched, rawhide ]
secondary = True
topics = [ fedora, epel, branched, rawhide ]
secondary = True
i = 0
data = None
while i < 5:
try:
data = requests.get(create_url(base_url, topics, delta), timeout=1).json()
break
except:
pass
try:
data = requests.get(create_url(base_url, topics, delta), timeout=1).json()
break
except:
pass
if not data:
sys.exit(1)
sys.exit(1)
repos = []
for i in range(0, data['count']):
try:
repo = "%s-%s" % (data['raw_messages'][i]['msg']['repo'], data['raw_messages'][i]['msg']['release'])
except:
# the rawhide and branch sync message has no repo information
arch = data['raw_messages'][i]['msg']['arch']
if arch == '':
arch = 'primary'
elif not secondary:
continue
repo = "%s-%s" % (data['raw_messages'][i]['msg']['branch'], arch)
try:
repo = "%s-%s" % (data['raw_messages'][i]['msg']['repo'], data['raw_messages'][i]['msg']['release'])
except:
# the rawhide and branch sync message has no repo information
arch = data['raw_messages'][i]['msg']['arch']
if arch == '':
arch = 'primary'
elif not secondary:
continue
repo = "%s-%s" % (data['raw_messages'][i]['msg']['branch'], arch)
repos.append([repo, data['raw_messages'][i]['timestamp']])
repos.append([repo, data['raw_messages'][i]['timestamp']])
if quiet == False:
for repo, timestamp in sorted(repos, key=getKey):
print "%s: %s" % (repo, time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(timestamp)))
for repo, timestamp in sorted(repos, key=getKey):
if rawtime == True:
# this is useful if you want to compare the timestamp in seconds versus string
print "%s: %s" % (repo, timestamp)
else:
print "%s: %s" % (repo, time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(timestamp)))
if data['count'] > 0:
sys.exit(0)
sys.exit(0)
else:
sys.exit(1)
sys.exit(1)

56
files/download/sync-up-downloads.sh.ib01
View file

@ -12,21 +12,57 @@ ALT_EXCLUDES=""
EPL_EXCLUDES=""
FED_EXCLUDES=""
DATE_EPEL='/root/last-epel-sync'
DATE_FED='/root/last-fed-sync'
touch ${DATE_EPEL}
touch ${DATE_FED}
LAST_SYNC='/usr/local/bin/last-sync'
SERVER=dl.fedoraproject.org
# The Buffet.
#${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-buffet0/ /srv/pub/ | tail -n2 | logger -p local0.notice -t rsync_updates_alt_stg
## Do the EPEL download if we have had a sync since the last update.
CURDATE=`date +%s`
if [[ -s ${DATE_EPEL} ]]; then
LASTRUN=$( cat DATE_EPEL | awk '{print int($NF)}' )
else
LASTRUN=$( date +%s --date="Jan 1 00:00:00 UTC 2007" )
fi
DELTA=`echo ${CURDATE}-${LASTRUN} | bc`
/path/to/last-sync -d ${DELTA} -q
if [ "$?" -eq "0" ]; then
# EPEL
${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-epel0/ /srv/pub/epel/ | tail -n2 | logger -p local0.notice -t rsync_epel
echo ${CURDATE} > ${DATE_EPEL}
else
logger -p local0.notice -t rsync_epel "No change found. Not syncing"
fi
## Do the FEDORA download if we have had a sync since the last update.
CURDATE=`date +%s`
if [[ -s ${DATE_FED} ]]; then
LASTRUN=$( cat DATE_FED | awk '{print int($NF)}' )
else
LASTRUN=$( date +%s --date="Jan 1 00:00:00 UTC 2007" )
fi
DELTA=`echo ${CURDATE}-${LASTRUN} | bc`
/path/to/last-sync -d ${DELTA} -q
if [ "$?" -eq "0" ]; then
# Enchilada
${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-enchilada0/ /srv/pub/fedora/ | tail -n2 | logger -p local0.notice -t rsync_fedora
echo ${CURDATE} > ${DATE_FED}
else
logger -p local0.notice -t rsync_fedora "No change found. Not syncing"
fi
# Alt
${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-alt0/ /srv/pub/alt/ | tail -n2 | logger -p local0.notice -t rsync_alt
# EPEL
${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-epel0/ /srv/pub/epel/ | tail -n2 | logger -p local0.notice -t rsync_epel
# Enchilada
${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-enchilada0/ /srv/pub/fedora/ | tail -n2 | logger -p local0.notice -t rsync_fedora
# Secondary
${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-secondary/ /srv/pub/fedora-secondary/ | tail -n2 | logger -p local0.notice -t rsync_2nd
# Let MM know I'm all up to date
#/usr/bin/report_mirror

8
files/download/sync-up-archives.sh.ib01 → files/download/sync-up-other.sh.ib01
View file

@ -16,13 +16,13 @@ LAST_SYNC='/usr/local/bin/last-sync'
SERVER=dl.fedoraproject.org
# The Buffet.
#${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-buffet0/ /srv/pub/ | tail -n2 | logger -p local0.notice -t rsync_updates_alt_stg
# Alt
${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-alt0/ /srv/pub/alt/ | tail -n2 | logger -p local0.notice -t rsync_alt
# Secondary
${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-secondary/ /srv/pub/fedora-secondary/ | tail -n2 | logger -p local0.notice -t rsync_2nd
# Archives
${RSYNC} ${RS_OPT} ${RS_DEADLY} ${ALT_EXCLUDES} ${SERVER}::fedora-archive/ /srv/pub/archive/ | tail -n2 | logger -p local0.notice -t rsync_archive
# Let MM know I'm all up to date
#/usr/bin/report_mirror

9
playbooks/groups/download.yml
View file

@ -50,15 +50,18 @@
- name: put in script for syncing on download-ib02
copy: src="{{ files }}/download/sync-up-downloads.sh" dest=/usr/local/bin/sync-up-downloads owner=root group=root mode=755
when: inventory_hostname == 'download-ib02.fedoraproject.org'
- name: put in cron job for syncing
copy: src="{{ files }}/download/download-sync.cron" dest=/etc/cron.d/download-sync owner=root group=root mode=644
when: inventory_hostname == 'download-ib02.fedoraproject.org'
- name: put in script for syncing on download-ib01
copy: src="{{ files }}/download/sync-up-downloads.sh.ib01" dest=/usr/local/bin/sync-up-downloads owner=root group=root mode=755
when: inventory_hostname == 'download-ib01.fedoraproject.org'
- name: put in script for syncing on download-ib01
copy: src="{{ files }}/download/sync-up-archives.sh.ib01" dest=/usr/local/bin/sync-up-archives owner=root group=root mode=755
copy: src="{{ files }}/download/sync-up-other.sh.ib01" dest=/usr/local/bin/sync-up-other owner=root group=root mode=755
when: inventory_hostname == 'download-ib01.fedoraproject.org'
- name: put in cron job for syncing
copy: src="{{ files }}/download/download-sync.cron" dest=/etc/cron.d/download-sync owner=root group=root mode=644
when: datacenter == 'ibiblio'
copy: src="{{ files }}/download/download-sync.cron.ib01" dest=/etc/cron.d/download-sync owner=root group=root mode=644
when: inventory_hostname == 'download-ib01.fedoraproject.org'
- name: put in last sync scrypt for download-ib01
copy: src="{{ files}}/download/last-sync" dest=/usr/local/bin/last-sync mode=0755
when: inventory_hostname == 'download-ib01.fedoraproject.org'