7efbd7f504
This changes the s3sync setup at multiple places: * The s3sync script was using wrong paths which are now corrected * After the sync report_mirror was started which is now installed * The necessary report_mirror files are installed * The s3sync script is now running from cron instead as a service * Before touching any files fedmsg is queried to see if new files are available for syncing * logrotate has been adapted to the new setup
84 lines
2.3 KiB
Bash
Executable file
84 lines
2.3 KiB
Bash
Executable file
#!/bin/bash
|
|
|
|
logfile="/var/log/s3-mirror/s3sync.log"
|
|
tree="/srv/pub/"
|
|
s3cmd=/usr/bin/s3cmd
|
|
curdate=`date +%s`
|
|
|
|
if [ ! -d ${tree} ]; then
|
|
echo "Specified root of the directory tree (${tree}) does not exist. Exiting." | tee ${logfile}
|
|
exit 1
|
|
fi
|
|
|
|
exec 1>>${logfile} 2>&1
|
|
|
|
function newer()
|
|
{
|
|
if [ "$1" -nt "$2" ] ; then
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
S3CMD_ARGS="sync \
|
|
--verbose \
|
|
--preserve \
|
|
--recursive \
|
|
--check-md5 \
|
|
--delay-updates \
|
|
--guess-mime-type --mime-type application/octet-stream \
|
|
--reduced-redundancy \
|
|
--acl-public \
|
|
--exclude-from /usr/local/etc/s3-mirror-excludes.txt \
|
|
"
|
|
|
|
content="epel"
|
|
targets="s3-mirror-us-east-1 s3-mirror-us-west-1 s3-mirror-us-west-2 s3-mirror-eu-west-1 s3-mirror-ap-northeast-1"
|
|
report=0
|
|
|
|
if [ -e /var/run/s3-mirror/lastrun ]; then
|
|
. /var/run/s3-mirror/lastrun
|
|
else
|
|
# 48 hours -> 172800 seconds
|
|
let lastrun=curdate-172800
|
|
fi
|
|
|
|
let delta=curdate-lastrun
|
|
|
|
# hardcoded to epel only '-e'
|
|
/usr/local/bin/last-sync -d ${delta} -e
|
|
|
|
if [ "$?" -ne "0" ]; then
|
|
# no changes since the last sync
|
|
# abort
|
|
exit 0
|
|
fi
|
|
|
|
for c in ${content}; do
|
|
if $(newer ${tree}/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist) ; then
|
|
echo "=============================================================="
|
|
echo -n "Starting at "
|
|
date
|
|
time $s3cmd $S3CMD_ARGS \
|
|
--cache-file /var/lib/s3-mirror/${c}.cache \
|
|
--delete-removed \
|
|
--delete-after \
|
|
--add-destination s3://s3-mirror-us-west-1.fedoraproject.org/pub/${c}/ \
|
|
--add-destination s3://s3-mirror-us-west-2.fedoraproject.org/pub/${c}/ \
|
|
--add-destination s3://s3-mirror-eu-west-1.fedoraproject.org/pub/${c}/ \
|
|
--add-destination s3://s3-mirror-ap-northeast-1.fedoraproject.org/pub/${c}/ \
|
|
${tree}/${c}/ s3://s3-mirror-us-east-1.fedoraproject.org/pub/${c}/
|
|
report=1
|
|
cp -a ${tree}/${c}/fullfilelist /var/lib/s3-mirror/${c}-fullfilelist
|
|
echo "lastrun=${curdate}" > /var/run/s3-mirror/lastrun
|
|
fi
|
|
done
|
|
|
|
|
|
if [ ${report} -ne 0 ] ; then
|
|
for target in ${targets} ; do
|
|
report_mirror -c /etc/mirrormanager-client/report_mirror_${target}.conf --exclude-from /usr/local/etc/s3-mirror-excludes.txt
|
|
done
|
|
echo -n "Ending at "
|
|
date
|
|
fi
|