Update create-filelist and wrapper from upstream quick mirror.
Merge all the various copies of the wrapper into one copy. Setup arguments as needed for each call.
This commit is contained in:
Kevin Fenzi
parent
6685841b7c
commit
03a8923c04
9 changed files with 126 additions and 198 deletions
|
|
@ -63,7 +63,7 @@ def parseopts():
|
|||
p.add_argument('-C', '--checksum-file', action='append', dest='checksum_files',
|
||||
help='Include checksums of all instances of the specified file.')
|
||||
p.add_argument('-s', '--skip', action='store_true',
|
||||
help='Skip the --filelist file in the top directory')
|
||||
help='Skip the file lists in the top directory')
|
||||
p.add_argument('-S', '--skip-file', action='append', dest='skip_files',
|
||||
help='Skip the specified file in the top directory.')
|
||||
|
||||
|
|
@ -87,6 +87,8 @@ def parseopts():
|
|||
if opts.skip:
|
||||
if not opts.timelist.name == '<stdout>':
|
||||
opts.skip_files += [opts.timelist.name]
|
||||
if not opts.filelist.name == '<stdout>':
|
||||
opts.skip_files += [opts.filelist.name]
|
||||
|
||||
return opts
|
||||
|
||||
|
|
@ -114,14 +116,15 @@ def main():
|
|||
# opts.timelist.write('{0}\t{1}\t{2}\n'.format(modtime, ftype, entry.path[2:]))
|
||||
print('{0}\t{1}\t{2}\t{3}'.format(modtime, ftype, size, entry.path[2:]), file=opts.timelist)
|
||||
|
||||
if not checksums:
|
||||
sys.exit(0)
|
||||
|
||||
print('\n[Checksums SHA1]', file=opts.timelist)
|
||||
|
||||
# It's OK if the checksum section is empty, but we should include it anyway
|
||||
# as the client expects it.
|
||||
for f in sorted(checksums):
|
||||
print('{0}\t{1}'.format(sha1(f), f), file=opts.timelist)
|
||||
|
||||
print('\n[End]', file=opts.timelist)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,29 +1,100 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Note: this is only an example of how you'd call create-filelist. Edit to fit
|
||||
# your requirements. Note that you must supply a valid path for the lockfile,
|
||||
# and it must be outside of your repository unless you want that lockfile to
|
||||
# show up in your file lists.
|
||||
|
||||
# Takes a list of module names. Generates file lists for all of them and them
|
||||
# moves them into place at once. If you are creating hardlinks between rsync
|
||||
# modules, it is required that you update the file lists of both mirrors at the
|
||||
# same time. Otherwise the clients may make separate copies of the files.
|
||||
|
||||
mods=$*
|
||||
|
||||
if [[ -z $mods ]]; then
|
||||
echo "usage: $0 <module> [<module> ...]"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
CREATE=/usr/local/bin/create-filelist
|
||||
|
||||
# A single lockfile for everything we're modifying
|
||||
LOCKFILE=/srv/.lock.create-filelist
|
||||
|
||||
# The directory where all of the modules live
|
||||
TOPD=/srv/pub
|
||||
# Or pass it with -t
|
||||
TOPD=/srv/mirror/pub
|
||||
|
||||
# The modules to process. Or pass them on the command line.
|
||||
MODS=()
|
||||
|
||||
# Path to the create-filelist program.
|
||||
# Or specify it with -p.
|
||||
CREATE=/usr/local/bin/create-filelist
|
||||
|
||||
# These strings will be eval'ed later with $mod replaced by its value in
|
||||
# context.
|
||||
FILELIST=fullfilelist
|
||||
TIMELIST='fullfiletimelist-$mod'
|
||||
|
||||
usage () {
|
||||
echo
|
||||
echo "Usage: $0 [-l lockfile] [-p creator path] [-t top directory] module [module ...]"
|
||||
echo
|
||||
echo " -l: Path to the lock file"
|
||||
echo " -p: Path to the create-filelist program"
|
||||
echo " -t: Path to directory containing modules"
|
||||
echo
|
||||
echo "At least one module to process must be provided."
|
||||
echo "All paths must be absolute."
|
||||
}
|
||||
|
||||
while [[ $# > 0 ]]; do
|
||||
opt=$1
|
||||
case $opt in
|
||||
-l)
|
||||
LOCKFILE=$(realpath $2)
|
||||
shift
|
||||
;;
|
||||
-p)
|
||||
CREATE=$(realpath $2)
|
||||
shift
|
||||
;;
|
||||
-t)
|
||||
TOPD=$(realpath $2)
|
||||
shift
|
||||
;;
|
||||
-*)
|
||||
(>&2 echo "Unknown option $opt."; usage)
|
||||
exit 1
|
||||
;;
|
||||
*) # Remaining args are modules
|
||||
MODS+=($opt)
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
if [[ -z $LOCKFILE ]]; then
|
||||
(>&2 echo "Must specify LOCKFILE, either by editing the source or via the -l option."; usage)
|
||||
exit 2
|
||||
fi
|
||||
if [[ ! -d $(dirname $LOCKFILE) ]]; then
|
||||
(>&2 echo "Given directory $(dirname $LOCKFILE) does not exist."; usage)
|
||||
exit 2
|
||||
fi
|
||||
if [[ ! -f $CREATE ]]; then
|
||||
(>&2 echo "Specified executable $CREATE does not exist."; usage)
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [[ ! -d $TOPD ]]; then
|
||||
(>&2 echo "Provided directory $TOPD does not exist."; usage)
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [[ ${#MODS[@]} -eq 0 ]]; then
|
||||
(>&2 echo "No modules specified"; usage)
|
||||
exit 2
|
||||
fi
|
||||
|
||||
tmpd=$(mktemp -d -t create-filelist.XXXXXXXXXX)
|
||||
if [[ $? -ne 0 ]]; then
|
||||
(>&2 echo "Creating temporary directory failed?")
|
||||
exit 1
|
||||
fi
|
||||
trap "rm -rf $tmpd" EXIT
|
||||
cd $tmpd
|
||||
|
||||
(
|
||||
# We want to wait forever until we can do what we're asked
|
||||
flock -x 9
|
||||
|
|
@ -33,42 +104,46 @@ TIMELIST='fullfiletimelist-$mod'
|
|||
# flock -w 120 9 || exit 1 - Waits 120 seconds and then gives up
|
||||
# Don't change the '9', unless you change the last line of this script.
|
||||
|
||||
tmpd=$(mktemp -d -t create-filelist.XXXXXXXXXX)
|
||||
trap "rm -rf $tmpd" EXIT
|
||||
cd $tmpd
|
||||
|
||||
# Create all of the filelists
|
||||
for mod in $mods; do
|
||||
$CREATE -c -s -d $TOPD/$mod -f fl-$mod -t tl-$mod
|
||||
done
|
||||
|
||||
# Now diff the file lists and delete the ones which didn't change
|
||||
for mod in $mods; do
|
||||
for mod in $MODS; do
|
||||
currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod}
|
||||
currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod}
|
||||
flname=$(basename $currentfl)
|
||||
tlname=$(basename $currenttl)
|
||||
|
||||
$CREATE -c -s -d $TOPD/$mod -f $flname -t $tlname
|
||||
|
||||
# If a file list exsts and doesn't differ from what we just generated,
|
||||
# delete the latter.
|
||||
if [[ -f $currentfl ]] && diff -q fl-$mod $currentfl > /dev/null; then
|
||||
rm -f $fl
|
||||
if [[ -f $currentfl ]] && diff -q $currentfl $flname > /dev/null; then
|
||||
rm -f $flname
|
||||
fi
|
||||
if [[ -f $currenttl ]] && diff -q tl-$mod $currenttl > /dev/null; then
|
||||
rm -f $fl
|
||||
if [[ -f $currenttl ]] && diff -q $currenttl $tlname > /dev/null; then
|
||||
rm -f $tlname
|
||||
fi
|
||||
done
|
||||
|
||||
# And finally, move all of the files which need updating into place
|
||||
for mod in $mods; do
|
||||
# Now we have the new file lists but in a temporary directory which
|
||||
# probably isn't on the same filesystem. Copy them to temporary files in
|
||||
# the right place.
|
||||
for mod in $MODS; do
|
||||
currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod}
|
||||
currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod}
|
||||
flname=$(basename $currentfl)
|
||||
fldir=$(dirname $currentfl)
|
||||
tlname=$(basename $currenttl)
|
||||
tldir=$(dirname $currenttl)
|
||||
|
||||
if [[ -f fl-$mod ]]; then
|
||||
chmod 644 fl-$mod
|
||||
cp -p fl-$mod $currentfl
|
||||
if [[ -f $flname ]]; then
|
||||
tmpf=$(mktemp -p $fldir $flname.XXXXXXXXXX)
|
||||
cp -p $flname $tmpf
|
||||
chmod 644 $tmpf
|
||||
mv $tmpf $currentfl
|
||||
fi
|
||||
if [[ -f tl-$mod ]]; then
|
||||
chmod 644 tl-$mod
|
||||
cp -p tl-$mod $currenttl
|
||||
if [[ -f $tlname ]]; then
|
||||
tmpf=$(mktemp -p $tldir $tlname.XXXXXXXXXX)
|
||||
cp -p $tlname $tmpf
|
||||
chmod 644 $tmpf
|
||||
mv $tmpf $currenttl
|
||||
fi
|
||||
done
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue