Update create-filelist and wrapper from upstream quick mirror.

Merge all the various copies of the wrapper into one copy.
Setup arguments as needed for each call.

files/scripts/create-filelist

@@ -63,7 +63,7 @@ def parseopts():
     p.add_argument('-C', '--checksum-file', action='append', dest='checksum_files',
                    help='Include checksums of all instances of the specified file.')
     p.add_argument('-s', '--skip', action='store_true',
-                   help='Skip the --filelist file in the top directory')
+                   help='Skip the file lists in the top directory')
     p.add_argument('-S', '--skip-file', action='append', dest='skip_files',
                    help='Skip the specified file in the top directory.')
 
@@ -87,6 +87,8 @@ def parseopts():
     if opts.skip:
         if not opts.timelist.name == '<stdout>':
             opts.skip_files += [opts.timelist.name]
+        if not opts.filelist.name == '<stdout>':
+            opts.skip_files += [opts.filelist.name]
 
     return opts
 
@@ -114,14 +116,15 @@ def main():
         # opts.timelist.write('{0}\t{1}\t{2}\n'.format(modtime, ftype, entry.path[2:]))
         print('{0}\t{1}\t{2}\t{3}'.format(modtime, ftype, size, entry.path[2:]), file=opts.timelist)
 
-    if not checksums:
-        sys.exit(0)
-
     print('\n[Checksums SHA1]', file=opts.timelist)
 
+    # It's OK if the checksum section is empty, but we should include it anyway
+    # as the client expects it.
     for f in sorted(checksums):
         print('{0}\t{1}'.format(sha1(f), f), file=opts.timelist)
 
+    print('\n[End]', file=opts.timelist)
+
 
 if __name__ == '__main__':
         main()

files/scripts/update-fullfiletimelist

@@ -1,29 +1,100 @@
 #!/bin/bash
+
+# Note: this is only an example of how you'd call create-filelist.  Edit to fit
+# your requirements.  Note that you must supply a valid path for the lockfile,
+# and it must be outside of your repository unless you want that lockfile to
+# show up in your file lists.
+
 # Takes a list of module names.  Generates file lists for all of them and them
 # moves them into place at once.  If you are creating hardlinks between rsync
 # modules, it is required that you update the file lists of both mirrors at the
 # same time.  Otherwise the clients may make separate copies of the files.
 
-mods=$*
-
-if [[ -z $mods ]]; then
-    echo "usage: $0 <module> [<module> ...]"
-    exit 2
-fi
-
-CREATE=/usr/local/bin/create-filelist
-
-# A single lockfile for everything we're modifying
-LOCKFILE=/srv/.lock.create-filelist
-
 # The directory where all of the modules live
-TOPD=/srv/pub
+# Or pass it with -t
+TOPD=/srv/mirror/pub
+
+# The modules to process.  Or pass them on the command line.
+MODS=()
+
+# Path to the create-filelist program.
+# Or specify it with -p.
+CREATE=/usr/local/bin/create-filelist
 
 # These strings will be eval'ed later with $mod replaced by its value in
 # context.
 FILELIST=fullfilelist
 TIMELIST='fullfiletimelist-$mod'
 
+usage () {
+    echo
+    echo "Usage: $0 [-l lockfile] [-p creator path] [-t top directory] module [module ...]"
+    echo
+    echo "    -l: Path to the lock file"
+    echo "    -p: Path to the create-filelist program"
+    echo "    -t: Path to directory containing modules"
+    echo
+    echo "At least one module to process must be provided."
+    echo "All paths must be absolute."
+}
+
+while [[ $# > 0 ]]; do
+    opt=$1
+    case $opt in
+        -l)
+            LOCKFILE=$(realpath $2)
+            shift
+            ;;
+        -p)
+            CREATE=$(realpath $2)
+            shift
+            ;;
+        -t)
+            TOPD=$(realpath $2)
+            shift
+            ;;
+        -*)
+            (>&2 echo "Unknown option $opt."; usage)
+            exit 1
+            ;;
+        *)  # Remaining args are modules
+            MODS+=($opt)
+            ;;
+    esac
+    shift
+done
+
+if [[ -z $LOCKFILE ]]; then
+    (>&2 echo "Must specify LOCKFILE, either by editing the source or via the -l option."; usage)
+    exit 2
+fi
+if [[ ! -d $(dirname $LOCKFILE) ]]; then
+    (>&2 echo "Given directory $(dirname $LOCKFILE) does not exist."; usage)
+    exit 2
+fi
+if [[ ! -f $CREATE ]]; then
+    (>&2 echo "Specified executable $CREATE does not exist."; usage)
+    exit 2
+fi
+
+if [[ ! -d $TOPD ]]; then
+    (>&2 echo "Provided directory $TOPD does not exist."; usage)
+    exit 2
+fi
+
+if [[ ${#MODS[@]} -eq 0 ]]; then
+    (>&2 echo "No modules specified"; usage)
+    exit 2
+fi
+
+tmpd=$(mktemp -d -t create-filelist.XXXXXXXXXX)
+if [[ $? -ne 0 ]]; then
+    (>&2 echo "Creating temporary directory failed?")
+    exit 1
+fi
+trap "rm -rf $tmpd" EXIT
+cd $tmpd
+
 (
     # We want to wait forever until we can do what we're asked
     flock -x 9
@@ -33,42 +104,46 @@ TIMELIST='fullfiletimelist-$mod'
     # flock -w 120 9 || exit 1  - Waits 120 seconds and then gives up
     # Don't change the '9', unless you change the last line of this script.
 
-    tmpd=$(mktemp -d -t create-filelist.XXXXXXXXXX)
+    for mod in $MODS; do
-    trap "rm -rf $tmpd" EXIT
-    cd $tmpd
-
-    # Create all of the filelists
-    for mod in $mods; do
-        $CREATE -c -s -d $TOPD/$mod -f fl-$mod -t tl-$mod
-    done
-
-    # Now diff the file lists and delete the ones which didn't change
-    for mod in $mods; do
         currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod}
         currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod}
+        flname=$(basename $currentfl)
+        tlname=$(basename $currenttl)
+
+        $CREATE -c -s -d $TOPD/$mod -f $flname -t $tlname
 
         # If a file list exsts and doesn't differ from what we just generated,
         # delete the latter.
-        if [[ -f $currentfl ]] && diff -q fl-$mod $currentfl > /dev/null; then
+        if [[ -f $currentfl ]] && diff -q $currentfl $flname > /dev/null; then
-            rm -f $fl
+            rm -f $flname
         fi
-        if [[ -f $currenttl ]] && diff -q tl-$mod $currenttl > /dev/null; then
+        if [[ -f $currenttl ]] && diff -q $currenttl $tlname > /dev/null; then
-            rm -f $fl
+            rm -f $tlname
         fi
     done
 
-    # And finally, move all of the files which need updating into place
+    # Now we have the new file lists but in a temporary directory which
-    for mod in $mods; do
+    # probably isn't on the same filesystem.  Copy them to temporary files in
+    # the right place.
+    for mod in $MODS; do
         currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod}
         currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod}
+        flname=$(basename $currentfl)
+        fldir=$(dirname $currentfl)
+        tlname=$(basename $currenttl)
+        tldir=$(dirname $currenttl)
 
-        if [[ -f fl-$mod ]]; then
+        if [[ -f $flname ]]; then
-            chmod 644 fl-$mod
+            tmpf=$(mktemp -p $fldir $flname.XXXXXXXXXX)
-            cp -p fl-$mod $currentfl
+            cp -p $flname $tmpf
+            chmod 644 $tmpf
+            mv $tmpf $currentfl
         fi
-        if [[ -f tl-$mod ]]; then
+        if [[ -f $tlname ]]; then
-            chmod 644 tl-$mod
+            tmpf=$(mktemp -p $tldir $tlname.XXXXXXXXXX)
-            cp -p tl-$mod $currenttl
+            cp -p $tlname $tmpf
+            chmod 644 $tmpf
+            mv $tmpf $currenttl
         fi
     done
 

playbooks/groups/secondary.yml

@@ -67,7 +67,7 @@
 
   - name: Update fullfiletimelist job
     cron: name="update-fullfiletimelist" hour="*" minute="55" user="root"
-        job="/usr/local/bin/lock-wrapper update-fullfiletimelist '/usr/local/bin/update-fullfiletimelist alt'"
+        job="/usr/local/bin/lock-wrapper update-fullfiletimelist '/usr/local/bin/update-fullfiletimelist -l /tmp/update-fullfiletimelist.lock -t /srv/pub alt'"
         cron_file=update-fullfiletimelist
 
   handlers:

roles/bodhi2/backend/files/update-fullfilelist

@@ -1,75 +0,0 @@
-#!/bin/bash
-# Takes a list of module names.  Generates file lists for all of them and them
-# moves them into place at once.  If you are creating hardlinks between rsync
-# modules, it is required that you update the file lists of both mirrors at the
-# same time.  Otherwise the clients may make separate copies of the files.
-
-mods=$*
-
-if [[ -z $mods ]]; then
-    echo "usage: $0 <module> [<module> ...]"
-    exit 2
-fi
-
-CREATE=/usr/local/bin/create-filelist
-
-# A single lockfile for everything we're modifying
-LOCKFILE=.lock.create-filelist
-
-# The directory where all of the modules live
-TOPD=/pub
-
-# These strings will be eval'ed later with $mod replaced by its value in
-# context.
-FILELIST=fullfilelist
-TIMELIST='fullfiletimelist-$mod'
-
-(
-    # We want to wait forever until we can do what we're asked
-    flock -x 9
-
-    # If you don't want to wait forever, try one of the following:
-    # flock -n 9 || exit 1      - Gives up immediately
-    # flock -w 120 9 || exit 1  - Waits 120 seconds and then gives up
-    # Don't change the '9', unless you change the last line of this script.
-
-    tmpd=$(mktemp -d -t create-filelist.XXXXXXXXXX)
-    trap "rm -rf $tmpd" EXIT
-    cd $tmpd
-
-    # Create all of the filelists
-    for mod in $mods; do
-        $CREATE -c -s -d $TOPD/$mod -f fl-$mod -t tl-$mod
-    done
-
-    # Now diff the file lists and delete the ones which didn't change
-    for mod in $mods; do
-        currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod}
-        currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod}
-
-        # If a file list exsts and doesn't differ from what we just generated,
-        # delete the latter.
-        if [[ -f $currentfl ]] && diff -q fl-$mod $currentfl > /dev/null; then
-            rm -f $fl
-        fi
-        if [[ -f $currenttl ]] && diff -q tl-$mod $currenttl > /dev/null; then
-            rm -f $fl
-        fi
-    done
-
-    # And finally, move all of the files which need updating into place
-    for mod in $mods; do
-        currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod}
-        currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod}
-
-        if [[ -f fl-$mod ]]; then
-            chmod 644 fl-$mod
-            cp -p fl-$mod $currentfl
-        fi
-        if [[ -f tl-$mod ]]; then
-            chmod 644 tl-$mod
-            cp -p tl-$mod $currenttl
-        fi
-    done
-
-) 9>$LOCKFILE

roles/bodhi2/backend/tasks/main.yml

@@ -211,8 +211,8 @@
   - bodhi
   - cron
 
-- name: put update-fullfilelist in place
+- name: put update-fullfiletimelist in place
-  copy: src=update-fullfilelist dest=/usr/local/bin/update-fullfilelist mode=0755
+  copy: src="{{ files }}/scripts/update-fullfiletimelist" dest=/usr/local/bin/update-fullfiletimelist mode=0755
   when: inventory_hostname.startswith('bodhi-backend01') and env == "production"
   tags:
   - config
@@ -229,7 +229,7 @@
 
 - name: Updates sync cron job. 
   cron: name="updates-sync" minute="15,45" user="ftpsync"
-        job="/usr/local/bin/lock-wrapper fedora-updates-push '/usr/local/bin/fedora-updates-push && /usr/local/bin/update-fullfilelist fedora'"
+        job="/usr/local/bin/lock-wrapper fedora-updates-push '/usr/local/bin/fedora-updates-push && /usr/local/bin/update-fullfiletimelist -l /tmp/update-fullfiletimelist.lock -t /pub fedora'"
         cron_file=updates-sync
   when: inventory_hostname.startswith('bodhi-backend01') and env == "production"
   tags:

roles/releng/files/branched

@@ -1,3 +1,3 @@
 # branched compose
 MAILTO=releng-cron@lists.fedoraproject.org
-#15 7 * * * root TMPDIR=`mktemp -d /tmp/branched.XXXXXX` && cd $TMPDIR && git clone https://pagure.io/pungi-fedora.git && cd pungi-fedora && git checkout f24 && LANG=en_US.UTF-8 ./nightly.sh && sudo -u ftpsync /usr/local/bin/update-fullfilelist fedora
+#15 7 * * * root TMPDIR=`mktemp -d /tmp/branched.XXXXXX` && cd $TMPDIR && git clone https://pagure.io/pungi-fedora.git && cd pungi-fedora && git checkout f24 && LANG=en_US.UTF-8 ./nightly.sh && sudo -u ftpsync /usr/local/bin/update-fullfiletimelist -l /tmp/update-fullfiletimelist.lock -t /pub fedora

roles/releng/files/rawhide

@@ -1,3 +1,3 @@
 # rawhide compose
 MAILTO=releng-cron@lists.fedoraproject.org
-15 5 * * * root TMPDIR=`mktemp -d /tmp/rawhide.XXXXXX` && cd $TMPDIR && git clone https://pagure.io/pungi-fedora.git && cd pungi-fedora && LANG=en_US.UTF-8 ./nightly.sh && sudo -u ftpsync /usr/local/bin/update-fullfilelist fedora
+15 5 * * * root TMPDIR=`mktemp -d /tmp/rawhide.XXXXXX` && cd $TMPDIR && git clone https://pagure.io/pungi-fedora.git && cd pungi-fedora && LANG=en_US.UTF-8 ./nightly.sh && sudo -u ftpsync /usr/local/bin/update-fullfiletimelist -l /tmp/update-fullfiletimelist.lock -t /pub fedora

roles/releng/files/update-fullfilelist

@@ -1,75 +0,0 @@
-#!/bin/bash
-# Takes a list of module names.  Generates file lists for all of them and them
-# moves them into place at once.  If you are creating hardlinks between rsync
-# modules, it is required that you update the file lists of both mirrors at the
-# same time.  Otherwise the clients may make separate copies of the files.
-
-mods=$*
-
-if [[ -z $mods ]]; then
-    echo "usage: $0 <module> [<module> ...]"
-    exit 2
-fi
-
-CREATE=/usr/local/bin/create-filelist
-
-# A single lockfile for everything we're modifying
-LOCKFILE=.lock.create-filelist
-
-# The directory where all of the modules live
-TOPD=/pub
-
-# These strings will be eval'ed later with $mod replaced by its value in
-# context.
-FILELIST=fullfilelist
-TIMELIST='fullfiletimelist-$mod'
-
-(
-    # We want to wait forever until we can do what we're asked
-    flock -x 9
-
-    # If you don't want to wait forever, try one of the following:
-    # flock -n 9 || exit 1      - Gives up immediately
-    # flock -w 120 9 || exit 1  - Waits 120 seconds and then gives up
-    # Don't change the '9', unless you change the last line of this script.
-
-    tmpd=$(mktemp -d -t create-filelist.XXXXXXXXXX)
-    trap "rm -rf $tmpd" EXIT
-    cd $tmpd
-
-    # Create all of the filelists
-    for mod in $mods; do
-        $CREATE -c -s -d $TOPD/$mod -f fl-$mod -t tl-$mod
-    done
-
-    # Now diff the file lists and delete the ones which didn't change
-    for mod in $mods; do
-        currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod}
-        currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod}
-
-        # If a file list exsts and doesn't differ from what we just generated,
-        # delete the latter.
-        if [[ -f $currentfl ]] && diff -q fl-$mod $currentfl > /dev/null; then
-            rm -f $fl
-        fi
-        if [[ -f $currenttl ]] && diff -q tl-$mod $currenttl > /dev/null; then
-            rm -f $fl
-        fi
-    done
-
-    # And finally, move all of the files which need updating into place
-    for mod in $mods; do
-        currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod}
-        currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod}
-
-        if [[ -f fl-$mod ]]; then
-            chmod 644 fl-$mod
-            cp -p fl-$mod $currentfl
-        fi
-        if [[ -f tl-$mod ]]; then
-            chmod 644 tl-$mod
-            cp -p tl-$mod $currenttl
-        fi
-    done
-
-) 9>$LOCKFILE

roles/releng/tasks/main.yml

@@ -21,8 +21,8 @@
 - name: add create-filelist script from quick-fedora-mirror
   copy: src="{{ files }}/scripts/create-filelist" dest=/usr/local/bin/create-filelist mode=0755 owner=ftpsync group=ftpsync
 
-- name: add the ftpsync update-fullfilelist script
+- name: add the ftpsync update-fullfiletimelist script
-  copy: src=update-fullfilelist dest=/usr/local/bin/update-fullfilelist owner=ftpsync group=ftpsync mode=555
+  copy: src="{{ files }}/scripts/update-fullfiletimelist" dest=/usr/local/bin/update-fullfiletimelist mode=0755
 
 - name: add masher group
   group: name=masher gid=751 system=yes state=present