Put new create-filelist in scripts so we can use it all the places we need.

files/scripts/create-filelist

@@ -0,0 +1,118 @@
+#!/usr/bin/python
+from __future__ import print_function
+
+# A simple script to generate a file list in a format easily consumable by a
+# shell script.
+
+# Originally written by Jason Tibbitts <tibbs@math.uh.edu> in 2016.
+# Donated to the public domain.  If you require a statement of license, please
+# consider this work to be licensed as "CC0 Universal", any version you choose.
+
+import argparse
+import hashlib
+import os
+import sys
+from scandir import scandir
+
+
+def get_ftype(entry):
+    """Return a simple indicator of the file type."""
+    if entry.is_symlink():
+        return 'l'
+    if entry.is_dir():
+        return 'd'
+    return 'f'
+
+
+def sha1(fname):
+    """Return the SHA1 checksum of a file in hex."""
+    fh = open(fname, 'rb')
+    sha1 = hashlib.sha1()
+    block = fh.read(2 ** 16)
+    while len(block) > 0:
+        sha1.update(block)
+        block = fh.read(2 ** 16)
+
+    return sha1.hexdigest()
+
+
+def recursedir(path='.'):
+    """Just like scandir, but recursively."""
+    for entry in scandir(path):
+        if entry.is_dir(follow_symlinks=False):
+            for rentry in recursedir(entry.path):
+                yield rentry
+        yield entry
+
+
+def parseopts():
+    p = argparse.ArgumentParser(
+        description='Generate a list of files and times, suitable for consumption by quick-fedora-mirror.')
+    p.add_argument('-c', '--checksum', action='store_true',
+                   help='Include checksums of all repomd.xml files in the file list.')
+    p.add_argument('-C', '--checksum-file', action='append', dest='checksum_files',
+                   help='Include checksums of all instances of the specified file.')
+    p.add_argument('-s', '--skip', action='store_true',
+                   help='Skip fullfiletimelist in the top directory')
+    p.add_argument('-S', '--skip-file', action='append', dest='skip_files',
+                   help='Skip the specified file in the top directory.')
+
+    p.add_argument('-d', '--dir', help='Directory to scan (default: .).')
+
+    p.add_argument('-t', '--timelist', type=argparse.FileType('w'), default=sys.stdout,
+                   help='Filename of the file list with times (default: fullfiletimelist).')
+    p.add_argument('-f', '--filelist', type=argparse.FileType('w'), default=sys.stdout,
+                   help='Filename of the file list without times (default: fullfilelist).')
+
+    opts = p.parse_args()
+
+    if not opts.dir:
+        opts.dir = '.'
+
+    opts.checksum_files = opts.checksum_files or []
+    if opts.checksum:
+        opts.checksum_files += ['repomd.xml']
+
+    opts.skip_files = opts.skip_files or []
+    if opts.skip:
+        opts.skip_files += ['fullfiletimelist']
+
+    return opts
+
+
+def main():
+    opts = parseopts()
+    checksums = {}
+
+    os.chdir(opts.dir)
+
+    print('[Version]', file=opts.timelist)
+    print('2', file=opts.timelist)
+    print(file=opts.timelist)
+    print('[Files]', file=opts.timelist)
+
+    for entry in recursedir():
+        # opts.filelist.write(entry.path + '\n')
+        print(entry.path, file=opts.filelist)
+        if entry.name in opts.skip_files:
+            continue
+        if entry.name in opts.checksum_files:
+            checksums[entry.path[2:]] = True
+        info = entry.stat(follow_symlinks=False)
+        modtime = max(info.st_mtime, info.st_ctime)
+        size = info.st_size
+        ftype = get_ftype(entry)
+        # opts.timelist.write('{0}\t{1}\t{2}\n'.format(modtime, ftype, entry.path[2:]))
+        print('{0}\t{1}\t{2}\t{3}'.format(modtime, ftype, size, entry.path[2:]), file=opts.timelist)
+
+    if not checksums:
+        sys.exit(0)
+
+    print('\n[Checksums SHA1]', file=opts.timelist)
+
+    for f in sorted(checksums):
+        print('{0}\t{1}'.format(sha1(f), f), file=opts.timelist)
+
+
+if __name__ == '__main__':
+        main()

files/scripts/update-fullfiletimelist

@@ -1,22 +1,33 @@
-#!/bin/bash
+#!/bin/sh
-
-# currently runs on secondary01 from cron to update alt
 
 MOD=$1
 [ -z "$MOD" ] && {
-	echo "usage: $0 <module>"
+    echo "usage: $0 <module>"
-	exit 1
+    exit 2
 }
 
-# This is the new list with timestamps
+TOPD=/srv/pub/
+FILELIST=fullfilelist
+TIMELIST=fullfiletimelist-$MOD
+LOCKFILE=.lock.$TIMELIST
+CREATE=/usr/local/bin/create-filelist
 
-TMPFILE=$(mktemp -p /tmp/)
+(
-pushd /srv/pub/$MOD > /dev/null
+    flock -n 9 || exit 1
-/usr/local/bin/create-filelist . > $TMPFILE
+
-if diff $TMPFILE fullfiletimelist > /dev/null; then
+    TMPD=$(mktemp -d -t create-filelist.XXXXXXXXXX)
-	rm -f $TMPFILE
+    trap "rm -rf $TMPD" EXIT
-else
+    cd $TMPD
-	mv $TMPFILE fullfiletimelist
+
-	chmod 0644 fullfiletimelist
+    $CREATE -c -s -d $TOPD/$MOD -f $FILELIST -t $TIMELIST
-fi
+    if diff -q $FILELIST $TOPD/$MOD/$FILELIST > /dev/null; then
-popd > /dev/null
+        mv $FILELIST $TOPD/$MOD/$FILELIST
+        chmod 0644 $TOPD/$MOD/$FILELIST
+    fi
+
+    if diff -q $TIMELIST $TOPD/$MOD/$TIMELIST > /dev/null; then
+        mv $TIMELIST $TOPD/$MOD/$TIMELIST
+        chmod 0644 $TOPD/$MOD/$TIMELIST
+    fi
+
+) 9>$LOCKFILE

playbooks/groups/secondary.yml

@@ -60,7 +60,7 @@
     - python-scandir
 
   - name: add create-filelist script from quick-fedora-mirror
-    copy: src="{{ roles }}/bodhi2/backend/files/create-filelist" dest=/usr/local/bin/create-filelist mode=0755
+    copy: src="{{ files }}/scripts/create-filelist" dest=/usr/local/bin/create-filelist mode=0755
 
   - name: add cron script to update fullfiletimelist
     copy: src="{{ files }}/scripts/update-fullfiletimelist" dest=/usr/local/bin/update-fullfiletimelist mode=0755