From 6db4fd0ca6c97b467508cb4931f33651ba74a9d4 Mon Sep 17 00:00:00 2001 From: Jason Tibbitts Date: Fri, 5 May 2017 15:23:46 +0000 Subject: [PATCH] Update create-filelist to latest upstream. --- files/scripts/create-filelist | 103 +++++++++++++++++++++++++--------- 1 file changed, 76 insertions(+), 27 deletions(-) diff --git a/files/scripts/create-filelist b/files/scripts/create-filelist index 3b831f8c05..d59c2781cd 100755 --- a/files/scripts/create-filelist +++ b/files/scripts/create-filelist @@ -11,8 +11,14 @@ from __future__ import print_function import argparse import hashlib import os +import stat import sys -from scandir import scandir + +# Get scandir from whatever module provides it today +try: + from os import scandir +except ImportError: + from scandir import scandir # productmd is optional, needed only for the imagelist feature try: @@ -21,13 +27,36 @@ except ImportError: SUPPORTED_IMAGE_FORMATS = [] -def get_ftype(entry): - """Return a simple indicator of the file type.""" - if entry.is_symlink(): - return 'l' - if entry.is_dir(): - return 'd' - return 'f' +class SEntry(object): + """A simpler DirEntry-like object.""" + + def __init__(self, direntry, restricted=False): + self.direntry = direntry + self.restricted = restricted + self.path = direntry.path + self.name = direntry.name + + info = direntry.stat(follow_symlinks=False) + self.modtime = max(info.st_mtime, info.st_ctime) + self.readable_group = info.st_mode & stat.S_IRGRP + self.readable_world = info.st_mode & stat.S_IROTH + self.size = info.st_size + + ftype = 'f' + perm = '' + if direntry.is_symlink(): + ftype = 'l' + elif direntry.is_dir(): + ftype = 'd' + + if self.restricted: + perm = '*' + + # Note that we want an unreadable state to override the restricted state + if not self.readable_world: + perm = '-' + + self.ftype = ftype + perm def sha1(fname): @@ -42,22 +71,40 @@ def sha1(fname): return sha1.hexdigest() -def recursedir(path='.', skip=[], alwaysskip=['.~tmp~']): - """Just like scandir, but recursively. +def recursedir(path='.', skip=[], alwaysskip=['.~tmp~'], in_restricted=False): + """Like scandir, but recursively. Will skip everything in the skip array, but only at the top level directory. + + Returns SEntry objects. If in_restricted is true, all returned entries will + be marked as restricted even if their permissions are not restricted. """ - for entry in scandir(path): - if entry.name in skip: + for dentry in scandir(path): + if dentry.name in skip: continue - if entry.name in alwaysskip: + if dentry.name in alwaysskip: continue - if entry.is_dir(follow_symlinks=False): + + # Skip things which are not at least group readable + # Symlinks are followed here so that clients won't see dangling + # symlinks to content they can't transfer. It's the default, but to + # avoid confusion it's been made explicit. + if not (dentry.stat(follow_symlinks=True).st_mode & stat.S_IRGRP): + # print('{} is not group readable; skipping.'.format(dentry.path)) + continue + + se = SEntry(dentry, in_restricted) + if dentry.is_dir(follow_symlinks=False): + this_restricted = in_restricted + if not se.readable_world: + # print('{} is not world readable; marking as restricted.'.format(se.path), file=sys.stderr) + this_restricted = True + # Don't pass skip here, because we only skip in the top level - for rentry in recursedir(entry.path, alwaysskip=alwaysskip): - yield rentry - yield entry + for re in recursedir(se.path, alwaysskip=alwaysskip, in_restricted=this_restricted): + yield re + yield se def parseopts(): @@ -97,11 +144,11 @@ def parseopts(): opts.skip_files = opts.skip_files or [] if opts.skip: if not opts.timelist.name == '': - opts.skip_files += [opts.timelist.name] + opts.skip_files += [os.path.basename(opts.timelist.name)] if not opts.filelist.name == '': - opts.skip_files += [opts.filelist.name] + opts.skip_files += [os.path.basename(opts.filelist.name)] if not opts.imagelist.name == '': - opts.skip_files += [opts.imagelist.name] + opts.skip_files += [os.path.basename(opts.imagelist.name)] return opts @@ -115,25 +162,27 @@ def main(): os.chdir(opts.dir) print('[Version]', file=opts.timelist) + # XXX Technically this should be version 3. But old clients will simply + # ignore the extended file types for restricted directories, and so we can + # add this now and let things simmer for a while before bumping the format + # and hard-breaking old clients. print('2', file=opts.timelist) print(file=opts.timelist) print('[Files]', file=opts.timelist) for entry in recursedir(skip=opts.skip_files): - # opts.filelist.write(entry.path + '\n') print(entry.path, file=opts.filelist) + # write to filtered list if appropriate imgs = ['.{0}'.format(form) for form in SUPPORTED_IMAGE_FORMATS] if any(entry.path.endswith(img) for img in imgs): print(entry.path, file=opts.imagelist) if entry.name in opts.checksum_files: checksums[entry.path[2:]] = True - info = entry.stat(follow_symlinks=False) - modtime = max(info.st_mtime, info.st_ctime) - size = info.st_size - ftype = get_ftype(entry) - # opts.timelist.write('{0}\t{1}\t{2}\n'.format(modtime, ftype, entry.path[2:])) - print('{0}\t{1}\t{2}\t{3}'.format(modtime, ftype, size, entry.path[2:]), file=opts.timelist) + + print('{0}\t{1}\t{2}\t{3}'.format(entry.modtime, entry.ftype, + entry.size, entry.path[2:]), + file=opts.timelist) print('\n[Checksums SHA1]', file=opts.timelist)