Update create-filelist to latest upstream.

files/scripts/create-filelist

@@ -11,8 +11,14 @@ from __future__ import print_function
 import argparse
 import hashlib
 import os
+import stat
 import sys
-from scandir import scandir
+
+# Get scandir from whatever module provides it today
+try:
+    from os import scandir
+except ImportError:
+    from scandir import scandir
 
 # productmd is optional, needed only for the imagelist feature
 try:
@@ -21,13 +27,36 @@ except ImportError:
     SUPPORTED_IMAGE_FORMATS = []
 
 
-def get_ftype(entry):
+class SEntry(object):
-    """Return a simple indicator of the file type."""
+    """A simpler DirEntry-like object."""
-    if entry.is_symlink():
+
-        return 'l'
+    def __init__(self, direntry, restricted=False):
-    if entry.is_dir():
+        self.direntry = direntry
-        return 'd'
+        self.restricted = restricted
-    return 'f'
+        self.path = direntry.path
+        self.name = direntry.name
+
+        info = direntry.stat(follow_symlinks=False)
+        self.modtime = max(info.st_mtime, info.st_ctime)
+        self.readable_group = info.st_mode & stat.S_IRGRP
+        self.readable_world = info.st_mode & stat.S_IROTH
+        self.size = info.st_size
+
+        ftype = 'f'
+        perm = ''
+        if direntry.is_symlink():
+            ftype = 'l'
+        elif direntry.is_dir():
+            ftype = 'd'
+
+        if self.restricted:
+            perm = '*'
+
+        # Note that we want an unreadable state to override the restricted state
+        if not self.readable_world:
+            perm = '-'
+
+        self.ftype = ftype + perm
 
 
 def sha1(fname):
@@ -42,22 +71,40 @@ def sha1(fname):
     return sha1.hexdigest()
 
 
-def recursedir(path='.', skip=[], alwaysskip=['.~tmp~']):
+def recursedir(path='.', skip=[], alwaysskip=['.~tmp~'], in_restricted=False):
-    """Just like scandir, but recursively.
+    """Like scandir, but recursively.
 
     Will skip everything in the skip array, but only at the top level
     directory.
+
+    Returns SEntry objects.  If in_restricted is true, all returned entries will
+    be marked as restricted even if their permissions are not restricted.
     """
-    for entry in scandir(path):
+    for dentry in scandir(path):
-        if entry.name in skip:
+        if dentry.name in skip:
             continue
-        if entry.name in alwaysskip:
+        if dentry.name in alwaysskip:
             continue
-        if entry.is_dir(follow_symlinks=False):
+
+        # Skip things which are not at least group readable
+        # Symlinks are followed here so that clients won't see dangling
+        # symlinks to content they can't transfer.  It's the default, but to
+        # avoid confusion it's been made explicit.
+        if not (dentry.stat(follow_symlinks=True).st_mode & stat.S_IRGRP):
+            # print('{} is not group readable; skipping.'.format(dentry.path))
+            continue
+
+        se = SEntry(dentry, in_restricted)
+        if dentry.is_dir(follow_symlinks=False):
+            this_restricted = in_restricted
+            if not se.readable_world:
+                # print('{} is not world readable; marking as restricted.'.format(se.path), file=sys.stderr)
+                this_restricted = True
+
             # Don't pass skip here, because we only skip in the top level
-            for rentry in recursedir(entry.path, alwaysskip=alwaysskip):
+            for re in recursedir(se.path, alwaysskip=alwaysskip, in_restricted=this_restricted):
-                yield rentry
+                yield re
-        yield entry
+        yield se
 
 
 def parseopts():
@@ -97,11 +144,11 @@ def parseopts():
     opts.skip_files = opts.skip_files or []
     if opts.skip:
         if not opts.timelist.name == '<stdout>':
-            opts.skip_files += [opts.timelist.name]
+            opts.skip_files += [os.path.basename(opts.timelist.name)]
         if not opts.filelist.name == '<stdout>':
-            opts.skip_files += [opts.filelist.name]
+            opts.skip_files += [os.path.basename(opts.filelist.name)]
         if not opts.imagelist.name == '<stdout>':
-            opts.skip_files += [opts.imagelist.name]
+            opts.skip_files += [os.path.basename(opts.imagelist.name)]
 
     return opts
 
@@ -115,25 +162,27 @@ def main():
     os.chdir(opts.dir)
 
     print('[Version]', file=opts.timelist)
+    # XXX Technically this should be version 3.  But old clients will simply
+    # ignore the extended file types for restricted directories, and so we can
+    # add this now and let things simmer for a while before bumping the format
+    # and hard-breaking old clients.
     print('2', file=opts.timelist)
     print(file=opts.timelist)
     print('[Files]', file=opts.timelist)
 
     for entry in recursedir(skip=opts.skip_files):
-        # opts.filelist.write(entry.path + '\n')
         print(entry.path, file=opts.filelist)
+
         # write to filtered list if appropriate
         imgs = ['.{0}'.format(form) for form in SUPPORTED_IMAGE_FORMATS]
         if any(entry.path.endswith(img) for img in imgs):
             print(entry.path, file=opts.imagelist)
         if entry.name in opts.checksum_files:
             checksums[entry.path[2:]] = True
-        info = entry.stat(follow_symlinks=False)
+
-        modtime = max(info.st_mtime, info.st_ctime)
+        print('{0}\t{1}\t{2}\t{3}'.format(entry.modtime, entry.ftype,
-        size = info.st_size
+                                          entry.size, entry.path[2:]),
-        ftype = get_ftype(entry)
+              file=opts.timelist)
-        # opts.timelist.write('{0}\t{1}\t{2}\n'.format(modtime, ftype, entry.path[2:]))
-        print('{0}\t{1}\t{2}\t{3}'.format(modtime, ftype, size, entry.path[2:]), file=opts.timelist)
 
     print('\n[Checksums SHA1]', file=opts.timelist)