Add support for long paths

* Add more file i/o wrappers in platform_utils to allow using
  long paths (length > MAX_PATH) on Windows.

* Paths using the long path syntax ("\\?\" prefix) should never
  escape the platform_utils API surface area, so that this
  specific syntax is not visible to the rest of the repo code base.

* Forward many calls from os.xxx to platform_utils.xxx in various place
  to ensure long paths support, specifically when repo decides to delete
  obsolete directories.

* There are more places that need to be converted to support long paths,
  this commit is an initial effort to unblock a few common use cases.

* Also, fix remove function to handle directory symlinks

Change-Id: If82ccc408e516e96ff7260be25f8fd2fe3f9571a

git_config.py

@@ -503,7 +503,7 @@ def close_ssh():
   d = ssh_sock(create=False)
   if d:
     try:
-      os.rmdir(os.path.dirname(d))
+      platform_utils.rmdir(os.path.dirname(d))
     except OSError:
       pass
 

git_refs.py

@@ -15,6 +15,7 @@
 
 import os
 from trace import Trace
+import platform_utils
 
 HEAD      = 'HEAD'
 R_CHANGES = 'refs/changes/'
@@ -127,9 +128,9 @@ class GitRefs(object):
 
   def _ReadLoose(self, prefix):
     base = os.path.join(self._gitdir, prefix)
-    for name in os.listdir(base):
+    for name in platform_utils.listdir(base):
       p = os.path.join(base, name)
-      if os.path.isdir(p):
+      if platform_utils.isdir(p):
         self._mtime[prefix] = os.path.getmtime(base)
         self._ReadLoose(prefix + name + '/')
       elif name.endswith('.lock'):

manifest_xml.py

@@ -446,7 +446,7 @@ class XmlManifest(object):
 
       local_dir = os.path.abspath(os.path.join(self.repodir, LOCAL_MANIFESTS_DIR_NAME))
       try:
-        for local_file in sorted(os.listdir(local_dir)):
+        for local_file in sorted(platform_utils.listdir(local_dir)):
           if local_file.endswith('.xml'):
             local = os.path.join(local_dir, local_file)
             nodes.append(self._ParseManifestXml(local, self.repodir))

platform_utils.py

@@ -187,10 +187,10 @@ def symlink(source, link_name):
     source = _validate_winpath(source)
     link_name = _validate_winpath(link_name)
     target = os.path.join(os.path.dirname(link_name), source)
-    if os.path.isdir(target):
+    if isdir(target):
-      platform_utils_win32.create_dirsymlink(source, link_name)
+      platform_utils_win32.create_dirsymlink(_makelongpath(source), link_name)
     else:
-      platform_utils_win32.create_filesymlink(source, link_name)
+      platform_utils_win32.create_filesymlink(_makelongpath(source), link_name)
   else:
     return os.symlink(source, link_name)
 
@@ -220,9 +220,32 @@ def _winpath_is_valid(path):
     return not drive  # "x:" is invalid
 
 
-def rmtree(path):
+def _makelongpath(path):
+  """Return the input path normalized to support the Windows long path syntax
+  ("\\\\?\\" prefix) if needed, i.e. if the input path is longer than the
+  MAX_PATH limit.
+  """
   if isWindows():
-    shutil.rmtree(path, onerror=handle_rmtree_error)
+    # Note: MAX_PATH is 260, but, for directories, the maximum value is actually 246.
+    if len(path) < 246:
+      return path
+    if path.startswith(u"\\\\?\\"):
+      return path
+    if not os.path.isabs(path):
+      return path
+    # Append prefix and ensure unicode so that the special longpath syntax
+    # is supported by underlying Win32 API calls
+    return u"\\\\?\\" + os.path.normpath(path)
+  else:
+    return path
+
+
+def rmtree(path):
+  """shutil.rmtree(path) wrapper with support for long paths on Windows.
+
+  Availability: Unix, Windows."""
+  if isWindows():
+    shutil.rmtree(_makelongpath(path), onerror=handle_rmtree_error)
   else:
     shutil.rmtree(path)
 
@@ -234,15 +257,18 @@ def handle_rmtree_error(function, path, excinfo):
 
 
 def rename(src, dst):
+  """os.rename(src, dst) wrapper with support for long paths on Windows.
+
+  Availability: Unix, Windows."""
   if isWindows():
     # On Windows, rename fails if destination exists, see
     # https://docs.python.org/2/library/os.html#os.rename
     try:
-      os.rename(src, dst)
+      os.rename(_makelongpath(src), _makelongpath(dst))
     except OSError as e:
       if e.errno == errno.EEXIST:
-        os.remove(dst)
+        os.remove(_makelongpath(dst))
-        os.rename(src, dst)
+        os.rename(_makelongpath(src), _makelongpath(dst))
       else:
         raise
   else:
@@ -250,30 +276,98 @@ def rename(src, dst):
 
 
 def remove(path):
-  """Remove (delete) the file path. This is a replacement for os.remove, but
+  """Remove (delete) the file path. This is a replacement for os.remove that
-  allows deleting read-only files on Windows.
+  allows deleting read-only files on Windows, with support for long paths and
-  """
+  for deleting directory symbolic links.
+
+  Availability: Unix, Windows."""
   if isWindows():
+    longpath = _makelongpath(path)
     try:
-      os.remove(path)
+      os.remove(longpath)
     except OSError as e:
       if e.errno == errno.EACCES:
-        os.chmod(path, stat.S_IWRITE)
+        os.chmod(longpath, stat.S_IWRITE)
-        os.remove(path)
+        # Directory symbolic links must be deleted with 'rmdir'.
+        if islink(longpath) and isdir(longpath):
+          os.rmdir(longpath)
+        else:
+          os.remove(longpath)
       else:
         raise
   else:
     os.remove(path)
 
 
+def walk(top, topdown=True, onerror=None, followlinks=False):
+  """os.walk(path) wrapper with support for long paths on Windows.
+
+  Availability: Windows, Unix.
+  """
+  if isWindows():
+    return _walk_windows_impl(top, topdown, onerror, followlinks)
+  else:
+    return os.walk(top, topdown, onerror, followlinks)
+
+
+def _walk_windows_impl(top, topdown, onerror, followlinks):
+  try:
+    names = listdir(top)
+  except error, err:
+    if onerror is not None:
+      onerror(err)
+    return
+
+  dirs, nondirs = [], []
+  for name in names:
+    if isdir(os.path.join(top, name)):
+      dirs.append(name)
+    else:
+      nondirs.append(name)
+
+  if topdown:
+    yield top, dirs, nondirs
+  for name in dirs:
+    new_path = os.path.join(top, name)
+    if followlinks or not islink(new_path):
+      for x in _walk_windows_impl(new_path, topdown, onerror, followlinks):
+        yield x
+  if not topdown:
+    yield top, dirs, nondirs
+
+
+def listdir(path):
+  """os.listdir(path) wrapper with support for long paths on Windows.
+
+  Availability: Windows, Unix.
+  """
+  return os.listdir(_makelongpath(path))
+
+
+def rmdir(path):
+  """os.rmdir(path) wrapper with support for long paths on Windows.
+
+  Availability: Windows, Unix.
+  """
+  os.rmdir(_makelongpath(path))
+
+
+def isdir(path):
+  """os.path.isdir(path) wrapper with support for long paths on Windows.
+
+  Availability: Windows, Unix.
+  """
+  return os.path.isdir(_makelongpath(path))
+
+
 def islink(path):
-  """Test whether a path is a symbolic link.
+  """os.path.islink(path) wrapper with support for long paths on Windows.
 
   Availability: Windows, Unix.
   """
   if isWindows():
     import platform_utils_win32
-    return platform_utils_win32.islink(path)
+    return platform_utils_win32.islink(_makelongpath(path))
   else:
     return os.path.islink(path)
 
@@ -288,7 +382,7 @@ def readlink(path):
   """
   if isWindows():
     import platform_utils_win32
-    return platform_utils_win32.readlink(path)
+    return platform_utils_win32.readlink(_makelongpath(path))
   else:
     return os.readlink(path)
 

project.py

@@ -103,7 +103,7 @@ def _ProjectHooks():
   if _project_hook_list is None:
     d = platform_utils.realpath(os.path.abspath(os.path.dirname(__file__)))
     d = os.path.join(d, 'hooks')
-    _project_hook_list = [os.path.join(d, x) for x in os.listdir(d)]
+    _project_hook_list = [os.path.join(d, x) for x in platform_utils.listdir(d)]
   return _project_hook_list
 
 
@@ -253,7 +253,7 @@ class _CopyFile(object):
           platform_utils.remove(dest)
         else:
           dest_dir = os.path.dirname(dest)
-          if not os.path.isdir(dest_dir):
+          if not platform_utils.isdir(dest_dir):
             os.makedirs(dest_dir)
         shutil.copy(src, dest)
         # make the file read-only
@@ -282,7 +282,7 @@ class _LinkFile(object):
           platform_utils.remove(absDest)
         else:
           dest_dir = os.path.dirname(absDest)
-          if not os.path.isdir(dest_dir):
+          if not platform_utils.isdir(dest_dir):
             os.makedirs(dest_dir)
         platform_utils.symlink(relSrc, absDest)
       except IOError:
@@ -302,7 +302,7 @@ class _LinkFile(object):
     else:
       # Entity doesn't exist assume there is a wild card
       absDestDir = self.abs_dest
-      if os.path.exists(absDestDir) and not os.path.isdir(absDestDir):
+      if os.path.exists(absDestDir) and not platform_utils.isdir(absDestDir):
         _error('Link error: src with wildcard, %s must be a directory',
                absDestDir)
       else:
@@ -750,7 +750,7 @@ class Project(object):
 
   @property
   def Exists(self):
-    return os.path.isdir(self.gitdir) and os.path.isdir(self.objdir)
+    return platform_utils.isdir(self.gitdir) and platform_utils.isdir(self.objdir)
 
   @property
   def CurrentBranch(self):
@@ -931,7 +931,7 @@ class Project(object):
       quiet:  If True then only print the project name.  Do not print
               the modified files, branch name, etc.
     """
-    if not os.path.isdir(self.worktree):
+    if not platform_utils.isdir(self.worktree):
       if output_redir is None:
         output_redir = sys.stdout
       print(file=output_redir)
@@ -2510,7 +2510,7 @@ class Project(object):
 
     to_copy = []
     if copy_all:
-      to_copy = os.listdir(gitdir)
+      to_copy = platform_utils.listdir(gitdir)
 
     dotgit = platform_utils.realpath(dotgit)
     for name in set(to_copy).union(to_symlink):
@@ -2529,7 +2529,7 @@ class Project(object):
           platform_utils.symlink(
               os.path.relpath(src, os.path.dirname(dst)), dst)
         elif copy_all and not platform_utils.islink(dst):
-          if os.path.isdir(src):
+          if platform_utils.isdir(src):
             shutil.copytree(src, dst)
           elif os.path.isfile(src):
             shutil.copy(src, dst)

subcmds/status.py

@@ -26,6 +26,7 @@ import itertools
 import os
 
 from color import Coloring
+import platform_utils
 
 class Status(PagedCommand):
   common = True
@@ -115,7 +116,7 @@ the following meanings:
     """find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'"""
     status_header = ' --\t'
     for item in dirs:
-      if not os.path.isdir(item):
+      if not platform_utils.isdir(item):
         outstring.append(''.join([status_header, item]))
         continue
       if item in proj_dirs:

subcmds/sync.py

@@ -474,8 +474,8 @@ later is required to fix a server side protocol bug.
     # so rmtree works.
     try:
       platform_utils.rmtree(os.path.join(path, '.git'))
-    except OSError:
+    except OSError as e:
-      print('Failed to remove %s' % os.path.join(path, '.git'), file=sys.stderr)
+      print('Failed to remove %s (%s)' % (os.path.join(path, '.git'), str(e)), file=sys.stderr)
       print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)
       print('       remove manually, then run sync again', file=sys.stderr)
       return -1
@@ -484,12 +484,12 @@ later is required to fix a server side protocol bug.
     # another git project
     dirs_to_remove = []
     failed = False
-    for root, dirs, files in os.walk(path):
+    for root, dirs, files in platform_utils.walk(path):
       for f in files:
         try:
           platform_utils.remove(os.path.join(root, f))
-        except OSError:
+        except OSError as e:
-          print('Failed to remove %s' % os.path.join(root, f), file=sys.stderr)
+          print('Failed to remove %s (%s)' % (os.path.join(root, f), str(e)), file=sys.stderr)
           failed = True
       dirs[:] = [d for d in dirs
                  if not os.path.lexists(os.path.join(root, d, '.git'))]
@@ -499,14 +499,14 @@ later is required to fix a server side protocol bug.
       if platform_utils.islink(d):
         try:
           platform_utils.remove(d)
-        except OSError:
+        except OSError as e:
-          print('Failed to remove %s' % os.path.join(root, d), file=sys.stderr)
+          print('Failed to remove %s (%s)' % (os.path.join(root, d), str(e)), file=sys.stderr)
           failed = True
-      elif len(os.listdir(d)) == 0:
+      elif len(platform_utils.listdir(d)) == 0:
         try:
-          os.rmdir(d)
+          platform_utils.rmdir(d)
-        except OSError:
+        except OSError as e:
-          print('Failed to remove %s' % os.path.join(root, d), file=sys.stderr)
+          print('Failed to remove %s (%s)' % (os.path.join(root, d), str(e)), file=sys.stderr)
           failed = True
           continue
     if failed:
@@ -517,8 +517,8 @@ later is required to fix a server side protocol bug.
     # Try deleting parent dirs if they are empty
     project_dir = path
     while project_dir != self.manifest.topdir:
-      if len(os.listdir(project_dir)) == 0:
+      if len(platform_utils.listdir(project_dir)) == 0:
-        os.rmdir(project_dir)
+        platform_utils.rmdir(project_dir)
       else:
         break
       project_dir = os.path.dirname(project_dir)