android/git-repo
1
Fork 0
mirror of https://gerrit.googlesource.com/git-repo synced 2025-01-02 16:14:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add support for long paths

Browse Source 
* Add more file i/o wrappers in platform_utils to allow using
  long paths (length > MAX_PATH) on Windows.

* Paths using the long path syntax ("\\?\" prefix) should never
  escape the platform_utils API surface area, so that this
  specific syntax is not visible to the rest of the repo code base.

* Forward many calls from os.xxx to platform_utils.xxx in various place
  to ensure long paths support, specifically when repo decides to delete
  obsolete directories.

* There are more places that need to be converted to support long paths,
  this commit is an initial effort to unblock a few common use cases.

* Also, fix remove function to handle directory symlinks

Change-Id: If82ccc408e516e96ff7260be25f8fd2fe3f9571a
This commit is contained in:
Renaud Paquay 2018-09-27 10:46:58 -07:00
parent b3133a3164
commit bed8b62345
7 changed files with 139 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
git_config.py
View File

@ -503,7 +503,7 @@ def close_ssh():
d = ssh_sock(create=False)
if d:
try:
os.rmdir(os.path.dirname(d))
platform_utils.rmdir(os.path.dirname(d))
except OSError:
pass

5
git_refs.py
View File

@ -15,6 +15,7 @@
import os
from trace import Trace
import platform_utils
HEAD = 'HEAD'
R_CHANGES = 'refs/changes/'
@ -127,9 +128,9 @@ class GitRefs(object):
def _ReadLoose(self, prefix):
base = os.path.join(self._gitdir, prefix)
for name in os.listdir(base):
for name in platform_utils.listdir(base):
p = os.path.join(base, name)
if os.path.isdir(p):
if platform_utils.isdir(p):
self._mtime[prefix] = os.path.getmtime(base)
self._ReadLoose(prefix + name + '/')
elif name.endswith('.lock'):

2
manifest_xml.py
View File

@ -446,7 +446,7 @@ class XmlManifest(object):
local_dir = os.path.abspath(os.path.join(self.repodir, LOCAL_MANIFESTS_DIR_NAME))
try:
for local_file in sorted(os.listdir(local_dir)):
for local_file in sorted(platform_utils.listdir(local_dir)):
if local_file.endswith('.xml'):
local = os.path.join(local_dir, local_file)
nodes.append(self._ParseManifestXml(local, self.repodir))

128
platform_utils.py
View File

@ -187,10 +187,10 @@ def symlink(source, link_name):
source = _validate_winpath(source)
link_name = _validate_winpath(link_name)
target = os.path.join(os.path.dirname(link_name), source)
if os.path.isdir(target):
platform_utils_win32.create_dirsymlink(source, link_name)
if isdir(target):
platform_utils_win32.create_dirsymlink(_makelongpath(source), link_name)
else:
platform_utils_win32.create_filesymlink(source, link_name)
platform_utils_win32.create_filesymlink(_makelongpath(source), link_name)
else:
return os.symlink(source, link_name)
@ -220,9 +220,32 @@ def _winpath_is_valid(path):
return not drive # "x:" is invalid
def rmtree(path):
def _makelongpath(path):
"""Return the input path normalized to support the Windows long path syntax
("\\\\?\\" prefix) if needed, i.e. if the input path is longer than the
MAX_PATH limit.
"""
if isWindows():
shutil.rmtree(path, onerror=handle_rmtree_error)
# Note: MAX_PATH is 260, but, for directories, the maximum value is actually 246.
if len(path) < 246:
return path
if path.startswith(u"\\\\?\\"):
return path
if not os.path.isabs(path):
return path
# Append prefix and ensure unicode so that the special longpath syntax
# is supported by underlying Win32 API calls
return u"\\\\?\\" + os.path.normpath(path)
else:
return path
def rmtree(path):
"""shutil.rmtree(path) wrapper with support for long paths on Windows.
Availability: Unix, Windows."""
if isWindows():
shutil.rmtree(_makelongpath(path), onerror=handle_rmtree_error)
else:
shutil.rmtree(path)
@ -234,15 +257,18 @@ def handle_rmtree_error(function, path, excinfo):
def rename(src, dst):
"""os.rename(src, dst) wrapper with support for long paths on Windows.
Availability: Unix, Windows."""
if isWindows():
# On Windows, rename fails if destination exists, see
# https://docs.python.org/2/library/os.html#os.rename
try:
os.rename(src, dst)
os.rename(_makelongpath(src), _makelongpath(dst))
except OSError as e:
if e.errno == errno.EEXIST:
os.remove(dst)
os.rename(src, dst)
os.remove(_makelongpath(dst))
os.rename(_makelongpath(src), _makelongpath(dst))
else:
raise
else:
@ -250,30 +276,98 @@ def rename(src, dst):
def remove(path):
"""Remove (delete) the file path. This is a replacement for os.remove, but
allows deleting read-only files on Windows.
"""
"""Remove (delete) the file path. This is a replacement for os.remove that
allows deleting read-only files on Windows, with support for long paths and
for deleting directory symbolic links.
Availability: Unix, Windows."""
if isWindows():
longpath = _makelongpath(path)
try:
os.remove(path)
os.remove(longpath)
except OSError as e:
if e.errno == errno.EACCES:
os.chmod(path, stat.S_IWRITE)
os.remove(path)
os.chmod(longpath, stat.S_IWRITE)
# Directory symbolic links must be deleted with 'rmdir'.
if islink(longpath) and isdir(longpath):
os.rmdir(longpath)
else:
os.remove(longpath)
else:
raise
else:
os.remove(path)
def walk(top, topdown=True, onerror=None, followlinks=False):
"""os.walk(path) wrapper with support for long paths on Windows.
Availability: Windows, Unix.
"""
if isWindows():
return _walk_windows_impl(top, topdown, onerror, followlinks)
else:
return os.walk(top, topdown, onerror, followlinks)
def _walk_windows_impl(top, topdown, onerror, followlinks):
try:
names = listdir(top)
except error, err:
if onerror is not None:
onerror(err)
return
dirs, nondirs = [], []
for name in names:
if isdir(os.path.join(top, name)):
dirs.append(name)
else:
nondirs.append(name)
if topdown:
yield top, dirs, nondirs
for name in dirs:
new_path = os.path.join(top, name)
if followlinks or not islink(new_path):
for x in _walk_windows_impl(new_path, topdown, onerror, followlinks):
yield x
if not topdown:
yield top, dirs, nondirs
def listdir(path):
"""os.listdir(path) wrapper with support for long paths on Windows.
Availability: Windows, Unix.
"""
return os.listdir(_makelongpath(path))
def rmdir(path):
"""os.rmdir(path) wrapper with support for long paths on Windows.
Availability: Windows, Unix.
"""
os.rmdir(_makelongpath(path))
def isdir(path):
"""os.path.isdir(path) wrapper with support for long paths on Windows.
Availability: Windows, Unix.
"""
return os.path.isdir(_makelongpath(path))
def islink(path):
"""Test whether a path is a symbolic link.
"""os.path.islink(path) wrapper with support for long paths on Windows.
Availability: Windows, Unix.
"""
if isWindows():
import platform_utils_win32
return platform_utils_win32.islink(path)
return platform_utils_win32.islink(_makelongpath(path))
else:
return os.path.islink(path)
@ -288,7 +382,7 @@ def readlink(path):
"""
if isWindows():
import platform_utils_win32
return platform_utils_win32.readlink(path)
return platform_utils_win32.readlink(_makelongpath(path))
else:
return os.readlink(path)

16
project.py
View File

@ -103,7 +103,7 @@ def _ProjectHooks():
if _project_hook_list is None:
d = platform_utils.realpath(os.path.abspath(os.path.dirname(__file__)))
d = os.path.join(d, 'hooks')
_project_hook_list = [os.path.join(d, x) for x in os.listdir(d)]
_project_hook_list = [os.path.join(d, x) for x in platform_utils.listdir(d)]
return _project_hook_list
@ -253,7 +253,7 @@ class _CopyFile(object):
platform_utils.remove(dest)
else:
dest_dir = os.path.dirname(dest)
if not os.path.isdir(dest_dir):
if not platform_utils.isdir(dest_dir):
os.makedirs(dest_dir)
shutil.copy(src, dest)
# make the file read-only
@ -282,7 +282,7 @@ class _LinkFile(object):
platform_utils.remove(absDest)
else:
dest_dir = os.path.dirname(absDest)
if not os.path.isdir(dest_dir):
if not platform_utils.isdir(dest_dir):
os.makedirs(dest_dir)
platform_utils.symlink(relSrc, absDest)
except IOError:
@ -302,7 +302,7 @@ class _LinkFile(object):
else:
# Entity doesn't exist assume there is a wild card
absDestDir = self.abs_dest
if os.path.exists(absDestDir) and not os.path.isdir(absDestDir):
if os.path.exists(absDestDir) and not platform_utils.isdir(absDestDir):
_error('Link error: src with wildcard, %s must be a directory',
absDestDir)
else:
@ -750,7 +750,7 @@ class Project(object):
@property
def Exists(self):
return os.path.isdir(self.gitdir) and os.path.isdir(self.objdir)
return platform_utils.isdir(self.gitdir) and platform_utils.isdir(self.objdir)
@property
def CurrentBranch(self):
@ -931,7 +931,7 @@ class Project(object):
quiet: If True then only print the project name. Do not print
the modified files, branch name, etc.
"""
if not os.path.isdir(self.worktree):
if not platform_utils.isdir(self.worktree):
if output_redir is None:
output_redir = sys.stdout
print(file=output_redir)
@ -2510,7 +2510,7 @@ class Project(object):
to_copy = []
if copy_all:
to_copy = os.listdir(gitdir)
to_copy = platform_utils.listdir(gitdir)
dotgit = platform_utils.realpath(dotgit)
for name in set(to_copy).union(to_symlink):
@ -2529,7 +2529,7 @@ class Project(object):
platform_utils.symlink(
os.path.relpath(src, os.path.dirname(dst)), dst)
elif copy_all and not platform_utils.islink(dst):
if os.path.isdir(src):
if platform_utils.isdir(src):
shutil.copytree(src, dst)
elif os.path.isfile(src):
shutil.copy(src, dst)

3
subcmds/status.py
View File

@ -26,6 +26,7 @@ import itertools
import os
from color import Coloring
import platform_utils
class Status(PagedCommand):
common = True
@ -115,7 +116,7 @@ the following meanings:
"""find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'"""
status_header = ' --\t'
for item in dirs:
if not os.path.isdir(item):
if not platform_utils.isdir(item):
outstring.append(''.join([status_header, item]))
continue
if item in proj_dirs:

26
subcmds/sync.py
View File

@ -474,8 +474,8 @@ later is required to fix a server side protocol bug.
# so rmtree works.
try:
platform_utils.rmtree(os.path.join(path, '.git'))
except OSError:
print('Failed to remove %s' % os.path.join(path, '.git'), file=sys.stderr)
except OSError as e:
print('Failed to remove %s (%s)' % (os.path.join(path, '.git'), str(e)), file=sys.stderr)
print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)
print(' remove manually, then run sync again', file=sys.stderr)
return -1
@ -484,12 +484,12 @@ later is required to fix a server side protocol bug.
# another git project
dirs_to_remove = []
failed = False
for root, dirs, files in os.walk(path):
for root, dirs, files in platform_utils.walk(path):
for f in files:
try:
platform_utils.remove(os.path.join(root, f))
except OSError:
print('Failed to remove %s' % os.path.join(root, f), file=sys.stderr)
except OSError as e:
print('Failed to remove %s (%s)' % (os.path.join(root, f), str(e)), file=sys.stderr)
failed = True
dirs[:] = [d for d in dirs
if not os.path.lexists(os.path.join(root, d, '.git'))]
@ -499,14 +499,14 @@ later is required to fix a server side protocol bug.
if platform_utils.islink(d):
try:
platform_utils.remove(d)
except OSError:
print('Failed to remove %s' % os.path.join(root, d), file=sys.stderr)
except OSError as e:
print('Failed to remove %s (%s)' % (os.path.join(root, d), str(e)), file=sys.stderr)
failed = True
elif len(os.listdir(d)) == 0:
elif len(platform_utils.listdir(d)) == 0:
try:
os.rmdir(d)
except OSError:
print('Failed to remove %s' % os.path.join(root, d), file=sys.stderr)
platform_utils.rmdir(d)
except OSError as e:
print('Failed to remove %s (%s)' % (os.path.join(root, d), str(e)), file=sys.stderr)
failed = True
continue
if failed:
@ -517,8 +517,8 @@ later is required to fix a server side protocol bug.
# Try deleting parent dirs if they are empty
project_dir = path
while project_dir != self.manifest.topdir:
if len(os.listdir(project_dir)) == 0:
os.rmdir(project_dir)
if len(platform_utils.listdir(project_dir)) == 0:
platform_utils.rmdir(project_dir)
else:
break
project_dir = os.path.dirname(project_dir)