2019-06-13 06:30:51 +00:00
|
|
|
# -*- coding:utf-8 -*-
|
2016-11-01 18:23:38 +00:00
|
|
|
#
|
|
|
|
# Copyright (C) 2016 The Android Open Source Project
|
|
|
|
#
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
|
2016-11-01 18:34:55 +00:00
|
|
|
import errno
|
2016-11-01 18:23:38 +00:00
|
|
|
import os
|
|
|
|
import platform
|
|
|
|
import select
|
2016-11-03 17:37:53 +00:00
|
|
|
import shutil
|
|
|
|
import stat
|
2016-11-01 18:23:38 +00:00
|
|
|
|
2018-06-23 07:02:26 +00:00
|
|
|
from pyversion import is_python3
|
|
|
|
if is_python3():
|
|
|
|
from queue import Queue
|
|
|
|
else:
|
|
|
|
from Queue import Queue
|
|
|
|
|
2016-11-01 18:23:38 +00:00
|
|
|
from threading import Thread
|
|
|
|
|
|
|
|
|
|
|
|
def isWindows():
|
|
|
|
""" Returns True when running with the native port of Python for Windows,
|
|
|
|
False when running on any other platform (including the Cygwin port of
|
|
|
|
Python).
|
|
|
|
"""
|
|
|
|
# Note: The cygwin port of Python returns "CYGWIN_NT_xxx"
|
|
|
|
return platform.system() == "Windows"
|
|
|
|
|
|
|
|
|
|
|
|
class FileDescriptorStreams(object):
|
|
|
|
""" Platform agnostic abstraction enabling non-blocking I/O over a
|
|
|
|
collection of file descriptors. This abstraction is required because
|
|
|
|
fctnl(os.O_NONBLOCK) is not supported on Windows.
|
|
|
|
"""
|
|
|
|
@classmethod
|
|
|
|
def create(cls):
|
|
|
|
""" Factory method: instantiates the concrete class according to the
|
|
|
|
current platform.
|
|
|
|
"""
|
|
|
|
if isWindows():
|
|
|
|
return _FileDescriptorStreamsThreads()
|
|
|
|
else:
|
|
|
|
return _FileDescriptorStreamsNonBlocking()
|
|
|
|
|
|
|
|
def __init__(self):
|
|
|
|
self.streams = []
|
|
|
|
|
|
|
|
def add(self, fd, dest, std_name):
|
|
|
|
""" Wraps an existing file descriptor as a stream.
|
|
|
|
"""
|
|
|
|
self.streams.append(self._create_stream(fd, dest, std_name))
|
|
|
|
|
|
|
|
def remove(self, stream):
|
|
|
|
""" Removes a stream, when done with it.
|
|
|
|
"""
|
|
|
|
self.streams.remove(stream)
|
|
|
|
|
|
|
|
@property
|
|
|
|
def is_done(self):
|
|
|
|
""" Returns True when all streams have been processed.
|
|
|
|
"""
|
|
|
|
return len(self.streams) == 0
|
|
|
|
|
|
|
|
def select(self):
|
|
|
|
""" Returns the set of streams that have data available to read.
|
|
|
|
The returned streams each expose a read() and a close() method.
|
|
|
|
When done with a stream, call the remove(stream) method.
|
|
|
|
"""
|
|
|
|
raise NotImplementedError
|
|
|
|
|
2020-01-25 12:32:37 +00:00
|
|
|
def _create_stream(self, fd, dest, std_name):
|
2016-11-01 18:23:38 +00:00
|
|
|
""" Creates a new stream wrapping an existing file descriptor.
|
|
|
|
"""
|
|
|
|
raise NotImplementedError
|
|
|
|
|
|
|
|
|
|
|
|
class _FileDescriptorStreamsNonBlocking(FileDescriptorStreams):
|
|
|
|
""" Implementation of FileDescriptorStreams for platforms that support
|
|
|
|
non blocking I/O.
|
|
|
|
"""
|
2019-12-18 01:51:29 +00:00
|
|
|
def __init__(self):
|
|
|
|
super(_FileDescriptorStreamsNonBlocking, self).__init__()
|
|
|
|
self._poll = select.poll()
|
|
|
|
self._fd_to_stream = {}
|
|
|
|
|
2016-11-01 18:23:38 +00:00
|
|
|
class Stream(object):
|
|
|
|
""" Encapsulates a file descriptor """
|
|
|
|
def __init__(self, fd, dest, std_name):
|
|
|
|
self.fd = fd
|
|
|
|
self.dest = dest
|
|
|
|
self.std_name = std_name
|
|
|
|
self.set_non_blocking()
|
|
|
|
|
|
|
|
def set_non_blocking(self):
|
|
|
|
import fcntl
|
|
|
|
flags = fcntl.fcntl(self.fd, fcntl.F_GETFL)
|
|
|
|
fcntl.fcntl(self.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
|
|
|
|
|
|
|
|
def fileno(self):
|
|
|
|
return self.fd.fileno()
|
|
|
|
|
|
|
|
def read(self):
|
|
|
|
return self.fd.read(4096)
|
|
|
|
|
|
|
|
def close(self):
|
|
|
|
self.fd.close()
|
|
|
|
|
|
|
|
def _create_stream(self, fd, dest, std_name):
|
2019-12-18 01:51:29 +00:00
|
|
|
stream = self.Stream(fd, dest, std_name)
|
|
|
|
self._fd_to_stream[stream.fileno()] = stream
|
|
|
|
self._poll.register(stream, select.POLLIN)
|
|
|
|
return stream
|
|
|
|
|
|
|
|
def remove(self, stream):
|
|
|
|
self._poll.unregister(stream)
|
|
|
|
del self._fd_to_stream[stream.fileno()]
|
|
|
|
super(_FileDescriptorStreamsNonBlocking, self).remove(stream)
|
2016-11-01 18:23:38 +00:00
|
|
|
|
|
|
|
def select(self):
|
2019-12-18 01:51:29 +00:00
|
|
|
return [self._fd_to_stream[fd] for fd, _ in self._poll.poll()]
|
2016-11-01 18:23:38 +00:00
|
|
|
|
|
|
|
|
|
|
|
class _FileDescriptorStreamsThreads(FileDescriptorStreams):
|
|
|
|
""" Implementation of FileDescriptorStreams for platforms that don't support
|
|
|
|
non blocking I/O. This implementation requires creating threads issuing
|
|
|
|
blocking read operations on file descriptors.
|
|
|
|
"""
|
|
|
|
def __init__(self):
|
|
|
|
super(_FileDescriptorStreamsThreads, self).__init__()
|
|
|
|
# The queue is shared accross all threads so we can simulate the
|
|
|
|
# behavior of the select() function
|
|
|
|
self.queue = Queue(10) # Limit incoming data from streams
|
|
|
|
|
|
|
|
def _create_stream(self, fd, dest, std_name):
|
|
|
|
return self.Stream(fd, dest, std_name, self.queue)
|
|
|
|
|
|
|
|
def select(self):
|
|
|
|
# Return only one stream at a time, as it is the most straighforward
|
|
|
|
# thing to do and it is compatible with the select() function.
|
|
|
|
item = self.queue.get()
|
|
|
|
stream = item.stream
|
|
|
|
stream.data = item.data
|
|
|
|
return [stream]
|
|
|
|
|
|
|
|
class QueueItem(object):
|
|
|
|
""" Item put in the shared queue """
|
|
|
|
def __init__(self, stream, data):
|
|
|
|
self.stream = stream
|
|
|
|
self.data = data
|
|
|
|
|
|
|
|
class Stream(object):
|
|
|
|
""" Encapsulates a file descriptor """
|
|
|
|
def __init__(self, fd, dest, std_name, queue):
|
|
|
|
self.fd = fd
|
|
|
|
self.dest = dest
|
|
|
|
self.std_name = std_name
|
|
|
|
self.queue = queue
|
|
|
|
self.data = None
|
|
|
|
self.thread = Thread(target=self.read_to_queue)
|
|
|
|
self.thread.daemon = True
|
|
|
|
self.thread.start()
|
|
|
|
|
|
|
|
def close(self):
|
|
|
|
self.fd.close()
|
|
|
|
|
|
|
|
def read(self):
|
|
|
|
data = self.data
|
|
|
|
self.data = None
|
|
|
|
return data
|
|
|
|
|
|
|
|
def read_to_queue(self):
|
|
|
|
""" The thread function: reads everything from the file descriptor into
|
|
|
|
the shared queue and terminates when reaching EOF.
|
|
|
|
"""
|
|
|
|
for line in iter(self.fd.readline, b''):
|
|
|
|
self.queue.put(_FileDescriptorStreamsThreads.QueueItem(self, line))
|
|
|
|
self.fd.close()
|
|
|
|
self.queue.put(_FileDescriptorStreamsThreads.QueueItem(self, None))
|
2016-11-01 18:24:03 +00:00
|
|
|
|
|
|
|
|
|
|
|
def symlink(source, link_name):
|
|
|
|
"""Creates a symbolic link pointing to source named link_name.
|
|
|
|
Note: On Windows, source must exist on disk, as the implementation needs
|
|
|
|
to know whether to create a "File" or a "Directory" symbolic link.
|
|
|
|
"""
|
|
|
|
if isWindows():
|
|
|
|
import platform_utils_win32
|
|
|
|
source = _validate_winpath(source)
|
|
|
|
link_name = _validate_winpath(link_name)
|
|
|
|
target = os.path.join(os.path.dirname(link_name), source)
|
2018-09-27 17:46:58 +00:00
|
|
|
if isdir(target):
|
|
|
|
platform_utils_win32.create_dirsymlink(_makelongpath(source), link_name)
|
2016-11-01 18:24:03 +00:00
|
|
|
else:
|
2018-09-27 17:46:58 +00:00
|
|
|
platform_utils_win32.create_filesymlink(_makelongpath(source), link_name)
|
2016-11-01 18:24:03 +00:00
|
|
|
else:
|
|
|
|
return os.symlink(source, link_name)
|
|
|
|
|
|
|
|
|
|
|
|
def _validate_winpath(path):
|
|
|
|
path = os.path.normpath(path)
|
|
|
|
if _winpath_is_valid(path):
|
|
|
|
return path
|
|
|
|
raise ValueError("Path \"%s\" must be a relative path or an absolute "
|
|
|
|
"path starting with a drive letter".format(path))
|
|
|
|
|
|
|
|
|
|
|
|
def _winpath_is_valid(path):
|
|
|
|
"""Windows only: returns True if path is relative (e.g. ".\\foo") or is
|
|
|
|
absolute including a drive letter (e.g. "c:\\foo"). Returns False if path
|
|
|
|
is ambiguous (e.g. "x:foo" or "\\foo").
|
|
|
|
"""
|
|
|
|
assert isWindows()
|
|
|
|
path = os.path.normpath(path)
|
|
|
|
drive, tail = os.path.splitdrive(path)
|
|
|
|
if tail:
|
|
|
|
if not drive:
|
|
|
|
return tail[0] != os.sep # "\\foo" is invalid
|
|
|
|
else:
|
|
|
|
return tail[0] == os.sep # "x:foo" is invalid
|
|
|
|
else:
|
|
|
|
return not drive # "x:" is invalid
|
2016-11-03 17:37:53 +00:00
|
|
|
|
|
|
|
|
2018-09-27 17:46:58 +00:00
|
|
|
def _makelongpath(path):
|
|
|
|
"""Return the input path normalized to support the Windows long path syntax
|
|
|
|
("\\\\?\\" prefix) if needed, i.e. if the input path is longer than the
|
|
|
|
MAX_PATH limit.
|
|
|
|
"""
|
|
|
|
if isWindows():
|
|
|
|
# Note: MAX_PATH is 260, but, for directories, the maximum value is actually 246.
|
|
|
|
if len(path) < 246:
|
|
|
|
return path
|
|
|
|
if path.startswith(u"\\\\?\\"):
|
|
|
|
return path
|
|
|
|
if not os.path.isabs(path):
|
|
|
|
return path
|
|
|
|
# Append prefix and ensure unicode so that the special longpath syntax
|
|
|
|
# is supported by underlying Win32 API calls
|
|
|
|
return u"\\\\?\\" + os.path.normpath(path)
|
|
|
|
else:
|
|
|
|
return path
|
|
|
|
|
|
|
|
|
2019-11-11 09:34:16 +00:00
|
|
|
def rmtree(path, ignore_errors=False):
|
2018-09-27 17:46:58 +00:00
|
|
|
"""shutil.rmtree(path) wrapper with support for long paths on Windows.
|
|
|
|
|
|
|
|
Availability: Unix, Windows."""
|
2019-11-11 09:34:16 +00:00
|
|
|
onerror = None
|
2016-11-03 17:37:53 +00:00
|
|
|
if isWindows():
|
2019-11-11 09:34:16 +00:00
|
|
|
path = _makelongpath(path)
|
|
|
|
onerror = handle_rmtree_error
|
|
|
|
shutil.rmtree(path, ignore_errors=ignore_errors, onerror=onerror)
|
2016-11-03 17:37:53 +00:00
|
|
|
|
|
|
|
|
|
|
|
def handle_rmtree_error(function, path, excinfo):
|
|
|
|
# Allow deleting read-only files
|
|
|
|
os.chmod(path, stat.S_IWRITE)
|
|
|
|
function(path)
|
2016-11-01 18:34:55 +00:00
|
|
|
|
|
|
|
|
|
|
|
def rename(src, dst):
|
2018-09-27 17:46:58 +00:00
|
|
|
"""os.rename(src, dst) wrapper with support for long paths on Windows.
|
|
|
|
|
|
|
|
Availability: Unix, Windows."""
|
2016-11-01 18:34:55 +00:00
|
|
|
if isWindows():
|
|
|
|
# On Windows, rename fails if destination exists, see
|
|
|
|
# https://docs.python.org/2/library/os.html#os.rename
|
|
|
|
try:
|
2018-09-27 17:46:58 +00:00
|
|
|
os.rename(_makelongpath(src), _makelongpath(dst))
|
2016-11-01 18:34:55 +00:00
|
|
|
except OSError as e:
|
|
|
|
if e.errno == errno.EEXIST:
|
2018-09-27 17:46:58 +00:00
|
|
|
os.remove(_makelongpath(dst))
|
|
|
|
os.rename(_makelongpath(src), _makelongpath(dst))
|
2016-11-01 18:34:55 +00:00
|
|
|
else:
|
|
|
|
raise
|
|
|
|
else:
|
|
|
|
os.rename(src, dst)
|
2016-11-01 21:37:13 +00:00
|
|
|
|
|
|
|
|
2016-11-11 22:25:29 +00:00
|
|
|
def remove(path):
|
2018-09-27 17:46:58 +00:00
|
|
|
"""Remove (delete) the file path. This is a replacement for os.remove that
|
|
|
|
allows deleting read-only files on Windows, with support for long paths and
|
|
|
|
for deleting directory symbolic links.
|
|
|
|
|
|
|
|
Availability: Unix, Windows."""
|
2016-11-11 22:25:29 +00:00
|
|
|
if isWindows():
|
2018-09-27 17:46:58 +00:00
|
|
|
longpath = _makelongpath(path)
|
2016-11-11 22:25:29 +00:00
|
|
|
try:
|
2018-09-27 17:46:58 +00:00
|
|
|
os.remove(longpath)
|
2016-11-11 22:25:29 +00:00
|
|
|
except OSError as e:
|
|
|
|
if e.errno == errno.EACCES:
|
2018-09-27 17:46:58 +00:00
|
|
|
os.chmod(longpath, stat.S_IWRITE)
|
|
|
|
# Directory symbolic links must be deleted with 'rmdir'.
|
|
|
|
if islink(longpath) and isdir(longpath):
|
|
|
|
os.rmdir(longpath)
|
|
|
|
else:
|
|
|
|
os.remove(longpath)
|
2016-11-11 22:25:29 +00:00
|
|
|
else:
|
|
|
|
raise
|
|
|
|
else:
|
|
|
|
os.remove(path)
|
|
|
|
|
|
|
|
|
2018-09-27 17:46:58 +00:00
|
|
|
def walk(top, topdown=True, onerror=None, followlinks=False):
|
|
|
|
"""os.walk(path) wrapper with support for long paths on Windows.
|
|
|
|
|
|
|
|
Availability: Windows, Unix.
|
|
|
|
"""
|
|
|
|
if isWindows():
|
|
|
|
return _walk_windows_impl(top, topdown, onerror, followlinks)
|
|
|
|
else:
|
|
|
|
return os.walk(top, topdown, onerror, followlinks)
|
|
|
|
|
|
|
|
|
|
|
|
def _walk_windows_impl(top, topdown, onerror, followlinks):
|
|
|
|
try:
|
|
|
|
names = listdir(top)
|
2018-11-01 02:54:10 +00:00
|
|
|
except Exception as err:
|
2018-09-27 17:46:58 +00:00
|
|
|
if onerror is not None:
|
|
|
|
onerror(err)
|
|
|
|
return
|
|
|
|
|
|
|
|
dirs, nondirs = [], []
|
|
|
|
for name in names:
|
|
|
|
if isdir(os.path.join(top, name)):
|
|
|
|
dirs.append(name)
|
|
|
|
else:
|
|
|
|
nondirs.append(name)
|
|
|
|
|
|
|
|
if topdown:
|
|
|
|
yield top, dirs, nondirs
|
|
|
|
for name in dirs:
|
|
|
|
new_path = os.path.join(top, name)
|
|
|
|
if followlinks or not islink(new_path):
|
|
|
|
for x in _walk_windows_impl(new_path, topdown, onerror, followlinks):
|
|
|
|
yield x
|
|
|
|
if not topdown:
|
|
|
|
yield top, dirs, nondirs
|
|
|
|
|
|
|
|
|
|
|
|
def listdir(path):
|
|
|
|
"""os.listdir(path) wrapper with support for long paths on Windows.
|
|
|
|
|
|
|
|
Availability: Windows, Unix.
|
|
|
|
"""
|
|
|
|
return os.listdir(_makelongpath(path))
|
|
|
|
|
|
|
|
|
|
|
|
def rmdir(path):
|
|
|
|
"""os.rmdir(path) wrapper with support for long paths on Windows.
|
|
|
|
|
|
|
|
Availability: Windows, Unix.
|
|
|
|
"""
|
|
|
|
os.rmdir(_makelongpath(path))
|
|
|
|
|
|
|
|
|
|
|
|
def isdir(path):
|
|
|
|
"""os.path.isdir(path) wrapper with support for long paths on Windows.
|
|
|
|
|
|
|
|
Availability: Windows, Unix.
|
|
|
|
"""
|
|
|
|
return os.path.isdir(_makelongpath(path))
|
|
|
|
|
|
|
|
|
2016-11-01 21:37:13 +00:00
|
|
|
def islink(path):
|
2018-09-27 17:46:58 +00:00
|
|
|
"""os.path.islink(path) wrapper with support for long paths on Windows.
|
2016-11-01 21:37:13 +00:00
|
|
|
|
|
|
|
Availability: Windows, Unix.
|
|
|
|
"""
|
|
|
|
if isWindows():
|
|
|
|
import platform_utils_win32
|
2018-09-27 17:46:58 +00:00
|
|
|
return platform_utils_win32.islink(_makelongpath(path))
|
2016-11-01 21:37:13 +00:00
|
|
|
else:
|
|
|
|
return os.path.islink(path)
|
|
|
|
|
|
|
|
|
|
|
|
def readlink(path):
|
|
|
|
"""Return a string representing the path to which the symbolic link
|
|
|
|
points. The result may be either an absolute or relative pathname;
|
|
|
|
if it is relative, it may be converted to an absolute pathname using
|
|
|
|
os.path.join(os.path.dirname(path), result).
|
|
|
|
|
|
|
|
Availability: Windows, Unix.
|
|
|
|
"""
|
|
|
|
if isWindows():
|
|
|
|
import platform_utils_win32
|
2018-09-27 17:46:58 +00:00
|
|
|
return platform_utils_win32.readlink(_makelongpath(path))
|
2016-11-01 21:37:13 +00:00
|
|
|
else:
|
|
|
|
return os.readlink(path)
|
|
|
|
|
|
|
|
|
|
|
|
def realpath(path):
|
|
|
|
"""Return the canonical path of the specified filename, eliminating
|
|
|
|
any symbolic links encountered in the path.
|
|
|
|
|
|
|
|
Availability: Windows, Unix.
|
|
|
|
"""
|
|
|
|
if isWindows():
|
|
|
|
current_path = os.path.abspath(path)
|
|
|
|
path_tail = []
|
|
|
|
for c in range(0, 100): # Avoid cycles
|
|
|
|
if islink(current_path):
|
|
|
|
target = readlink(current_path)
|
|
|
|
current_path = os.path.join(os.path.dirname(current_path), target)
|
|
|
|
else:
|
|
|
|
basename = os.path.basename(current_path)
|
|
|
|
if basename == '':
|
|
|
|
path_tail.append(current_path)
|
|
|
|
break
|
|
|
|
path_tail.append(basename)
|
|
|
|
current_path = os.path.dirname(current_path)
|
|
|
|
path_tail.reverse()
|
|
|
|
result = os.path.normpath(os.path.join(*path_tail))
|
|
|
|
return result
|
|
|
|
else:
|
|
|
|
return os.path.realpath(path)
|