Merge changes from topic "windows-support"

* changes:
  Port os.rename calls to work on Windows
  Workaround shutil.rmtree limitation on Windows
  Add support for creating symbolic links on Windows
  Make "git command" and "forall" work on Windows

git_command.py

@@ -14,14 +14,14 @@
 # limitations under the License.
 
 from __future__ import print_function
-import fcntl
 import os
-import select
 import sys
 import subprocess
 import tempfile
 from signal import SIGTERM
+
 from error import GitError
+import platform_utils
 from trace import REPO_TRACE, IsTrace, Trace
 from wrapper import Wrapper
 
@@ -78,16 +78,6 @@ def terminate_ssh_clients():
 
 _git_version = None
 
-class _sfd(object):
-  """select file descriptor class"""
-  def __init__(self, fd, dest, std_name):
-    assert std_name in ('stdout', 'stderr')
-    self.fd = fd
-    self.dest = dest
-    self.std_name = std_name
-  def fileno(self):
-    return self.fd.fileno()
-
 class _GitCall(object):
   def version(self):
     p = GitCommand(None, ['--version'], capture_stdout=True)
@@ -253,19 +243,16 @@ class GitCommand(object):
 
   def _CaptureOutput(self):
     p = self.process
-    s_in = [_sfd(p.stdout, sys.stdout, 'stdout'),
+    s_in = platform_utils.FileDescriptorStreams.create()
-            _sfd(p.stderr, sys.stderr, 'stderr')]
+    s_in.add(p.stdout, sys.stdout, 'stdout')
+    s_in.add(p.stderr, sys.stderr, 'stderr')
     self.stdout = ''
     self.stderr = ''
 
-    for s in s_in:
+    while not s_in.is_done:
-      flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
+      in_ready = s_in.select()
-      fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
-
-    while s_in:
-      in_ready, _, _ = select.select(s_in, [], [])
       for s in in_ready:
-        buf = s.fd.read(4096)
+        buf = s.read()
         if not buf:
           s_in.remove(s)
           continue

manifest_xml.py

@@ -32,6 +32,7 @@ else:
 import gitc_utils
 from git_config import GitConfig
 from git_refs import R_HEADS, HEAD
+import platform_utils
 from project import RemoteSpec, Project, MetaProject
 from error import ManifestParseError, ManifestInvalidRevisionError
 
@@ -166,7 +167,7 @@ class XmlManifest(object):
     try:
       if os.path.lexists(self.manifestFile):
         os.remove(self.manifestFile)
-      os.symlink(os.path.join('manifests', name), self.manifestFile)
+      platform_utils.symlink(os.path.join('manifests', name), self.manifestFile)
     except OSError as e:
       raise ManifestParseError('cannot link manifest %s: %s' % (name, str(e)))
 

platform_utils.py

@@ -0,0 +1,244 @@
+#
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+import os
+import platform
+import select
+import shutil
+import stat
+
+from Queue import Queue
+from threading import Thread
+
+
+def isWindows():
+  """ Returns True when running with the native port of Python for Windows,
+  False when running on any other platform (including the Cygwin port of
+  Python).
+  """
+  # Note: The cygwin port of Python returns "CYGWIN_NT_xxx"
+  return platform.system() == "Windows"
+
+
+class FileDescriptorStreams(object):
+  """ Platform agnostic abstraction enabling non-blocking I/O over a
+  collection of file descriptors. This abstraction is required because
+  fctnl(os.O_NONBLOCK) is not supported on Windows.
+  """
+  @classmethod
+  def create(cls):
+    """ Factory method: instantiates the concrete class according to the
+    current platform.
+    """
+    if isWindows():
+      return _FileDescriptorStreamsThreads()
+    else:
+      return _FileDescriptorStreamsNonBlocking()
+
+  def __init__(self):
+    self.streams = []
+
+  def add(self, fd, dest, std_name):
+    """ Wraps an existing file descriptor as a stream.
+    """
+    self.streams.append(self._create_stream(fd, dest, std_name))
+
+  def remove(self, stream):
+    """ Removes a stream, when done with it.
+    """
+    self.streams.remove(stream)
+
+  @property
+  def is_done(self):
+    """ Returns True when all streams have been processed.
+    """
+    return len(self.streams) == 0
+
+  def select(self):
+    """ Returns the set of streams that have data available to read.
+    The returned streams each expose a read() and a close() method.
+    When done with a stream, call the remove(stream) method.
+    """
+    raise NotImplementedError
+
+  def _create_stream(fd, dest, std_name):
+    """ Creates a new stream wrapping an existing file descriptor.
+    """
+    raise NotImplementedError
+
+
+class _FileDescriptorStreamsNonBlocking(FileDescriptorStreams):
+  """ Implementation of FileDescriptorStreams for platforms that support
+  non blocking I/O.
+  """
+  class Stream(object):
+    """ Encapsulates a file descriptor """
+    def __init__(self, fd, dest, std_name):
+      self.fd = fd
+      self.dest = dest
+      self.std_name = std_name
+      self.set_non_blocking()
+
+    def set_non_blocking(self):
+      import fcntl
+      flags = fcntl.fcntl(self.fd, fcntl.F_GETFL)
+      fcntl.fcntl(self.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+    def fileno(self):
+      return self.fd.fileno()
+
+    def read(self):
+      return self.fd.read(4096)
+
+    def close(self):
+      self.fd.close()
+
+  def _create_stream(self, fd, dest, std_name):
+    return self.Stream(fd, dest, std_name)
+
+  def select(self):
+    ready_streams, _, _ = select.select(self.streams, [], [])
+    return ready_streams
+
+
+class _FileDescriptorStreamsThreads(FileDescriptorStreams):
+  """ Implementation of FileDescriptorStreams for platforms that don't support
+  non blocking I/O. This implementation requires creating threads issuing
+  blocking read operations on file descriptors.
+  """
+  def __init__(self):
+    super(_FileDescriptorStreamsThreads, self).__init__()
+    # The queue is shared accross all threads so we can simulate the
+    # behavior of the select() function
+    self.queue = Queue(10)  # Limit incoming data from streams
+
+  def _create_stream(self, fd, dest, std_name):
+    return self.Stream(fd, dest, std_name, self.queue)
+
+  def select(self):
+    # Return only one stream at a time, as it is the most straighforward
+    # thing to do and it is compatible with the select() function.
+    item = self.queue.get()
+    stream = item.stream
+    stream.data = item.data
+    return [stream]
+
+  class QueueItem(object):
+    """ Item put in the shared queue """
+    def __init__(self, stream, data):
+      self.stream = stream
+      self.data = data
+
+  class Stream(object):
+    """ Encapsulates a file descriptor """
+    def __init__(self, fd, dest, std_name, queue):
+      self.fd = fd
+      self.dest = dest
+      self.std_name = std_name
+      self.queue = queue
+      self.data = None
+      self.thread = Thread(target=self.read_to_queue)
+      self.thread.daemon = True
+      self.thread.start()
+
+    def close(self):
+      self.fd.close()
+
+    def read(self):
+      data = self.data
+      self.data = None
+      return data
+
+    def read_to_queue(self):
+      """ The thread function: reads everything from the file descriptor into
+      the shared queue and terminates when reaching EOF.
+      """
+      for line in iter(self.fd.readline, b''):
+        self.queue.put(_FileDescriptorStreamsThreads.QueueItem(self, line))
+      self.fd.close()
+      self.queue.put(_FileDescriptorStreamsThreads.QueueItem(self, None))
+
+
+def symlink(source, link_name):
+  """Creates a symbolic link pointing to source named link_name.
+  Note: On Windows, source must exist on disk, as the implementation needs
+  to know whether to create a "File" or a "Directory" symbolic link.
+  """
+  if isWindows():
+    import platform_utils_win32
+    source = _validate_winpath(source)
+    link_name = _validate_winpath(link_name)
+    target = os.path.join(os.path.dirname(link_name), source)
+    if os.path.isdir(target):
+      platform_utils_win32.create_dirsymlink(source, link_name)
+    else:
+      platform_utils_win32.create_filesymlink(source, link_name)
+  else:
+    return os.symlink(source, link_name)
+
+
+def _validate_winpath(path):
+  path = os.path.normpath(path)
+  if _winpath_is_valid(path):
+    return path
+  raise ValueError("Path \"%s\" must be a relative path or an absolute "
+                   "path starting with a drive letter".format(path))
+
+
+def _winpath_is_valid(path):
+  """Windows only: returns True if path is relative (e.g. ".\\foo") or is
+  absolute including a drive letter (e.g. "c:\\foo"). Returns False if path
+  is ambiguous (e.g. "x:foo" or "\\foo").
+  """
+  assert isWindows()
+  path = os.path.normpath(path)
+  drive, tail = os.path.splitdrive(path)
+  if tail:
+    if not drive:
+      return tail[0] != os.sep  # "\\foo" is invalid
+    else:
+      return tail[0] == os.sep  # "x:foo" is invalid
+  else:
+    return not drive  # "x:" is invalid
+
+
+def rmtree(path):
+  if isWindows():
+    shutil.rmtree(path, onerror=handle_rmtree_error)
+  else:
+    shutil.rmtree(path)
+
+
+def handle_rmtree_error(function, path, excinfo):
+  # Allow deleting read-only files
+  os.chmod(path, stat.S_IWRITE)
+  function(path)
+
+
+def rename(src, dst):
+  if isWindows():
+    # On Windows, rename fails if destination exists, see
+    # https://docs.python.org/2/library/os.html#os.rename
+    try:
+      os.rename(src, dst)
+    except OSError as e:
+      if e.errno == errno.EEXIST:
+        os.remove(dst)
+        os.rename(src, dst)
+      else:
+        raise
+  else:
+    os.rename(src, dst)

platform_utils_win32.py

@@ -0,0 +1,63 @@
+#
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+
+from ctypes import WinDLL, get_last_error, FormatError, WinError
+from ctypes.wintypes import BOOL, LPCWSTR, DWORD
+
+kernel32 = WinDLL('kernel32', use_last_error=True)
+
+# Win32 error codes
+ERROR_SUCCESS = 0
+ERROR_PRIVILEGE_NOT_HELD = 1314
+
+# Win32 API entry points
+CreateSymbolicLinkW = kernel32.CreateSymbolicLinkW
+CreateSymbolicLinkW.restype = BOOL
+CreateSymbolicLinkW.argtypes = (LPCWSTR,  # lpSymlinkFileName In
+                                LPCWSTR,  # lpTargetFileName In
+                                DWORD)    # dwFlags In
+
+# Symbolic link creation flags
+SYMBOLIC_LINK_FLAG_FILE = 0x00
+SYMBOLIC_LINK_FLAG_DIRECTORY = 0x01
+
+
+def create_filesymlink(source, link_name):
+  """Creates a Windows file symbolic link source pointing to link_name."""
+  _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_FILE)
+
+
+def create_dirsymlink(source, link_name):
+  """Creates a Windows directory symbolic link source pointing to link_name.
+  """
+  _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_DIRECTORY)
+
+
+def _create_symlink(source, link_name, dwFlags):
+  # Note: Win32 documentation for CreateSymbolicLink is incorrect.
+  # On success, the function returns "1".
+  # On error, the function returns some random value (e.g. 1280).
+  # The best bet seems to use "GetLastError" and check for error/success.
+  CreateSymbolicLinkW(link_name, source, dwFlags)
+  code = get_last_error()
+  if code != ERROR_SUCCESS:
+    error_desc = FormatError(code).strip()
+    if code == ERROR_PRIVILEGE_NOT_HELD:
+      raise OSError(errno.EPERM, error_desc, link_name)
+    error_desc = 'Error creating symbolic link %s: %s'.format(
+        link_name, error_desc)
+    raise WinError(code, error_desc)

project.py

@@ -35,6 +35,7 @@ from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \
 from error import GitError, HookError, UploadError, DownloadError
 from error import ManifestInvalidRevisionError
 from error import NoManifestException
+import platform_utils
 from trace import IsTrace, Trace
 
 from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
@@ -62,7 +63,7 @@ def _lwrite(path, content):
     fd.close()
 
   try:
-    os.rename(lock, path)
+    platform_utils.rename(lock, path)
   except OSError:
     os.remove(lock)
     raise
@@ -281,7 +282,7 @@ class _LinkFile(object):
           dest_dir = os.path.dirname(absDest)
           if not os.path.isdir(dest_dir):
             os.makedirs(dest_dir)
-        os.symlink(relSrc, absDest)
+        platform_utils.symlink(relSrc, absDest)
       except IOError:
         _error('Cannot link file %s to %s', relSrc, absDest)
 
@@ -2210,7 +2211,7 @@ class Project(object):
 
     if os.path.exists(tmpPath):
       if curlret == 0 and self._IsValidBundle(tmpPath, quiet):
-        os.rename(tmpPath, dstPath)
+        platform_utils.rename(tmpPath, dstPath)
         return True
       else:
         os.remove(tmpPath)
@@ -2311,10 +2312,10 @@ class Project(object):
             print("Retrying clone after deleting %s" %
                   self.gitdir, file=sys.stderr)
             try:
-              shutil.rmtree(os.path.realpath(self.gitdir))
+              platform_utils.rmtree(os.path.realpath(self.gitdir))
               if self.worktree and os.path.exists(os.path.realpath
                                                   (self.worktree)):
-                shutil.rmtree(os.path.realpath(self.worktree))
+                platform_utils.rmtree(os.path.realpath(self.worktree))
               return self._InitGitDir(mirror_git=mirror_git, force_sync=False)
             except:
               raise e
@@ -2356,9 +2357,9 @@ class Project(object):
           self.config.SetString('core.bare', None)
     except Exception:
       if init_obj_dir and os.path.exists(self.objdir):
-        shutil.rmtree(self.objdir)
+        platform_utils.rmtree(self.objdir)
       if init_git_dir and os.path.exists(self.gitdir):
-        shutil.rmtree(self.gitdir)
+        platform_utils.rmtree(self.gitdir)
       raise
 
   def _UpdateHooks(self):
@@ -2392,7 +2393,8 @@ class Project(object):
                 self.relpath, name)
           continue
       try:
-        os.symlink(os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
+        platform_utils.symlink(
+            os.path.relpath(stock_hook, os.path.dirname(dst)), dst)
       except OSError as e:
         if e.errno == errno.EPERM:
           raise GitError('filesystem must support symlinks')
@@ -2491,7 +2493,8 @@ class Project(object):
           os.makedirs(src)
 
         if name in to_symlink:
-          os.symlink(os.path.relpath(src, os.path.dirname(dst)), dst)
+          platform_utils.symlink(
+              os.path.relpath(src, os.path.dirname(dst)), dst)
         elif copy_all and not os.path.islink(dst):
           if os.path.isdir(src):
             shutil.copytree(src, dst)
@@ -2526,7 +2529,7 @@ class Project(object):
       except GitError as e:
         if force_sync:
           try:
-            shutil.rmtree(dotgit)
+            platform_utils.rmtree(dotgit)
             return self._InitWorkTree(force_sync=False, submodules=submodules)
           except:
             raise e
@@ -2546,7 +2549,7 @@ class Project(object):
         self._CopyAndLinkFiles()
     except Exception:
       if init_dotgit:
-        shutil.rmtree(dotgit)
+        platform_utils.rmtree(dotgit)
       raise
 
   def _gitdir_path(self, path):

subcmds/forall.py

@@ -15,17 +15,16 @@
 
 from __future__ import print_function
 import errno
-import fcntl
 import multiprocessing
 import re
 import os
-import select
 import signal
 import sys
 import subprocess
 
 from color import Coloring
 from command import Command, MirrorSafeCommand
+import platform_utils
 
 _CAN_COLOR = [
   'branch',
@@ -344,35 +343,25 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
   if opt.project_header:
     out = ForallColoring(config)
     out.redirect(sys.stdout)
-    class sfd(object):
-      def __init__(self, fd, dest):
-        self.fd = fd
-        self.dest = dest
-      def fileno(self):
-        return self.fd.fileno()
-
     empty = True
     errbuf = ''
 
     p.stdin.close()
-    s_in = [sfd(p.stdout, sys.stdout),
+    s_in = platform_utils.FileDescriptorStreams.create()
-            sfd(p.stderr, sys.stderr)]
+    s_in.add(p.stdout, sys.stdout, 'stdout')
+    s_in.add(p.stderr, sys.stderr, 'stderr')
 
-    for s in s_in:
+    while not s_in.is_done:
-      flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
+      in_ready = s_in.select()
-      fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
-
-    while s_in:
-      in_ready, _out_ready, _err_ready = select.select(s_in, [], [])
       for s in in_ready:
-        buf = s.fd.read(4096)
+        buf = s.read()
         if not buf:
-          s.fd.close()
+          s.close()
           s_in.remove(s)
           continue
 
         if not opt.verbose:
-          if s.fd != p.stdout:
+          if s.std_name == 'stderr':
             errbuf += buf
             continue
 

subcmds/gitc_delete.py

@@ -14,10 +14,10 @@
 # limitations under the License.
 
 from __future__ import print_function
-import shutil
 import sys
 
 from command import Command, GitcClientCommand
+import platform_utils
 
 from pyversion import is_python3
 if not is_python3():
@@ -50,4 +50,4 @@ and all locally downloaded sources.
       if not response == 'yes':
         print('Response was not "yes"\n Exiting...')
         sys.exit(1)
-    shutil.rmtree(self.gitc_manifest.gitc_client_dir)
+    platform_utils.rmtree(self.gitc_manifest.gitc_client_dir)

subcmds/init.py

@@ -17,7 +17,6 @@ from __future__ import print_function
 import os
 import platform
 import re
-import shutil
 import sys
 
 from pyversion import is_python3
@@ -35,6 +34,7 @@ from error import ManifestParseError
 from project import SyncBuffer
 from git_config import GitConfig
 from git_command import git_require, MIN_GIT_VERSION
+import platform_utils
 
 class Init(InteractiveCommand, MirrorSafeCommand):
   common = True
@@ -252,7 +252,7 @@ to update the working directory files.
       # Better delete the manifest git dir if we created it; otherwise next
       # time (when user fixes problems) we won't go through the "is_new" logic.
       if is_new:
-        shutil.rmtree(m.gitdir)
+        platform_utils.rmtree(m.gitdir)
       sys.exit(1)
 
     if opt.manifest_branch:

subcmds/sync.py

@@ -19,7 +19,6 @@ import netrc
 from optparse import SUPPRESS_HELP
 import os
 import re
-import shutil
 import socket
 import subprocess
 import sys
@@ -73,6 +72,7 @@ from project import Project
 from project import RemoteSpec
 from command import Command, MirrorSafeCommand
 from error import RepoChangedException, GitError, ManifestParseError
+import platform_utils
 from project import SyncBuffer
 from progress import Progress
 from wrapper import Wrapper
@@ -475,7 +475,7 @@ later is required to fix a server side protocol bug.
     # working git repository around. There shouldn't be any git projects here,
     # so rmtree works.
     try:
-      shutil.rmtree(os.path.join(path, '.git'))
+      platform_utils.rmtree(os.path.join(path, '.git'))
     except OSError:
       print('Failed to remove %s' % os.path.join(path, '.git'), file=sys.stderr)
       print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)