Fix mirror clients with no worktree

Commit "Make path references OS independent" (df14a70c45)
broke mirror clients by trying to invoke replace() on None
when there is no worktree.

Change-Id: Ie0a187058358f7dcdf83119e45cc65409c980f11

command.py

@@ -74,7 +74,7 @@ class Command(object):
         project = all.get(arg)
 
         if not project:
-          path = os.path.abspath(arg)
+          path = os.path.abspath(arg).replace('\\', '/')
 
           if not by_path:
             by_path = dict()
@@ -82,13 +82,15 @@ class Command(object):
               by_path[p.worktree] = p
 
           if os.path.exists(path):
+            oldpath = None
             while path \
-              and path != '/' \
+              and path != oldpath \
               and path != self.manifest.topdir:
               try:
                 project = by_path[path]
                 break
               except KeyError:
+                oldpath = path
                 path = os.path.dirname(path)
           else:
             try:

git_command.py

@@ -112,6 +112,9 @@ def git_require(min_version, fail=False):
     sys.exit(1)
   return False
 
+def _setenv(env, name, value):
+  env[name] = value.encode()
+
 class GitCommand(object):
   def __init__(self,
                project,
@@ -124,7 +127,7 @@ class GitCommand(object):
                ssh_proxy = False,
                cwd = None,
                gitdir = None):
-    env = dict(os.environ)
+    env = os.environ.copy()
 
     for e in [REPO_TRACE,
               GIT_DIR,
@@ -137,10 +140,10 @@ class GitCommand(object):
         del env[e]
 
     if disable_editor:
-      env['GIT_EDITOR'] = ':'
+      _setenv(env, 'GIT_EDITOR', ':')
     if ssh_proxy:
-      env['REPO_SSH_SOCK'] = ssh_sock()
-      env['GIT_SSH'] = _ssh_proxy()
+      _setenv(env, 'REPO_SSH_SOCK', ssh_sock())
+      _setenv(env, 'GIT_SSH', _ssh_proxy())
 
     if project:
       if not cwd:
@@ -151,7 +154,7 @@ class GitCommand(object):
     command = [GIT]
     if bare:
       if gitdir:
-        env[GIT_DIR] = gitdir
+        _setenv(env, GIT_DIR, gitdir)
       cwd = None
     command.extend(cmdv)
 

git_config.py

@@ -18,7 +18,13 @@ import os
 import re
 import subprocess
 import sys
+try:
+  import threading as _threading
+except ImportError:
+  import dummy_threading as _threading
 import time
+import urllib2
+
 from signal import SIGTERM
 from urllib2 import urlopen, HTTPError
 from error import GitError, UploadError
@@ -361,76 +367,97 @@ class RefSpec(object):
 _master_processes = []
 _master_keys = set()
 _ssh_master = True
+_master_keys_lock = None
+
+def init_ssh():
+  """Should be called once at the start of repo to init ssh master handling.
+
+  At the moment, all we do is to create our lock.
+  """
+  global _master_keys_lock
+  assert _master_keys_lock is None, "Should only call init_ssh once"
+  _master_keys_lock = _threading.Lock()
 
 def _open_ssh(host, port=None):
   global _ssh_master
 
-  # Check to see whether we already think that the master is running; if we
-  # think it's already running, return right away.
-  if port is not None:
-    key = '%s:%s' % (host, port)
-  else:
-    key = host
-
-  if key in _master_keys:
-    return True
-
-  if not _ssh_master \
-  or 'GIT_SSH' in os.environ \
-  or sys.platform in ('win32', 'cygwin'):
-    # failed earlier, or cygwin ssh can't do this
-    #
-    return False
-
-  # We will make two calls to ssh; this is the common part of both calls.
-  command_base = ['ssh',
-                   '-o','ControlPath %s' % ssh_sock(),
-                   host]
-  if port is not None:
-    command_base[1:1] = ['-p',str(port)]
-
-  # Since the key wasn't in _master_keys, we think that master isn't running.
-  # ...but before actually starting a master, we'll double-check.  This can
-  # be important because we can't tell that that 'git@myhost.com' is the same
-  # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file.
-  check_command = command_base + ['-O','check']
+  # Acquire the lock.  This is needed to prevent opening multiple masters for
+  # the same host when we're running "repo sync -jN" (for N > 1) _and_ the
+  # manifest <remote fetch="ssh://xyz"> specifies a different host from the
+  # one that was passed to repo init.
+  _master_keys_lock.acquire()
   try:
-    Trace(': %s', ' '.join(check_command))
-    check_process = subprocess.Popen(check_command,
-                                     stdout=subprocess.PIPE,
-                                     stderr=subprocess.PIPE)
-    check_process.communicate() # read output, but ignore it...
-    isnt_running = check_process.wait()
 
-    if not isnt_running:
-      # Our double-check found that the master _was_ infact running.  Add to
-      # the list of keys.
-      _master_keys.add(key)
+    # Check to see whether we already think that the master is running; if we
+    # think it's already running, return right away.
+    if port is not None:
+      key = '%s:%s' % (host, port)
+    else:
+      key = host
+
+    if key in _master_keys:
       return True
-  except Exception:
-    # Ignore excpetions.  We we will fall back to the normal command and print
-    # to the log there.
-    pass
 
-  command = command_base[:1] + \
-            ['-M', '-N'] + \
-            command_base[1:]
-  try:
-    Trace(': %s', ' '.join(command))
-    p = subprocess.Popen(command)
-  except Exception, e:
-    _ssh_master = False
-    print >>sys.stderr, \
-      '\nwarn: cannot enable ssh control master for %s:%s\n%s' \
-      % (host,port, str(e))
-    return False
+    if not _ssh_master \
+    or 'GIT_SSH' in os.environ \
+    or sys.platform in ('win32', 'cygwin'):
+      # failed earlier, or cygwin ssh can't do this
+      #
+      return False
 
-  _master_processes.append(p)
-  _master_keys.add(key)
-  time.sleep(1)
-  return True
+    # We will make two calls to ssh; this is the common part of both calls.
+    command_base = ['ssh',
+                     '-o','ControlPath %s' % ssh_sock(),
+                     host]
+    if port is not None:
+      command_base[1:1] = ['-p',str(port)]
+
+    # Since the key wasn't in _master_keys, we think that master isn't running.
+    # ...but before actually starting a master, we'll double-check.  This can
+    # be important because we can't tell that that 'git@myhost.com' is the same
+    # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file.
+    check_command = command_base + ['-O','check']
+    try:
+      Trace(': %s', ' '.join(check_command))
+      check_process = subprocess.Popen(check_command,
+                                       stdout=subprocess.PIPE,
+                                       stderr=subprocess.PIPE)
+      check_process.communicate() # read output, but ignore it...
+      isnt_running = check_process.wait()
+
+      if not isnt_running:
+        # Our double-check found that the master _was_ infact running.  Add to
+        # the list of keys.
+        _master_keys.add(key)
+        return True
+    except Exception:
+      # Ignore excpetions.  We we will fall back to the normal command and print
+      # to the log there.
+      pass
+
+    command = command_base[:1] + \
+              ['-M', '-N'] + \
+              command_base[1:]
+    try:
+      Trace(': %s', ' '.join(command))
+      p = subprocess.Popen(command)
+    except Exception, e:
+      _ssh_master = False
+      print >>sys.stderr, \
+        '\nwarn: cannot enable ssh control master for %s:%s\n%s' \
+        % (host,port, str(e))
+      return False
+
+    _master_processes.append(p)
+    _master_keys.add(key)
+    time.sleep(1)
+    return True
+  finally:
+    _master_keys_lock.release()
 
 def close_ssh():
+  global _master_keys_lock
+
   terminate_ssh_clients()
 
   for p in _master_processes:
@@ -449,6 +476,9 @@ def close_ssh():
     except OSError:
       pass
 
+  # We're done with the lock, so we can delete it.
+  _master_keys_lock = None
+
 URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
 URI_ALL = re.compile(r'^([a-z][a-z+]*)://([^@/]*@?[^/]*)/')
 
@@ -535,23 +565,25 @@ class Remote(object):
         try:
           info = urlopen(u).read()
           if info == 'NOT_AVAILABLE':
-            raise UploadError('Upload over ssh unavailable')
+            raise UploadError('%s: SSH disabled' % self.review)
           if '<' in info:
             # Assume the server gave us some sort of HTML
             # response back, like maybe a login page.
             #
-            raise UploadError('Cannot read %s:\n%s' % (u, info))
+            raise UploadError('%s: Cannot parse response' % u)
 
           self._review_protocol = 'ssh'
           self._review_host = info.split(" ")[0]
           self._review_port = info.split(" ")[1]
+        except urllib2.URLError, e:
+          raise UploadError('%s: %s' % (self.review, e.reason[1]))
         except HTTPError, e:
           if e.code == 404:
             self._review_protocol = 'http-post'
             self._review_host = None
             self._review_port = None
           else:
-            raise UploadError('Cannot guess Gerrit version')
+            raise UploadError('Upload over ssh unavailable')
 
         REVIEW_CACHE[u] = (
           self._review_protocol,

main.py

@@ -28,7 +28,7 @@ import re
 import sys
 
 from trace import SetTrace
-from git_config import close_ssh
+from git_config import init_ssh, close_ssh
 from command import InteractiveCommand
 from command import MirrorSafeCommand
 from command import PagedCommand
@@ -61,6 +61,8 @@ class _Repo(object):
   def __init__(self, repodir):
     self.repodir = repodir
     self.commands = all_commands
+    # add 'branch' as an alias for 'branches'
+    all_commands['branch'] = all_commands['branches']
 
   def _Run(self, argv):
     name = None
@@ -214,6 +216,7 @@ def _Main(argv):
   repo = _Repo(opt.repodir)
   try:
     try:
+      init_ssh()
       repo._Run(argv)
     finally:
       close_ssh()

manifest_xml.py

@@ -435,7 +435,7 @@ class XmlManifest(object):
       worktree = None
       gitdir = os.path.join(self.topdir, '%s.git' % name)
     else:
-      worktree = os.path.join(self.topdir, path)
+      worktree = os.path.join(self.topdir, path).replace('\\', '/')
       gitdir = os.path.join(self.repodir, 'projects/%s.git' % path)
 
     project = Project(manifest = self,

project.py

@@ -236,8 +236,11 @@ class Project(object):
     self.manifest = manifest
     self.name = name
     self.remote = remote
-    self.gitdir = gitdir
-    self.worktree = worktree
+    self.gitdir = gitdir.replace('\\', '/')
+    if worktree:
+      self.worktree = worktree.replace('\\', '/')
+    else:
+      self.worktree = None
     self.relpath = relpath
     self.revisionExpr = revisionExpr
 

repo

@@ -28,7 +28,7 @@ if __name__ == '__main__':
 del magic
 
 # increment this whenever we make important changes to this script
-VERSION = (1, 9)
+VERSION = (1, 10)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (1,0)
@@ -259,8 +259,8 @@ def _SetupGnuPG(quiet):
             gpg_dir, e.strerror)
       sys.exit(1)
 
-  env = dict(os.environ)
-  env['GNUPGHOME'] = gpg_dir
+  env = os.environ.copy()
+  env['GNUPGHOME'] = gpg_dir.encode()
 
   cmd = ['gpg', '--import']
   try:
@@ -378,8 +378,8 @@ def _Verify(cwd, branch, quiet):
         % (branch, cur)
       print >>sys.stderr
 
-  env = dict(os.environ)
-  env['GNUPGHOME'] = gpg_dir
+  env = os.environ.copy()
+  env['GNUPGHOME'] = gpg_dir.encode()
 
   cmd = [GIT, 'tag', '-v', cur]
   proc = subprocess.Popen(cmd,
@@ -430,10 +430,14 @@ def _FindRepo():
   dir = os.getcwd()
   repo = None
 
-  while dir != '/' and not repo:
+  olddir = None
+  while dir != '/' \
+    and dir != olddir \
+    and not repo:
     repo = os.path.join(dir, repodir, REPO_MAIN)
     if not os.path.isfile(repo):
       repo = None
+      olddir = dir
       dir = os.path.dirname(dir)
   return (repo, os.path.join(dir, repodir))
 
@@ -479,6 +483,7 @@ def _Help(args):
   if args:
     if args[0] == 'init':
       init_optparse.print_help()
+      sys.exit(0)
     else:
       print >>sys.stderr,\
       "error: '%s' is not a bootstrap command.\n"\

subcmds/download.py

@@ -36,6 +36,9 @@ makes it available in your project's local working directory.
     pass
 
   def _ParseChangeIds(self, args):
+    if not args:
+      self.Usage()
+
     to_get = []
     project = None
 

subcmds/forall.py

@@ -151,11 +151,11 @@ terminal and are not redirected.
     first = True
 
     for project in self.GetProjects(args):
-      env = dict(os.environ.iteritems())
+      env = os.environ.copy()
       def setenv(name, val):
         if val is None:
           val = ''
-        env[name] = val
+        env[name] = val.encode()
 
       setenv('REPO_PROJECT', project.name)
       setenv('REPO_PATH', project.relpath)
@@ -169,6 +169,12 @@ terminal and are not redirected.
       else:
         cwd = project.worktree
 
+      if not os.path.exists(cwd):
+        if (opt.project_header and opt.verbose) \
+        or not opt.project_header:
+          print >>sys.stderr, 'skipping %s/' % project.relpath
+        continue
+
       if opt.project_header:
         stdin = subprocess.PIPE
         stdout = subprocess.PIPE

subcmds/help.py

@@ -94,6 +94,8 @@ See 'repo help --all' for a complete list of recognized commands.
           body = getattr(cmd, bodyAttr)
         except AttributeError:
           return
+        if body == '' or body is None:
+          return
 
         self.nl()
 

subcmds/selfupdate.py

@@ -55,6 +55,7 @@ need to be performed by an end-user.
         print >>sys.stderr, "error: can't update repo"
         sys.exit(1)
 
+      rp.bare_git.gc('--auto')
       _PostRepoFetch(rp,
                      no_repo_verify = opt.no_repo_verify,
                      verbose = True)

subcmds/sync.py

@@ -185,6 +185,8 @@ later is required to fix a server side protocol bug.
         t.join()
 
     pm.end()
+    for project in projects:
+      project.bare_git.gc('--auto')
     return fetched
 
   def UpdateProjectList(self):
@@ -269,7 +271,7 @@ uncommitted changes are present' % project.relpath
         if branch.startswith(R_HEADS):
           branch = branch[len(R_HEADS):]
 
-        env = dict(os.environ)
+        env = os.environ.copy()
         if (env.has_key('TARGET_PRODUCT') and
             env.has_key('TARGET_BUILD_VARIANT')):
           target = '%s-%s' % (env['TARGET_PRODUCT'],
@@ -413,9 +415,9 @@ warning: Cannot automatically authenticate repo."""
       % (project.name, rev)
     return False
 
-  env = dict(os.environ)
-  env['GIT_DIR'] = project.gitdir
-  env['GNUPGHOME'] = gpg_dir
+  env = os.environ.copy()
+  env['GIT_DIR'] = project.gitdir.encode()
+  env['GNUPGHOME'] = gpg_dir.encode()
 
   cmd = [GIT, 'tag', '-v', cur]
   proc = subprocess.Popen(cmd,

subcmds/upload.py

@@ -283,15 +283,19 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
         have_errors = True
 
     print >>sys.stderr, ''
-    print >>sys.stderr, '--------------------------------------------'
+    print >>sys.stderr, '----------------------------------------------------------------------'
 
     if have_errors:
       for branch in todo:
         if not branch.uploaded:
-          print >>sys.stderr, '[FAILED] %-15s %-15s  (%s)' % (
+          if len(str(branch.error)) <= 30:
+            fmt = ' (%s)'
+          else:
+            fmt = '\n       (%s)'
+          print >>sys.stderr, ('[FAILED] %-15s %-15s' + fmt) % (
                  branch.project.relpath + '/', \
                  branch.name, \
-                 branch.error)
+                 str(branch.error))
       print >>sys.stderr, ''
 
     for branch in todo: