sync: kill git fetch process before SSH control master process

If the SSH control master process is killed while an active git
fetch is using its network socket, the underlying SSH client may
not realize the connection was broken.  This can lead to both the
client and the server waiting indefinitely for network messages
which will never be sent.

Work around the problem by keeping track of any processes that use
the tunnels we establish.  If we are about to kill any of the SSH
control masters that we started, ensure the clients using them are
successfully killed first.

Change-Id: Ida6c124dcb0c6a26bf7dd69cba2fbdc2ecd5b2fc
Signed-off-by: Shawn O. Pearce <sop@google.com>

git_command.py

@@ -17,6 +17,7 @@ import os
 import sys
 import subprocess
 import tempfile
+from signal import SIGTERM
 from error import GitError
 from trace import REPO_TRACE, IsTrace, Trace
 
@@ -29,8 +30,9 @@ LAST_CWD = None
 
 _ssh_proxy_path = None
 _ssh_sock_path = None
+_ssh_clients = []
 
-def _ssh_sock(create=True):
+def ssh_sock(create=True):
   global _ssh_sock_path
   if _ssh_sock_path is None:
     if not create:
@@ -51,6 +53,24 @@ def _ssh_proxy():
       'git_ssh')
   return _ssh_proxy_path
 
+def _add_ssh_client(p):
+  _ssh_clients.append(p)
+
+def _remove_ssh_client(p):
+  try:
+    _ssh_clients.remove(p)
+  except ValueError:
+    pass
+
+def terminate_ssh_clients():
+  global _ssh_clients
+  for p in _ssh_clients:
+    try:
+      os.kill(p.pid, SIGTERM)
+      p.wait()
+    except OSError:
+      pass
+  _ssh_clients = []
 
 class _GitCall(object):
   def version(self):
@@ -119,7 +139,7 @@ class GitCommand(object):
     if disable_editor:
       env['GIT_EDITOR'] = ':'
     if ssh_proxy:
-      env['REPO_SSH_SOCK'] = _ssh_sock()
+      env['REPO_SSH_SOCK'] = ssh_sock()
       env['GIT_SSH'] = _ssh_proxy()
 
     if project:
@@ -188,6 +208,9 @@ class GitCommand(object):
     except Exception, e:
       raise GitError('%s: %s' % (command[1], e))
 
+    if ssh_proxy:
+      _add_ssh_client(p)
+
     self.process = p
     self.stdin = p.stdin
 
@@ -210,4 +233,8 @@ class GitCommand(object):
     else:
       p.stderr = None
 
-    return self.process.wait()
+    try:
+      rc = p.wait()
+    finally:
+      _remove_ssh_client(p)
+    return rc

git_config.py

@@ -23,7 +23,10 @@ from signal import SIGTERM
 from urllib2 import urlopen, HTTPError
 from error import GitError, UploadError
 from trace import Trace
-from git_command import GitCommand, _ssh_sock
+
+from git_command import GitCommand
+from git_command import ssh_sock
+from git_command import terminate_ssh_clients
 
 R_HEADS = 'refs/heads/'
 R_TAGS  = 'refs/tags/'
@@ -371,7 +374,7 @@ def _open_ssh(host, port):
     return False
 
   command = ['ssh',
-             '-o','ControlPath %s' % _ssh_sock(),
+             '-o','ControlPath %s' % ssh_sock(),
              '-p',str(port),
              '-M',
              '-N',
@@ -391,6 +394,8 @@ def _open_ssh(host, port):
   return True
 
 def close_ssh():
+  terminate_ssh_clients()
+
   for key,p in _ssh_cache.iteritems():
     try:
       os.kill(p.pid, SIGTERM)
@@ -399,7 +404,7 @@ def close_ssh():
       pass
   _ssh_cache.clear()
 
-  d = _ssh_sock(create=False)
+  d = ssh_sock(create=False)
   if d:
     try:
       os.rmdir(os.path.dirname(d))

subcmds/branches.py

@@ -136,7 +136,7 @@ is shown, then the branch appears in all projects.
       hdr('%c%c %-*s' % (current, published, width, name))
       out.write(' |')
 
-      if in_cnt < project_cnt and (in_cnt == 1):
+      if in_cnt < project_cnt:
         fmt = out.write
         paths = []
         if in_cnt < project_cnt - in_cnt: 
@@ -150,15 +150,17 @@ is shown, then the branch appears in all projects.
           for b in i.projects:
             have.add(b.project)
           for p in projects:
-            paths.append(p.relpath)
+            if not p in have:
+              paths.append(p.relpath)
 
         s = ' %s %s' % (type, ', '.join(paths))
         if width + 7 + len(s) < 80:
           fmt(s)
         else:
-          out.nl()
+          fmt(' %s:' % type)
-          fmt('    %s:' % type)
           for p in paths:
             out.nl()
-            fmt('      %s' % p)
+            fmt(width*' ' + '          %s' % p)
+      else:
+        out.write(' in all projects')
       out.nl()

subcmds/sync.py

@@ -24,6 +24,7 @@ import time
 import xmlrpclib
 
 from git_command import GIT
+from git_refs import R_HEADS
 from project import HEAD
 from project import Project
 from project import RemoteSpec
@@ -147,32 +148,36 @@ later is required to fix a server side protocol bug.
         if not path:
           continue
         if path not in new_project_paths:
-          project = Project(
+          """If the path has already been deleted, we don't need to do it
-                         manifest = self.manifest,
+          """
-                         name = path,
+          if os.path.exists(self.manifest.topdir + '/' + path):
-                         remote = RemoteSpec('origin'),
+              project = Project(
-                         gitdir = os.path.join(self.manifest.topdir,
+                             manifest = self.manifest,
-                                               path, '.git'),
+                             name = path,
-                         worktree = os.path.join(self.manifest.topdir, path),
+                             remote = RemoteSpec('origin'),
-                         relpath = path,
+                             gitdir = os.path.join(self.manifest.topdir,
-                         revisionExpr = 'HEAD',
+                                                   path, '.git'),
-                         revisionId = None)
+                             worktree = os.path.join(self.manifest.topdir, path),
-          if project.IsDirty():
+                             relpath = path,
-            print >>sys.stderr, 'error: Cannot remove project "%s": \
+                             revisionExpr = 'HEAD',
+                             revisionId = None)
+
+              if project.IsDirty():
+                print >>sys.stderr, 'error: Cannot remove project "%s": \
 uncommitted changes are present' % project.relpath
-            print >>sys.stderr, '       commit changes, then run sync again'
+                print >>sys.stderr, '       commit changes, then run sync again'
-            return -1
+                return -1
-          else:
+              else:
-            print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
+                print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
-            shutil.rmtree(project.worktree)
+                shutil.rmtree(project.worktree)
-            # Try deleting parent subdirs if they are empty
+                # Try deleting parent subdirs if they are empty
-            dir = os.path.dirname(project.worktree)
+                dir = os.path.dirname(project.worktree)
-            while dir != self.manifest.topdir:
+                while dir != self.manifest.topdir:
-              try:
+                  try:
-                os.rmdir(dir)
+                    os.rmdir(dir)
-              except OSError:
+                  except OSError:
-                break
+                    break
-              dir = os.path.dirname(dir)
+                  dir = os.path.dirname(dir)
 
     new_project_paths.sort()
     fd = open(file_path, 'w')
@@ -201,6 +206,8 @@ uncommitted changes are present' % project.relpath
         p = self.manifest.manifestProject
         b = p.GetBranch(p.CurrentBranch)
         branch = b.merge
+        if branch.startswith(R_HEADS):
+          branch = branch[len(R_HEADS):]
 
         env = dict(os.environ)
         if (env.has_key('TARGET_PRODUCT') and
@@ -219,13 +226,13 @@ uncommitted changes are present' % project.relpath
             f = open(manifest_path, 'w')
             try:
               f.write(manifest_str)
-              self.manifest.Override(manifest_name)
             finally:
               f.close()
           except IOError:
             print >>sys.stderr, 'error: cannot write manifest to %s' % \
                 manifest_path
             sys.exit(1)
+          self.manifest.Override(manifest_name)
         else:
           print >>sys.stderr, 'error: %s' % manifest_str
           sys.exit(1)

subcmds/upload.py

@@ -20,6 +20,17 @@ from command import InteractiveCommand
 from editor import Editor
 from error import UploadError
 
+UNUSUAL_COMMIT_THRESHOLD = 5
+
+def _ConfirmManyUploads(multiple_branches=False):
+  if multiple_branches:
+    print "ATTENTION: One or more branches has an unusually high number of commits."
+  else:
+    print "ATTENTION: You are uploading an unusually high number of commits."
+  print "YOU PROBABLY DO NOT MEAN TO DO THIS. (Did you rebase across branches?)"
+  answer = raw_input("If you are sure you intend to do this, type 'yes': ").strip()
+  return answer == "yes"
+
 def _die(fmt, *args):
   msg = fmt % args
   print >>sys.stderr, 'error: %s' % msg
@@ -128,6 +139,10 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       answer = sys.stdin.readline().strip()
       answer = answer in ('y', 'Y', 'yes', '1', 'true', 't')
 
+    if answer:
+      if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD:
+        answer = _ConfirmManyUploads()
+
     if answer:
       self._UploadAndReport([branch], people)
     else:
@@ -192,6 +207,16 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
         todo.append(branch)
     if not todo:
       _die("nothing uncommented for upload")
+
+    many_commits = False
+    for branch in todo:
+      if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD:
+        many_commits = True
+        break
+    if many_commits:
+      if not _ConfirmManyUploads(multiple_branches=True):
+        _die("upload aborted by user")
+
     self._UploadAndReport(todo, people)
 
   def _FindGerritChange(self, branch):
@@ -258,6 +283,10 @@ Gerrit Code Review:  http://code.google.com/p/gerrit/
       print >>sys.stderr, "       use 'repo upload' without --replace"
       sys.exit(1)
 
+    if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD:
+      if not _ConfirmManyUploads(multiple_branches=True):
+        _die("upload aborted by user")
+
     branch.replace_changes = to_replace
     self._UploadAndReport([branch], people)