Perform copy file activity when creating a new work directory

Performance improvements in repo sync caused us to skip out of the
initial Sync_LocalHalf without ever running CopyFiles, so we didn't
create the top level Makefile in new clients whose manifest request
one with a <copyfile> element.

Now we run CopyFiles after the initial read-tree that populates
the project working directory.

Signed-off-by: Shawn O. Pearce <sop@google.com>

color.py

@@ -110,6 +110,9 @@ class Coloring(object):
   def write(self, fmt, *args):
     self._out.write(fmt % args)
 
+  def flush(self):
+    self._out.flush()
+
   def nl(self):
     self._out.write('\n')
 

command.py

@@ -27,6 +27,9 @@ class Command(object):
   manifest = None
   _optparse = None
 
+  def WantPager(self, opt):
+    return False
+
   @property
   def OptionParser(self):
     if self._optparse is None:
@@ -109,11 +112,15 @@ class InteractiveCommand(Command):
   """Command which requires user interaction on the tty and
      must not run within a pager, even if the user asks to.
   """
+  def WantPager(self, opt):
+    return False
 
 class PagedCommand(Command):
   """Command which defaults to output in a pager, as its
      display tends to be larger than one screen full.
   """
+  def WantPager(self, opt):
+    return True
 
 class MirrorSafeCommand(object):
   """Command permits itself to run within a mirror,

docs/manifest-format.txt

@@ -191,8 +191,3 @@ For example:
 Users may add projects to the local manifest prior to a `repo sync`
 invocation, instructing repo to automatically download and manage
 these extra projects.
-
-Currently the only supported feature of a local manifest is to
-add new remotes and/or projects.  In the future a local manifest
-may support picking different revisions of a project, or deleting
-projects specified in the default manifest.

editor.py

@@ -78,7 +78,11 @@ least one of these before using this command."""
 
       if subprocess.Popen(editor + [path]).wait() != 0:
         raise EditorError()
-      return open(path).read()
+      fd2 = open(path)
+      try:
+        return fd2.read()
+      finally:
+        fd2.close()
     finally:
       if fd:
         os.close(fd)

git_command.py

@@ -16,19 +16,40 @@
 import os
 import sys
 import subprocess
+import tempfile
 from error import GitError
+from trace import REPO_TRACE, IsTrace, Trace
 
 GIT = 'git'
 MIN_GIT_VERSION = (1, 5, 4)
 GIT_DIR = 'GIT_DIR'
-REPO_TRACE = 'REPO_TRACE'
 
 LAST_GITDIR = None
 LAST_CWD = None
-try:
-  TRACE = os.environ[REPO_TRACE] == '1'
-except KeyError:
-  TRACE = False
+
+_ssh_proxy_path = None
+_ssh_sock_path = None
+
+def _ssh_sock(create=True):
+  global _ssh_sock_path
+  if _ssh_sock_path is None:
+    if not create:
+      return None
+    dir = '/tmp'
+    if not os.path.exists(dir):
+      dir = tempfile.gettempdir()
+    _ssh_sock_path = os.path.join(
+      tempfile.mkdtemp('', 'ssh-', dir),
+      'master-%r@%h:%p')
+  return _ssh_sock_path
+
+def _ssh_proxy():
+  global _ssh_proxy_path
+  if _ssh_proxy_path is None:
+    _ssh_proxy_path = os.path.join(
+      os.path.dirname(__file__),
+      'git_ssh')
+  return _ssh_proxy_path
 
 
 class _GitCall(object):
@@ -56,6 +77,7 @@ class GitCommand(object):
                capture_stdout = False,
                capture_stderr = False,
                disable_editor = False,
+               ssh_proxy = False,
                cwd = None,
                gitdir = None):
     env = dict(os.environ)
@@ -72,6 +94,9 @@ class GitCommand(object):
 
     if disable_editor:
       env['GIT_EDITOR'] = ':'
+    if ssh_proxy:
+      env['REPO_SSH_SOCK'] = _ssh_sock()
+      env['GIT_SSH'] = _ssh_proxy()
 
     if project:
       if not cwd:
@@ -101,7 +126,7 @@ class GitCommand(object):
     else:
       stderr = None
 
-    if TRACE:
+    if IsTrace():
       global LAST_CWD
       global LAST_GITDIR
 
@@ -127,7 +152,7 @@ class GitCommand(object):
         dbg += ' 1>|'
       if stderr == subprocess.PIPE:
         dbg += ' 2>|'
-      print >>sys.stderr, dbg
+      Trace('%s', dbg)
 
     try:
       p = subprocess.Popen(command,

git_config.py

@@ -13,12 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import cPickle
 import os
 import re
+import subprocess
 import sys
+import time
+from signal import SIGTERM
 from urllib2 import urlopen, HTTPError
 from error import GitError, UploadError
-from git_command import GitCommand
+from trace import Trace
+from git_command import GitCommand, _ssh_sock
 
 R_HEADS = 'refs/heads/'
 R_TAGS  = 'refs/tags/'
@@ -29,6 +34,13 @@ REVIEW_CACHE = dict()
 def IsId(rev):
   return ID_RE.match(rev)
 
+def _key(name):
+  parts = name.split('.')
+  if len(parts) < 2:
+    return name.lower()
+  parts[ 0] = parts[ 0].lower()
+  parts[-1] = parts[-1].lower()
+  return '.'.join(parts)
 
 class GitConfig(object):
   _ForUser = None
@@ -48,14 +60,17 @@ class GitConfig(object):
     self.file = file
     self.defaults = defaults
     self._cache_dict = None
+    self._section_dict = None
     self._remotes = {}
     self._branches = {}
+    self._pickle = os.path.join(
+      os.path.dirname(self.file),
+      '.repopickle_' + os.path.basename(self.file))
 
   def Has(self, name, include_defaults = True):
     """Return true if this configuration file has the key.
     """
-    name = name.lower()
-    if name in self._cache:
+    if _key(name) in self._cache:
       return True
     if include_defaults and self.defaults:
       return self.defaults.Has(name, include_defaults = True)
@@ -83,10 +98,8 @@ class GitConfig(object):
        This configuration file is used first, if the key is not
        defined or all = True then the defaults are also searched.
     """
-    name = name.lower()
-
     try:
-      v = self._cache[name]
+      v = self._cache[_key(name)]
     except KeyError:
       if self.defaults:
         return self.defaults.GetString(name, all = all)
@@ -110,16 +123,16 @@ class GitConfig(object):
        The supplied value should be either a string,
        or a list of strings (to store multiple values).
     """
-    name = name.lower()
+    key = _key(name)
 
     try:
-      old = self._cache[name]
+      old = self._cache[key]
     except KeyError:
       old = []
 
     if value is None:
       if old:
-        del self._cache[name]
+        del self._cache[key]
         self._do('--unset-all', name)
 
     elif isinstance(value, list):
@@ -130,13 +143,13 @@ class GitConfig(object):
         self.SetString(name, value[0])
 
       elif old != value:
-        self._cache[name] = list(value)
+        self._cache[key] = list(value)
         self._do('--replace-all', name, value[0])
         for i in xrange(1, len(value)):
           self._do('--add', name, value[i])
 
     elif len(old) != 1 or old[0] != value:
-      self._cache[name] = [value]
+      self._cache[key] = [value]
       self._do('--replace-all', name, value)
 
   def GetRemote(self, name):
@@ -159,6 +172,33 @@ class GitConfig(object):
       self._branches[b.name] = b
     return b
 
+  def HasSection(self, section, subsection = ''):
+    """Does at least one key in section.subsection exist?
+    """
+    try:
+      return subsection in self._sections[section]
+    except KeyError:
+      return False
+
+  @property
+  def _sections(self):
+    d = self._section_dict
+    if d is None:
+      d = {}
+      for name in self._cache.keys():
+        p = name.split('.')
+        if 2 == len(p):
+          section = p[0]
+          subsect = ''
+        else:
+          section = p[0]
+          subsect = '.'.join(p[1:-1])
+        if section not in d:
+          d[section] = set()
+        d[section].add(subsect)
+        self._section_dict = d
+    return d
+
   @property
   def _cache(self):
     if self._cache_dict is None:
@@ -166,13 +206,54 @@ class GitConfig(object):
     return self._cache_dict
 
   def _Read(self):
+    d = self._ReadPickle()
+    if d is None:
+      d = self._ReadGit()
+      self._SavePickle(d)
+    return d
+
+  def _ReadPickle(self):
+    try:
+      if os.path.getmtime(self._pickle) \
+      <= os.path.getmtime(self.file):
+        os.remove(self._pickle)
+        return None
+    except OSError:
+      return None
+    try:
+      Trace(': unpickle %s', self.file)
+      fd = open(self._pickle, 'rb')
+      try:
+        return cPickle.load(fd)
+      finally:
+        fd.close()
+    except IOError:
+      os.remove(self._pickle)
+      return None
+    except cPickle.PickleError:
+      os.remove(self._pickle)
+      return None
+
+  def _SavePickle(self, cache):
+    try:
+      fd = open(self._pickle, 'wb')
+      try:
+        cPickle.dump(cache, fd, cPickle.HIGHEST_PROTOCOL)
+      finally:
+        fd.close()
+    except IOError:
+      os.remove(self._pickle)
+    except cPickle.PickleError:
+      os.remove(self._pickle)
+
+  def _ReadGit(self):
     d = self._do('--null', '--list')
     c = {}
     while d:
       lf = d.index('\n')
       nul = d.index('\0', lf + 1)
 
-      key = d[0:lf]
+      key = _key(d[0:lf])
       val = d[lf + 1:nul]
 
       if key in c:
@@ -253,6 +334,79 @@ class RefSpec(object):
     return s
 
 
+_ssh_cache = {}
+_ssh_master = True
+
+def _open_ssh(host, port=None):
+  global _ssh_master
+
+  if port is None:
+    port = 22
+
+  key = '%s:%s' % (host, port)
+  if key in _ssh_cache:
+    return True
+
+  if not _ssh_master \
+  or 'GIT_SSH' in os.environ \
+  or sys.platform == 'win32':
+    # failed earlier, or cygwin ssh can't do this
+    #
+    return False
+
+  command = ['ssh',
+             '-o','ControlPath %s' % _ssh_sock(),
+             '-p',str(port),
+             '-M',
+             '-N',
+             host]
+  try:
+    Trace(': %s', ' '.join(command))
+    p = subprocess.Popen(command)
+  except Exception, e:
+    _ssh_master = False
+    print >>sys.stderr, \
+      '\nwarn: cannot enable ssh control master for %s:%s\n%s' \
+      % (host,port, str(e))
+    return False
+
+  _ssh_cache[key] = p
+  time.sleep(1)
+  return True
+
+def close_ssh():
+  for key,p in _ssh_cache.iteritems():
+    os.kill(p.pid, SIGTERM)
+    p.wait()
+  _ssh_cache.clear()
+
+  d = _ssh_sock(create=False)
+  if d:
+    try:
+      os.rmdir(os.path.dirname(d))
+    except OSError:
+      pass
+
+URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
+URI_ALL = re.compile(r'^([a-z][a-z+]*)://([^@/]*@?[^/])/')
+
+def _preconnect(url):
+  m = URI_ALL.match(url)
+  if m:
+    scheme = m.group(1)
+    host = m.group(2)
+    if ':' in host:
+      host, port = host.split(':')
+    if scheme in ('ssh', 'git+ssh', 'ssh+git'):
+      return _open_ssh(host, port)
+    return False
+
+  m = URI_SCP.match(url)
+  if m:
+    host = m.group(1)
+    return _open_ssh(host)
+
+
 class Remote(object):
   """Configuration options related to a remote.
   """
@@ -266,6 +420,9 @@ class Remote(object):
                      self._Get('fetch', all=True))
     self._review_protocol = None
 
+  def PreConnectFetch(self):
+    return _preconnect(self.url)
+
   @property
   def ReviewProtocol(self):
     if self._review_protocol is None:
@@ -399,11 +556,23 @@ class Branch(object):
   def Save(self):
     """Save this branch back into the configuration.
     """
-    self._Set('merge', self.merge)
-    if self.remote:
-      self._Set('remote', self.remote.name)
+    if self._config.HasSection('branch', self.name):
+      if self.remote:
+        self._Set('remote', self.remote.name)
+      else:
+        self._Set('remote', None)
+      self._Set('merge', self.merge)
+
     else:
-      self._Set('remote', None)
+      fd = open(self._config.file, 'ab')
+      try:
+        fd.write('[branch "%s"]\n' % self.name)
+        if self.remote:
+          fd.write('\tremote = %s\n' % self.remote.name)
+        if self.merge:
+          fd.write('\tmerge = %s\n' % self.merge)
+      finally:
+        fd.close()
 
   def _Set(self, key, value):
     key = 'branch.%s.%s' % (self.name, key)

git_refs.py

@@ -0,0 +1,160 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+from trace import Trace
+
+HEAD    = 'HEAD'
+R_HEADS = 'refs/heads/'
+R_TAGS  = 'refs/tags/'
+R_PUB   = 'refs/published/'
+R_M     = 'refs/remotes/m/'
+
+
+class GitRefs(object):
+  def __init__(self, gitdir):
+    self._gitdir = gitdir
+    self._phyref = None
+    self._symref = None
+    self._mtime = {}
+
+  @property
+  def all(self):
+    self._EnsureLoaded()
+    return self._phyref
+
+  def get(self, name):
+    try:
+      return self.all[name]
+    except KeyError:
+      return ''
+
+  def deleted(self, name):
+    if self._phyref is not None:
+      if name in self._phyref:
+        del self._phyref[name]
+
+      if name in self._symref:
+        del self._symref[name]
+
+      if name in self._mtime:
+        del self._mtime[name]
+
+  def symref(self, name):
+    try:
+      self._EnsureLoaded()
+      return self._symref[name]
+    except KeyError:
+      return ''
+
+  def _EnsureLoaded(self):
+    if self._phyref is None or self._NeedUpdate():
+      self._LoadAll()
+
+  def _NeedUpdate(self):
+    Trace(': scan refs %s', self._gitdir)
+
+    for name, mtime in self._mtime.iteritems():
+      try:
+        if mtime != os.path.getmtime(os.path.join(self._gitdir, name)):
+          return True
+      except OSError:
+        return True
+    return False
+
+  def _LoadAll(self):
+    Trace(': load refs %s', self._gitdir)
+
+    self._phyref = {}
+    self._symref = {}
+    self._mtime = {}
+
+    self._ReadPackedRefs()
+    self._ReadLoose('refs/')
+    self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD)
+
+    scan = self._symref
+    attempts = 0
+    while scan and attempts < 5:
+      scan_next = {}
+      for name, dest in scan.iteritems():
+        if dest in self._phyref:
+          self._phyref[name] = self._phyref[dest]
+        else:
+          scan_next[name] = dest
+      scan = scan_next
+      attempts += 1
+
+  def _ReadPackedRefs(self):
+    path = os.path.join(self._gitdir, 'packed-refs')
+    try:
+      fd = open(path, 'rb')
+      mtime = os.path.getmtime(path)
+    except IOError:
+      return
+    except OSError:
+      return
+    try:
+      for line in fd:
+        if line[0] == '#':
+          continue
+        if line[0] == '^':
+          continue
+
+        line = line[:-1]
+        p = line.split(' ')
+        id = p[0]
+        name = p[1]
+
+        self._phyref[name] = id
+    finally:
+      fd.close()
+    self._mtime['packed-refs'] = mtime
+
+  def _ReadLoose(self, prefix):
+    base = os.path.join(self._gitdir, prefix)
+    for name in os.listdir(base):
+      p = os.path.join(base, name)
+      if os.path.isdir(p):
+        self._mtime[prefix] = os.path.getmtime(base)
+        self._ReadLoose(prefix + name + '/')
+      elif name.endswith('.lock'):
+        pass
+      else:
+        self._ReadLoose1(p, prefix + name)
+
+  def _ReadLoose1(self, path, name):
+    try:
+      fd = open(path, 'rb')
+      mtime = os.path.getmtime(path)
+    except OSError:
+      return
+    except IOError:
+      return
+    try:
+      id = fd.readline()
+    finally:
+      fd.close()
+
+    if not id:
+      return
+    id = id[:-1]
+
+    if id.startswith('ref: '):
+      self._symref[name] = id[5:]
+    else:
+      self._phyref[name] = id
+    self._mtime[name] = mtime

git_ssh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec ssh -o "ControlPath $REPO_SSH_SOCK" "$@"

main.py

@@ -27,7 +27,8 @@ import os
 import re
 import sys
 
-import git_command
+from trace import SetTrace
+from git_config import close_ssh
 from command import InteractiveCommand
 from command import MirrorSafeCommand
 from command import PagedCommand
@@ -79,7 +80,7 @@ class _Repo(object):
     gopts, gargs = global_options.parse_args(glob)
 
     if gopts.trace:
-      git_command.TRACE = True
+      SetTrace()
     if gopts.show_version:
       if name == 'help':
         name = 'version'
@@ -105,6 +106,8 @@ class _Repo(object):
             % name
       sys.exit(1)
 
+    copts, cargs = cmd.OptionParser.parse_args(argv)
+
     if not gopts.no_pager and not isinstance(cmd, InteractiveCommand):
       config = cmd.manifest.globalConfig
       if gopts.pager:
@@ -112,11 +115,10 @@ class _Repo(object):
       else:
         use_pager = config.GetBoolean('pager.%s' % name)
         if use_pager is None:
-          use_pager = isinstance(cmd, PagedCommand)
+          use_pager = cmd.WantPager(copts)
       if use_pager:
         RunPager(config)
 
-    copts, cargs = cmd.OptionParser.parse_args(argv)
     try:
       cmd.Execute(copts, cargs)
     except ManifestInvalidRevisionError, e:
@@ -211,7 +213,10 @@ def _Main(argv):
 
   repo = _Repo(opt.repodir)
   try:
-    repo._Run(argv)
+    try:
+      repo._Run(argv)
+    finally:
+      close_ssh()
   except KeyboardInterrupt:
     sys.exit(1)
   except RepoChangedException, rce:

progress.py

@@ -14,6 +14,8 @@
 # limitations under the License.
 
 import sys
+from time import time
+from trace import IsTrace
 
 class Progress(object):
   def __init__(self, title, total=0):
@@ -21,10 +23,21 @@ class Progress(object):
     self._total = total
     self._done = 0
     self._lastp = -1
+    self._start = time()
+    self._show = False
 
   def update(self, inc=1):
     self._done += inc
 
+    if IsTrace():
+      return
+
+    if not self._show:
+      if 0.5 <= time() - self._start:
+        self._show = True
+      else:
+        return
+
     if self._total <= 0:
       sys.stderr.write('\r%s: %d, ' % (
         self._title,
@@ -43,6 +56,9 @@ class Progress(object):
         sys.stderr.flush()
 
   def end(self):
+    if IsTrace() or not self._show:
+      return
+
     if self._total <= 0:
       sys.stderr.write('\r%s: %d, done.  \n' % (
         self._title,

project.py

@@ -28,11 +28,22 @@ from error import GitError, ImportError, UploadError
 from error import ManifestInvalidRevisionError
 from remote import Remote
 
-HEAD    = 'HEAD'
-R_HEADS = 'refs/heads/'
-R_TAGS  = 'refs/tags/'
-R_PUB   = 'refs/published/'
-R_M     = 'refs/remotes/m/'
+from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
+
+def _lwrite(path, content):
+  lock = '%s.lock' % path
+
+  fd = open(lock, 'wb')
+  try:
+    fd.write(content)
+  finally:
+    fd.close()
+
+  try:
+    os.rename(lock, path)
+  except OSError:
+    os.remove(lock)
+    raise
 
 def _error(fmt, *args):
   msg = fmt % args
@@ -144,16 +155,6 @@ class ReviewableBranch(object):
                                  self.replace_changes,
                                  people)
 
-  @property
-  def tip_url(self):
-    me = self.project.GetBranch(self.name)
-    commit = self.project.bare_git.rev_parse(R_HEADS + self.name)
-    return 'http://%s/r/%s' % (me.remote.review, commit[0:12])
-
-  @property
-  def owner_email(self):
-    return self.project.UserEmail
-
 
 class StatusColoring(Coloring):
   def __init__(self, config):
@@ -161,6 +162,7 @@ class StatusColoring(Coloring):
     self.project   = self.printer('header',    attr = 'bold')
     self.branch    = self.printer('header',    attr = 'bold')
     self.nobranch  = self.printer('nobranch',  fg = 'red')
+    self.important = self.printer('important', fg = 'red')
 
     self.added     = self.printer('added',     fg = 'green')
     self.changed   = self.printer('changed',   fg = 'red')
@@ -226,6 +228,7 @@ class Project(object):
     else:
       self.work_git = None
     self.bare_git = self._GitGetByExec(self, bare=True)
+    self.bare_ref = GitRefs(gitdir)
 
   @property
   def Exists(self):
@@ -237,14 +240,18 @@ class Project(object):
        The branch name omits the 'refs/heads/' prefix.
        None is returned if the project is on a detached HEAD.
     """
-    try:
-      b = self.work_git.GetHead()
-    except GitError:
-      return None
+    b = self.work_git.GetHead()
     if b.startswith(R_HEADS):
       return b[len(R_HEADS):]
     return None
 
+  def IsRebaseInProgress(self):
+    w = self.worktree
+    g = os.path.join(w, '.git')
+    return os.path.exists(os.path.join(g, 'rebase-apply')) \
+        or os.path.exists(os.path.join(g, 'rebase-merge')) \
+        or os.path.exists(os.path.join(w, '.dotest'))
+    
   def IsDirty(self, consider_untracked=True):
     """Is the working directory modified in some way?
     """
@@ -304,7 +311,7 @@ class Project(object):
     """Get all existing local branches.
     """
     current = self.CurrentBranch
-    all = self.bare_git.ListRefs()
+    all = self._allrefs
     heads = {}
     pubd = {}
 
@@ -342,10 +349,11 @@ class Project(object):
                                '--unmerged',
                                '--ignore-missing',
                                '--refresh')
+    rb = self.IsRebaseInProgress()
     di = self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD)
     df = self.work_git.DiffZ('diff-files')
     do = self.work_git.LsOthers()
-    if not di and not df and not do:
+    if not rb and not di and not df and not do:
       return 'CLEAN'
 
     out = StatusColoring(self.config)
@@ -358,6 +366,10 @@ class Project(object):
       out.branch('branch %s', branch)
     out.nl()
 
+    if rb:
+      out.important('prior sync failed; rebase still in progress')
+      out.nl()
+
     paths = list()
     paths.extend(di.keys())
     paths.extend(df.keys())
@@ -422,22 +434,31 @@ class Project(object):
 
 ## Publish / Upload ##
 
-  def WasPublished(self, branch):
+  def WasPublished(self, branch, all=None):
     """Was the branch published (uploaded) for code review?
        If so, returns the SHA-1 hash of the last published
        state for the branch.
     """
-    try:
-      return self.bare_git.rev_parse(R_PUB + branch)
-    except GitError:
-      return None
+    key = R_PUB + branch
+    if all is None:
+      try:
+        return self.bare_git.rev_parse(key)
+      except GitError:
+        return None
+    else:
+      try:
+        return all[key]
+      except KeyError:
+        return None
 
-  def CleanPublishedCache(self):
+  def CleanPublishedCache(self, all=None):
     """Prunes any stale published refs.
     """
+    if all is None:
+      all = self._allrefs
     heads = set()
     canrm = {}
-    for name, id in self._allrefs.iteritems():
+    for name, id in all.iteritems():
       if name.startswith(R_HEADS):
         heads.add(name)
       elif name.startswith(R_PUB):
@@ -573,27 +594,45 @@ class Project(object):
        Network access is not required.
     """
     self._InitWorkTree()
-    self.CleanPublishedCache()
+    all = self.bare_ref.all
+    self.CleanPublishedCache(all)
 
     rem = self.GetRemote(self.remote.name)
     rev = rem.ToLocal(self.revision)
-    try:
-      self.bare_git.rev_parse('--verify', '%s^0' % rev)
-    except GitError:
-      raise ManifestInvalidRevisionError(
-        'revision %s in %s not found' % (self.revision, self.name))
+    if rev in all:
+      revid = all[rev]
+    elif IsId(rev):
+      revid = rev
+    else:
+      try:
+        revid = self.bare_git.rev_parse('--verify', '%s^0' % rev)
+      except GitError:
+        raise ManifestInvalidRevisionError(
+          'revision %s in %s not found' % (self.revision, self.name))
 
-    branch = self.CurrentBranch
+    head = self.work_git.GetHead()
+    if head.startswith(R_HEADS):
+      branch = head[len(R_HEADS):]
+      try:
+        head = all[head]
+      except KeyError:
+        head = None
+    else:
+      branch = None
 
     if branch is None or syncbuf.detach_head:
       # Currently on a detached HEAD.  The user is assumed to
       # not have any local modifications worth worrying about.
       #
-      if os.path.exists(os.path.join(self.worktree, '.dotest')) \
-      or os.path.exists(os.path.join(self.worktree, '.git', 'rebase-apply')):
+      if self.IsRebaseInProgress():
         syncbuf.fail(self, _PriorSyncFailedError())
         return
 
+      if head == revid:
+        # No changes; don't do anything further.
+        #
+        return
+
       lost = self._revlist(not_rev(rev), HEAD)
       if lost:
         syncbuf.info(self, "discarding %d commits", len(lost))
@@ -605,6 +644,11 @@ class Project(object):
       self._CopyFiles()
       return
 
+    if head == revid:
+      # No changes; don't do anything further.
+      #
+      return
+
     branch = self.GetBranch(branch)
     merge = branch.LocalMerge
 
@@ -624,7 +668,7 @@ class Project(object):
       return
 
     upstream_gain = self._revlist(not_rev(HEAD), rev)
-    pub = self.WasPublished(branch.name)
+    pub = self.WasPublished(branch.name, all)
     if pub:
       not_merged = self._revlist(not_rev(rev), pub)
       if not_merged:
@@ -637,20 +681,16 @@ class Project(object):
                        "branch %s is published but is now %d commits behind",
                        branch.name,
                        len(upstream_gain))
-          syncbuf.info(self, "consider merging or rebasing the unpublished commits")
         return
-      elif upstream_gain:
-        # We can fast-forward safely.
+      elif pub == head:
+        # All published commits are merged, and thus we are a
+        # strict subset.  We can fast-forward safely.
         #
         def _doff():
           self._FastForward(rev)
           self._CopyFiles()
         syncbuf.later1(self, _doff)
         return
-      else:
-        # Trivially no changes in the upstream.
-        #
-        return
 
     if merge == rev:
       try:
@@ -742,61 +782,127 @@ class Project(object):
   def StartBranch(self, name):
     """Create a new branch off the manifest's revision.
     """
-    try:
-      self.bare_git.rev_parse(R_HEADS + name)
-      exists = True
-    except GitError:
-      exists = False;
+    head = self.work_git.GetHead()
+    if head == (R_HEADS + name):
+      return True
 
-    if exists:
-      if name == self.CurrentBranch:
-        return True
-      else:
-        cmd = ['checkout', name, '--']
-        return GitCommand(self, cmd).Wait() == 0
+    all = self.bare_ref.all
+    if (R_HEADS + name) in all:
+      return GitCommand(self,
+                        ['checkout', name, '--'],
+                        capture_stdout = True,
+                        capture_stderr = True).Wait() == 0
 
+    branch = self.GetBranch(name)
+    branch.remote = self.GetRemote(self.remote.name)
+    branch.merge = self.revision
+
+    rev = branch.LocalMerge
+    if rev in all:
+      revid = all[rev]
+    elif IsId(rev):
+      revid = rev
     else:
-      branch = self.GetBranch(name)
-      branch.remote = self.GetRemote(self.remote.name)
-      branch.merge = self.revision
+      revid = None
 
-      rev = branch.LocalMerge
-      cmd = ['checkout', '-b', branch.name, rev]
-      if GitCommand(self, cmd).Wait() == 0:
-        branch.Save()
-        return True
-      else:
-        return False
+    if head.startswith(R_HEADS):
+      try:
+        head = all[head]
+      except KeyError:
+        head = None
+
+    if revid and head and revid == head:
+      ref = os.path.join(self.gitdir, R_HEADS + name)
+      try:
+        os.makedirs(os.path.dirname(ref))
+      except OSError:
+        pass
+      _lwrite(ref, '%s\n' % revid)
+      _lwrite(os.path.join(self.worktree, '.git', HEAD),
+              'ref: %s%s\n' % (R_HEADS, name))
+      branch.Save()
+      return True
+
+    if GitCommand(self,
+                  ['checkout', '-b', branch.name, rev],
+                  capture_stdout = True,
+                  capture_stderr = True).Wait() == 0:
+      branch.Save()
+      return True
+    return False
 
   def CheckoutBranch(self, name):
     """Checkout a local topic branch.
     """
+    rev = R_HEADS + name
+    head = self.work_git.GetHead()
+    if head == rev:
+      # Already on the branch
+      #
+      return True
 
-    # Be sure the branch exists
+    all = self.bare_ref.all
     try:
-      tip_rev = self.bare_git.rev_parse(R_HEADS + name)
-    except GitError:
-      return False;
+      revid = all[rev]
+    except KeyError:
+      # Branch does not exist in this project
+      #
+      return False
 
-    # Do the checkout
-    cmd = ['checkout', name, '--']
-    return GitCommand(self, cmd).Wait() == 0
+    if head.startswith(R_HEADS):
+      try:
+        head = all[head]
+      except KeyError:
+        head = None
+
+    if head == revid:
+      # Same revision; just update HEAD to point to the new
+      # target branch, but otherwise take no other action.
+      #
+      _lwrite(os.path.join(self.worktree, '.git', HEAD),
+              'ref: %s%s\n' % (R_HEADS, name))
+      return True
+
+    return GitCommand(self,
+                      ['checkout', name, '--'],
+                      capture_stdout = True,
+                      capture_stderr = True).Wait() == 0
 
   def AbandonBranch(self, name):
     """Destroy a local topic branch.
     """
-    try:
-      tip_rev = self.bare_git.rev_parse(R_HEADS + name)
-    except GitError:
-      return
+    rev = R_HEADS + name
+    all = self.bare_ref.all
+    if rev not in all:
+      # Doesn't exist; assume already abandoned.
+      #
+      return True
 
-    if self.CurrentBranch == name:
-      self._Checkout(
-        self.GetRemote(self.remote.name).ToLocal(self.revision),
-        quiet=True)
+    head = self.work_git.GetHead()
+    if head == rev:
+      # We can't destroy the branch while we are sitting
+      # on it.  Switch to a detached HEAD.
+      #
+      head = all[head]
 
-    cmd = ['branch', '-D', name]
-    GitCommand(self, cmd, capture_stdout=True).Wait()
+      rev = self.GetRemote(self.remote.name).ToLocal(self.revision)
+      if rev in all:
+        revid = all[rev]
+      elif IsId(rev):
+        revid = rev
+      else:
+        revid = None
+
+      if revid and head == revid:
+        _lwrite(os.path.join(self.worktree, '.git', HEAD),
+                '%s\n' % revid)
+      else:
+        self._Checkout(rev, quiet=True)
+
+    return GitCommand(self,
+                      ['branch', '-D', name],
+                      capture_stdout = True,
+                      capture_stderr = True).Wait() == 0
 
   def PruneHeads(self):
     """Prune any topic branches already merged into upstream.
@@ -818,9 +924,8 @@ class Project(object):
       kill.append(cb)
 
     if kill:
-      try:
-        old = self.bare_git.GetHead()
-      except GitError:
+      old = self.bare_git.GetHead()
+      if old is None:
         old = 'refs/heads/please_never_use_this_as_a_branch_name'
 
       try:
@@ -861,11 +966,19 @@ class Project(object):
   def _RemoteFetch(self, name=None):
     if not name:
       name = self.remote.name
+
+    ssh_proxy = False
+    if self.GetRemote(name).PreConnectFetch():
+      ssh_proxy = True
+
     cmd = ['fetch']
     if not self.worktree:
       cmd.append('--update-head-ok')
     cmd.append(name)
-    return GitCommand(self, cmd, bare = True).Wait() == 0
+    return GitCommand(self,
+                      cmd,
+                      bare = True,
+                      ssh_proxy = ssh_proxy).Wait() == 0
 
   def _Checkout(self, rev, quiet=False):
     cmd = ['checkout']
@@ -972,14 +1085,17 @@ class Project(object):
     if self.manifest.branch:
       msg = 'manifest set to %s' % self.revision
       ref = R_M + self.manifest.branch
+      cur = self.bare_ref.symref(ref)
 
       if IsId(self.revision):
-        dst = self.revision + '^0'
-        self.bare_git.UpdateRef(ref, dst, message = msg, detach = True)
+        if cur != '' or self.bare_ref.get(ref) != self.revision:
+          dst = self.revision + '^0'
+          self.bare_git.UpdateRef(ref, dst, message = msg, detach = True)
       else:
         remote = self.GetRemote(self.remote.name)
         dst = remote.ToLocal(self.revision)
-        self.bare_git.symbolic_ref('-m', msg, ref, dst)
+        if cur != dst:
+          self.bare_git.symbolic_ref('-m', msg, ref, dst)
 
   def _InitMirrorHead(self):
     dst = self.GetRemote(self.remote.name).ToLocal(self.revision)
@@ -1014,15 +1130,14 @@ class Project(object):
       rev = self.GetRemote(self.remote.name).ToLocal(self.revision)
       rev = self.bare_git.rev_parse('%s^0' % rev)
 
-      f = open(os.path.join(dotgit, HEAD), 'wb')
-      f.write("%s\n" % rev)
-      f.close()
+      _lwrite(os.path.join(dotgit, HEAD), '%s\n' % rev)
 
       cmd = ['read-tree', '--reset', '-u']
       cmd.append('-v')
       cmd.append('HEAD')
       if GitCommand(self, cmd).Wait() != 0:
         raise GitError("cannot initialize work tree")
+      self._CopyFiles()
 
   def _gitdir_path(self, path):
     return os.path.join(self.gitdir, path)
@@ -1035,32 +1150,13 @@ class Project(object):
 
   @property
   def _allrefs(self):
-    return self.bare_git.ListRefs()
+    return self.bare_ref.all
 
   class _GitGetByExec(object):
     def __init__(self, project, bare):
       self._project = project
       self._bare = bare
 
-    def ListRefs(self, *args):
-      cmdv = ['for-each-ref', '--format=%(objectname) %(refname)']
-      cmdv.extend(args)
-      p = GitCommand(self._project,
-                     cmdv,
-                     bare = self._bare,
-                     capture_stdout = True,
-                     capture_stderr = True)
-      r = {}
-      for line in p.process.stdout:
-        id, name = line[:-1].split(' ', 2)
-        r[name] = id
-      if p.Wait() != 0:
-        raise GitError('%s for-each-ref %s: %s' % (
-                       self._project.name,
-                       str(args),
-                       p.stderr))
-      return r
-
     def LsOthers(self):
       p = GitCommand(self._project,
                      ['ls-files',
@@ -1126,7 +1222,18 @@ class Project(object):
         p.Wait()
 
     def GetHead(self):
-      return self.symbolic_ref(HEAD)
+      if self._bare:
+        path = os.path.join(self._project.gitdir, HEAD)
+      else:
+        path = os.path.join(self._project.worktree, '.git', HEAD)
+      fd = open(path, 'rb')
+      try:
+        line = fd.read()
+      finally:
+        fd.close()
+      if line.startswith('ref: '):
+        return line[5:-1]
+      return line[:-1]
 
     def SetHead(self, ref, message=None):
       cmdv = []
@@ -1162,6 +1269,7 @@ class Project(object):
       if not old:
         old = self.rev_parse(name)
       self.update_ref('-d', name, old)
+      self._project.bare_ref.deleted(name)
 
     def rev_list(self, *args):
       cmdv = ['rev-list']
@@ -1332,11 +1440,37 @@ class MetaProject(Project):
         if base:
           self.revision = base
 
+  @property
+  def LastFetch(self):
+    try:
+      fh = os.path.join(self.gitdir, 'FETCH_HEAD')
+      return os.path.getmtime(fh)
+    except OSError:
+      return 0
+
   @property
   def HasChanges(self):
     """Has the remote received new commits not yet checked out?
     """
+    if not self.remote or not self.revision:
+      return False
+
+    all = self.bare_ref.all
     rev = self.GetRemote(self.remote.name).ToLocal(self.revision)
-    if self._revlist(not_rev(HEAD), rev):
+    if rev in all:
+      revid = all[rev]
+    else:
+      revid = rev
+
+    head = self.work_git.GetHead()
+    if head.startswith(R_HEADS):
+      try:
+        head = all[head]
+      except KeyError:
+        head = None
+
+    if revid == head:
+      return False
+    elif self._revlist(not_rev(HEAD), rev):
       return True
     return False

repo

@@ -120,7 +120,7 @@ group.add_option('--mirror',
                  help='mirror the forrest')
 
 # Tool
-group = init_optparse.add_option_group('Version options')
+group = init_optparse.add_option_group('repo Version options')
 group.add_option('--repo-url',
                  dest='repo_url',
                  help='repo repository location', metavar='URL')

subcmds/abandon.py

@@ -16,6 +16,7 @@
 import sys
 from command import Command
 from git_command import git
+from progress import Progress
 
 class Abandon(Command):
   common = True
@@ -38,5 +39,23 @@ It is equivalent to "git branch -D <branchname>".
       print >>sys.stderr, "error: '%s' is not a valid name" % nb
       sys.exit(1)
 
-    for project in self.GetProjects(args[1:]):
-      project.AbandonBranch(nb)
+    nb = args[0]
+    err = []
+    all = self.GetProjects(args[1:])
+
+    pm = Progress('Abandon %s' % nb, len(all))
+    for project in all:
+      pm.update()
+      if not project.AbandonBranch(nb):
+        err.append(project)
+    pm.end()
+
+    if err:
+      if len(err) == len(all):
+        print >>sys.stderr, 'error: no project has branch %s' % nb
+      else:
+        for p in err:
+          print >>sys.stderr,\
+            "error: %s/: cannot abandon %s" \
+            % (p.relpath, nb)
+      sys.exit(1)

subcmds/branches.py

@@ -96,6 +96,10 @@ Summarizes the currently available topic branches.
           n.append(name)
       names = n
 
+    if not names:
+      print >>sys.stderr, '   (no branches)'
+      return
+
     width = 25
     for name in names:
       if width < len(name):

subcmds/checkout.py

@@ -15,6 +15,7 @@
 
 import sys
 from command import Command
+from progress import Progress
 
 class Checkout(Command):
   common = True
@@ -35,13 +36,23 @@ The command is equivalent to:
     if not args:
       self.Usage()
 
-    retValue = 0;
+    nb = args[0]
+    err = []
+    all = self.GetProjects(args[1:])
 
-    branch = args[0]
-    for project in self.GetProjects(args[1:]):
-      if not project.CheckoutBranch(branch):
-        retValue = 1;
-        print >>sys.stderr, "error: checking out branch '%s' in %s failed" % (branch, project.name)
+    pm = Progress('Checkout %s' % nb, len(all))
+    for project in all:
+      pm.update()
+      if not project.CheckoutBranch(nb):
+        err.append(project)
+    pm.end()
 
-    if (retValue != 0):
-      sys.exit(retValue);
+    if err:
+      if len(err) == len(all):
+        print >>sys.stderr, 'error: no project has branch %s' % nb
+      else:
+        for p in err:
+          print >>sys.stderr,\
+            "error: %s/: cannot checkout %s" \
+            % (p.relpath, nb)
+      sys.exit(1)

subcmds/forall.py

@@ -13,12 +13,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import fcntl
 import re
 import os
+import select
 import sys
 import subprocess
+
+from color import Coloring
 from command import Command, MirrorSafeCommand
 
+_CAN_COLOR = [
+  'branch',
+  'diff',
+  'grep',
+  'log',
+]
+
+class ForallColoring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, 'forall')
+    self.project = self.printer('project', attr='bold')
+
+
 class Forall(Command, MirrorSafeCommand):
   common = False
   helpSummary = "Run a shell command in each project"
@@ -28,6 +45,24 @@ class Forall(Command, MirrorSafeCommand):
   helpDescription = """
 Executes the same shell command in each project.
 
+Output Formatting
+-----------------
+
+The -p option causes '%prog' to bind pipes to the command's stdin,
+stdout and stderr streams, and pipe all output into a continuous
+stream that is displayed in a single pager session.  Project headings
+are inserted before the output of each command is displayed.  If the
+command produces no output in a project, no heading is displayed.
+
+The formatting convention used by -p is very suitable for some
+types of searching, e.g. `repo forall -p -c git log -SFoo` will
+print all commits that add or remove references to Foo.
+
+The -v option causes '%prog' to display stderr messages if a
+command produces output only on stderr.  Normally the -p option
+causes command output to be suppressed until the command produces
+at least one byte of output on stdout.
+
 Environment
 -----------
 
@@ -50,8 +85,8 @@ as written in the manifest.
 shell positional arguments ($1, $2, .., $#) are set to any arguments
 following <command>.
 
-stdin, stdout, stderr are inherited from the terminal and are
-not redirected.
+Unless -p is used, stdin, stdout, stderr are inherited from the
+terminal and are not redirected.
 """
 
   def _Options(self, p):
@@ -65,6 +100,17 @@ not redirected.
                  action='callback',
                  callback=cmd)
 
+    g = p.add_option_group('Output')
+    g.add_option('-p',
+                 dest='project_header', action='store_true',
+                 help='Show project headers before output')
+    g.add_option('-v', '--verbose',
+                 dest='verbose', action='store_true',
+                 help='Show command error messages')
+
+  def WantPager(self, opt):
+    return opt.project_header
+
   def Execute(self, opt, args):
     if not opt.command:
       self.Usage()
@@ -79,8 +125,31 @@ not redirected.
       cmd.append(cmd[0])
     cmd.extend(opt.command[1:])
 
+    if  opt.project_header \
+    and not shell \
+    and cmd[0] == 'git':
+      # If this is a direct git command that can enable colorized
+      # output and the user prefers coloring, add --color into the
+      # command line because we are going to wrap the command into
+      # a pipe and git won't know coloring should activate.
+      #
+      for cn in cmd[1:]:
+        if not cn.startswith('-'):
+          break
+      if cn in _CAN_COLOR:
+        class ColorCmd(Coloring):
+          def __init__(self, config, cmd):
+            Coloring.__init__(self, config, cmd)
+        if ColorCmd(self.manifest.manifestProject.config, cn).is_on:
+          cmd.insert(cmd.index(cn) + 1, '--color')
+
     mirror = self.manifest.IsMirror
+    out = ForallColoring(self.manifest.manifestProject.config)
+    out.redirect(sys.stdout)
+
     rc = 0
+    first = True
+
     for project in self.GetProjects(args):
       env = dict(os.environ.iteritems())
       def setenv(name, val):
@@ -102,10 +171,76 @@ not redirected.
       else:
         cwd = project.worktree
 
+      if opt.project_header:
+        stdin = subprocess.PIPE
+        stdout = subprocess.PIPE
+        stderr = subprocess.PIPE
+      else:
+        stdin = None
+        stdout = None
+        stderr = None
+
       p = subprocess.Popen(cmd,
                            cwd = cwd,
                            shell = shell,
-                           env = env)
+                           env = env,
+                           stdin = stdin,
+                           stdout = stdout,
+                           stderr = stderr)
+
+      if opt.project_header:
+        class sfd(object):
+          def __init__(self, fd, dest):
+            self.fd = fd
+            self.dest = dest
+          def fileno(self):
+            return self.fd.fileno()
+
+        empty = True
+        didout = False
+        errbuf = ''
+
+        p.stdin.close()
+        s_in = [sfd(p.stdout, sys.stdout),
+                sfd(p.stderr, sys.stderr)]
+
+        for s in s_in:
+          flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
+          fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+        while s_in:
+          in_ready, out_ready, err_ready = select.select(s_in, [], [])
+          for s in in_ready:
+            buf = s.fd.read(4096)
+            if not buf:
+              s.fd.close()
+              s_in.remove(s)
+              continue
+
+            if not opt.verbose:
+              if s.fd == p.stdout:
+                didout = True
+              else:
+                errbuf += buf
+                continue
+
+            if empty:
+              if first:
+                first = False
+              else:
+                out.nl()
+              out.project('project %s/', project.relpath)
+              out.nl()
+              out.flush()
+              if errbuf:
+                sys.stderr.write(errbuf)
+                sys.stderr.flush()
+                errbuf = ''
+              empty = False
+
+            s.dest.write(buf)
+            s.dest.flush()
+
       r = p.wait()
       if r != 0 and r != rc:
         rc = r

subcmds/grep.py

@@ -33,8 +33,8 @@ class Grep(PagedCommand):
   helpDescription = """
 Search for the specified patterns in all project files.
 
-Options
--------
+Boolean Options
+---------------
 
 The following options can appear as often as necessary to express
 the pattern to locate:

subcmds/help.py

@@ -107,7 +107,7 @@ See 'repo help --all' for a complete list of recognized commands.
         body = body.strip()
         body = body.replace('%prog', me)
 
-        asciidoc_hdr = re.compile(r'^\n?([^\n]{1,})\n(={2,}|-{2,})$')
+        asciidoc_hdr = re.compile(r'^\n?([^\n]{1,})\n([=~-]{2,})$')
         for para in body.split("\n\n"):
           if para.startswith(' '):
             self.write('%s', para)
@@ -117,9 +117,19 @@ See 'repo help --all' for a complete list of recognized commands.
 
           m = asciidoc_hdr.match(para)
           if m:
-            self.heading('%s', m.group(1))
+            title = m.group(1)
+            type = m.group(2)
+            if type[0] in ('=', '-'):
+              p = self.heading
+            else:
+              def _p(fmt, *args):
+                self.write('  ')
+                self.heading(fmt, *args)
+              p = _p
+
+            p('%s', title)
             self.nl()
-            self.heading('%s', ''.ljust(len(m.group(1)),'-'))
+            p('%s', ''.ljust(len(title),type[0]))
             self.nl()
             continue
 
@@ -128,8 +138,8 @@ See 'repo help --all' for a complete list of recognized commands.
         self.wrap.end_paragraph(0)
 
     out = _Out(self.manifest.globalConfig)
-    cmd.OptionParser.print_help()
     out._PrintSection('Summary', 'helpSummary')
+    cmd.OptionParser.print_help()
     out._PrintSection('Description', 'helpDescription')
 
   def _Options(self, p):

subcmds/init.py

@@ -35,9 +35,20 @@ The latest repo source code and manifest collection is downloaded
 from the server and is installed in the .repo/ directory in the
 current working directory.
 
-The optional <manifest> argument can be used to specify an alternate
-manifest to be used. If no manifest is specified, the manifest
-default.xml will be used.
+The optional -b argument can be used to select the manifest branch
+to checkout and use.  If no branch is specified, master is assumed.
+
+The optional -m argument can be used to specify an alternate manifest
+to be used. If no manifest is specified, the manifest default.xml
+will be used.
+
+Switching Manifest Branches
+---------------------------
+
+To switch to another manifest branch, `repo init -b otherbranch`
+may be used in an existing client.  However, as this only updates the
+manifest, a subsequent `repo sync` (or `repo sync -d`) is necessary
+to update the working directory files.
 """
 
   def _Options(self, p):
@@ -64,7 +75,7 @@ default.xml will be used.
 
 
     # Tool
-    g = p.add_option_group('Version options')
+    g = p.add_option_group('repo Version options')
     g.add_option('--repo-url',
                  dest='repo_url',
                  help='repo repository location', metavar='URL')

subcmds/selfupdate.py

@@ -35,10 +35,11 @@ need to be performed by an end-user.
 """
 
   def _Options(self, p):
-    p.add_option('--no-repo-verify',
+    g = p.add_option_group('repo Version options')
+    g.add_option('--no-repo-verify',
                  dest='no_repo_verify', action='store_true',
                  help='do not verify repo source code')
-    p.add_option('--repo-upgraded',
+    g.add_option('--repo-upgraded',
                  dest='repo_upgraded', action='store_true',
                  help=SUPPRESS_HELP)
 

subcmds/stage.py

@@ -55,12 +55,12 @@ The '%prog' command stages files to prepare the next commit.
 
     out = _ProjectList(self.manifest.manifestProject.config)
     while True:
-      out.header('        %-20s %s', 'project', 'path')
+      out.header('        %s', 'project')
       out.nl()
 
       for i in xrange(0, len(all)):
         p = all[i]
-        out.write('%3d:    %-20s %s', i + 1, p.name, p.relpath + '/')
+        out.write('%3d:    %s', i + 1, p.relpath + '/')
         out.nl()
       out.nl()
 

subcmds/start.py

@@ -16,26 +16,17 @@
 import sys
 from command import Command
 from git_command import git
+from progress import Progress
 
 class Start(Command):
   common = True
   helpSummary = "Start a new branch for development"
   helpUsage = """
 %prog <newbranchname> [<project>...]
-
-This subcommand starts a new branch of development that is automatically
-pulled from a remote branch.
-
-It is equivalent to the following git commands:
-
-"git branch --track <newbranchname> m/<codeline>",
-or 
-"git checkout --track -b <newbranchname> m/<codeline>".
-
-All three forms set up the config entries that repo bases some of its
-processing on.  Use %prog or git branch or checkout with --track to ensure
-the configuration data is set up properly.
-
+"""
+  helpDescription = """
+'%prog' begins a new branch of development, starting from the
+revision specified in the manifest.
 """
 
   def Execute(self, opt, args):
@@ -48,12 +39,18 @@ the configuration data is set up properly.
       sys.exit(1)
 
     err = []
-    for project in self.GetProjects(args[1:]):
+    all = self.GetProjects(args[1:])
+
+    pm = Progress('Starting %s' % nb, len(all))
+    for project in all:
+      pm.update()
       if not project.StartBranch(nb):
         err.append(project)
+    pm.end()
 
     if err:
-      err.sort()
       for p in err:
-        print >>sys.stderr, "error: cannot start in %s" % p.relpath
+        print >>sys.stderr,\
+          "error: %s/: cannot start %s" \
+          % (p.relpath, nb)
       sys.exit(1)

subcmds/status.py

@@ -64,6 +64,19 @@ the following meanings:
     all = self.GetProjects(args)
     clean = 0
 
+    on = {}
+    for project in all:
+      cb = project.CurrentBranch
+      if cb:
+        if cb not in on:
+          on[cb] = []
+        on[cb].append(project)
+
+    branch_names = list(on.keys())
+    branch_names.sort()
+    for cb in branch_names:
+      print '# on branch %s' % cb
+
     for project in all:
       state = project.PrintWorkTreeStatus()
       if state == 'CLEAN':

subcmds/sync.py

@@ -18,6 +18,7 @@ import os
 import re
 import subprocess
 import sys
+import time
 
 from git_command import GIT
 from project import HEAD
@@ -52,6 +53,35 @@ The -d/--detach option can be used to switch specified projects
 back to the manifest revision.  This option is especially helpful
 if the project is currently on a topic branch, but the manifest
 revision is temporarily needed.
+
+SSH Connections
+---------------
+
+If at least one project remote URL uses an SSH connection (ssh://,
+git+ssh://, or user@host:path syntax) repo will automatically
+enable the SSH ControlMaster option when connecting to that host.
+This feature permits other projects in the same '%prog' session to
+reuse the same SSH tunnel, saving connection setup overheads.
+
+To disable this behavior on UNIX platforms, set the GIT_SSH
+environment variable to 'ssh'.  For example:
+
+  export GIT_SSH=ssh
+  %prog
+
+Compatibility
+~~~~~~~~~~~~~
+
+This feature is automatically disabled on Windows, due to the lack
+of UNIX domain socket support.
+
+This feature is not compatible with url.insteadof rewrites in the
+user's ~/.gitconfig.  '%prog' is currently not able to perform the
+rewrite early enough to establish the ControlMaster tunnel.
+
+If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
+later is required to fix a server side protocol bug.
+
 """
 
   def _Options(self, p):
@@ -65,14 +95,15 @@ revision is temporarily needed.
                  dest='detach_head', action='store_true',
                  help='detach projects back to manifest revision')
 
-    p.add_option('--no-repo-verify',
+    g = p.add_option_group('repo Version options')
+    g.add_option('--no-repo-verify',
                  dest='no_repo_verify', action='store_true',
                  help='do not verify repo source code')
-    p.add_option('--repo-upgraded',
+    g.add_option('--repo-upgraded',
                  dest='repo_upgraded', action='store_true',
                  help=SUPPRESS_HELP)
 
-  def _Fetch(self, *projects):
+  def _Fetch(self, projects):
     fetched = set()
     pm = Progress('Fetching projects', len(projects))
     for project in projects:
@@ -106,7 +137,14 @@ revision is temporarily needed.
     all = self.GetProjects(args, missing_ok=True)
 
     if not opt.local_only:
-      fetched = self._Fetch(rp, mp, *all)
+      to_fetch = []
+      now = time.time()
+      if (24 * 60 * 60) <= (now - rp.LastFetch):
+        to_fetch.append(rp)
+      to_fetch.append(mp)
+      to_fetch.extend(all)
+
+      fetched = self._Fetch(to_fetch)
       _PostRepoFetch(rp, opt.no_repo_verify)
       if opt.network_only:
         # bail out now; the rest touches the working tree
@@ -124,7 +162,7 @@ revision is temporarily needed.
         for project in all:
           if project.gitdir not in fetched:
             missing.append(project)
-        self._Fetch(*missing)
+        self._Fetch(missing)
 
     syncbuf = SyncBuffer(mp.config,
                          detach_head = opt.detach_head)

subcmds/upload.py

@@ -38,22 +38,21 @@ class Upload(InteractiveCommand):
 %prog [--re --cc] {[<project>]... | --replace <project>}
 """
   helpDescription = """
-The '%prog' command is used to send changes to the Gerrit code
-review system.  It searches for changes in local projects that do
-not yet exist in the corresponding remote repository.  If multiple
-changes are found, '%prog' opens an editor to allow the
-user to choose which change to upload.  After a successful upload,
-repo prints the URL for the change in the Gerrit code review system.
+The '%prog' command is used to send changes to the Gerrit Code
+Review system.  It searches for topic branches in local projects
+that have not yet been published for review.  If multiple topic
+branches are found, '%prog' opens an editor to allow the user to
+select which branches to upload.
 
-'%prog' searches for uploadable changes in all projects listed
-at the command line.  Projects can be specified either by name, or
-by a relative or absolute path to the project's local directory. If
-no projects are specified, '%prog' will search for uploadable
-changes in all projects listed in the manifest.
+'%prog' searches for uploadable changes in all projects listed at
+the command line.  Projects can be specified either by name, or by
+a relative or absolute path to the project's local directory. If no
+projects are specified, '%prog' will search for uploadable changes
+in all projects listed in the manifest.
 
 If the --reviewers or --cc options are passed, those emails are
 added to the respective list of users, and emails are sent to any
-new users.  Users passed to --reviewers must be already registered
+new users.  Users passed as --reviewers must already be registered
 with the code review system, or the upload will fail.
 
 If the --replace option is passed the user can designate which
@@ -61,6 +60,33 @@ existing change(s) in Gerrit match up to the commits in the branch
 being uploaded.  For each matched pair of change,commit the commit
 will be added as a new patch set, completely replacing the set of
 files and description associated with the change in Gerrit.
+
+Configuration
+-------------
+
+review.URL.autoupload:
+
+To disable the "Upload ... (y/n)?" prompt, you can set a per-project
+or global Git configuration option.  If review.URL.autoupload is set
+to "true" then repo will assume you always answer "y" at the prompt,
+and will not prompt you further.  If it is set to "false" then repo
+will assume you always answer "n", and will abort.
+
+The URL must match the review URL listed in the manifest XML file,
+or in the .git/config within the project.  For example:
+
+  [remote "origin"]
+    url = git://git.example.com/project.git
+    review = http://review.example.com/
+
+  [review "http://review.example.com/"]
+    autoupload = true
+
+References
+----------
+
+Gerrit Code Review:  http://code.google.com/p/gerrit/
+
 """
 
   def _Options(self, p):
@@ -77,21 +103,32 @@ files and description associated with the change in Gerrit.
   def _SingleBranch(self, branch, people):
     project = branch.project
     name = branch.name
-    date = branch.date
-    list = branch.commits
+    remote = project.GetBranch(name).remote
 
-    print 'Upload project %s/:' % project.relpath
-    print '  branch %s (%2d commit%s, %s):' % (
-                  name,
-                  len(list),
-                  len(list) != 1 and 's' or '',
-                  date)
-    for commit in list:
-      print '         %s' % commit
+    key = 'review.%s.autoupload' % remote.review
+    answer = project.config.GetBoolean(key)
 
-    sys.stdout.write('(y/n)? ')
-    answer = sys.stdin.readline().strip()
-    if answer in ('y', 'Y', 'yes', '1', 'true', 't'):
+    if answer is False:
+      _die("upload blocked by %s = false" % key)
+
+    if answer is None:
+      date = branch.date
+      list = branch.commits
+
+      print 'Upload project %s/:' % project.relpath
+      print '  branch %s (%2d commit%s, %s):' % (
+                    name,
+                    len(list),
+                    len(list) != 1 and 's' or '',
+                    date)
+      for commit in list:
+        print '         %s' % commit
+
+      sys.stdout.write('to %s (y/n)? ' % remote.review)
+      answer = sys.stdin.readline().strip()
+      answer = answer in ('y', 'Y', 'yes', '1', 'true', 't')
+
+    if answer:
       self._UploadAndReport([branch], people)
     else:
       _die("upload aborted by user")

trace.py

@@ -0,0 +1,34 @@
+#
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import os
+REPO_TRACE = 'REPO_TRACE'
+
+try:
+  _TRACE = os.environ[REPO_TRACE] == '1'
+except KeyError:
+  _TRACE = False
+
+def IsTrace():
+  return _TRACE
+
+def SetTrace():
+  global _TRACE
+  _TRACE = True
+
+def Trace(fmt, *args):
+  if IsTrace():
+    print >>sys.stderr, fmt % args