Fix ssh://user@hostname/ style URLs parsing

I only tested this with ssh://hostname/ style URLs, so I failed
to test ssh://user@hostname/ format, which failed if the hostname
portion was longer than 1 character.

Signed-off-by: Shawn O. Pearce <sop@google.com>

color.py

@@ -100,6 +100,9 @@ class Coloring(object):
     else:
       self._on = False
 
+  def redirect(self, out):
+    self._out = out
+
   @property
   def is_on(self):
     return self._on
@@ -107,6 +110,9 @@ class Coloring(object):
   def write(self, fmt, *args):
     self._out.write(fmt % args)
 
+  def flush(self):
+    self._out.flush()
+
   def nl(self):
     self._out.write('\n')
 

command.py

@@ -27,6 +27,9 @@ class Command(object):
   manifest = None
   _optparse = None
 
+  def WantPager(self, opt):
+    return False
+
   @property
   def OptionParser(self):
     if self._optparse is None:
@@ -109,11 +112,15 @@ class InteractiveCommand(Command):
   """Command which requires user interaction on the tty and
      must not run within a pager, even if the user asks to.
   """
+  def WantPager(self, opt):
+    return False
 
 class PagedCommand(Command):
   """Command which defaults to output in a pager, as its
      display tends to be larger than one screen full.
   """
+  def WantPager(self, opt):
+    return True
 
 class MirrorSafeCommand(object):
   """Command permits itself to run within a mirror,

docs/manifest-format.txt

@@ -191,8 +191,3 @@ For example:
 Users may add projects to the local manifest prior to a `repo sync`
 invocation, instructing repo to automatically download and manage
 these extra projects.
-
-Currently the only supported feature of a local manifest is to
-add new remotes and/or projects.  In the future a local manifest
-may support picking different revisions of a project, or deleting
-projects specified in the default manifest.

editor.py

@@ -78,7 +78,11 @@ least one of these before using this command."""
 
       if subprocess.Popen(editor + [path]).wait() != 0:
         raise EditorError()
-      return open(path).read()
+      fd2 = open(path)
+      try:
+        return fd2.read()
+      finally:
+        fd2.close()
     finally:
       if fd:
         os.close(fd)

git_command.py

@@ -16,19 +16,40 @@
 import os
 import sys
 import subprocess
+import tempfile
 from error import GitError
+from trace import REPO_TRACE, IsTrace, Trace
 
 GIT = 'git'
 MIN_GIT_VERSION = (1, 5, 4)
 GIT_DIR = 'GIT_DIR'
-REPO_TRACE = 'REPO_TRACE'
 
 LAST_GITDIR = None
 LAST_CWD = None
-try:
+
-  TRACE = os.environ[REPO_TRACE] == '1'
+_ssh_proxy_path = None
-except KeyError:
+_ssh_sock_path = None
-  TRACE = False
+
+def _ssh_sock(create=True):
+  global _ssh_sock_path
+  if _ssh_sock_path is None:
+    if not create:
+      return None
+    dir = '/tmp'
+    if not os.path.exists(dir):
+      dir = tempfile.gettempdir()
+    _ssh_sock_path = os.path.join(
+      tempfile.mkdtemp('', 'ssh-', dir),
+      'master-%r@%h:%p')
+  return _ssh_sock_path
+
+def _ssh_proxy():
+  global _ssh_proxy_path
+  if _ssh_proxy_path is None:
+    _ssh_proxy_path = os.path.join(
+      os.path.dirname(__file__),
+      'git_ssh')
+  return _ssh_proxy_path
 
 
 class _GitCall(object):
@@ -56,6 +77,7 @@ class GitCommand(object):
                capture_stdout = False,
                capture_stderr = False,
                disable_editor = False,
+               ssh_proxy = False,
                cwd = None,
                gitdir = None):
     env = dict(os.environ)
@@ -72,6 +94,9 @@ class GitCommand(object):
 
     if disable_editor:
       env['GIT_EDITOR'] = ':'
+    if ssh_proxy:
+      env['REPO_SSH_SOCK'] = _ssh_sock()
+      env['GIT_SSH'] = _ssh_proxy()
 
     if project:
       if not cwd:
@@ -101,7 +126,7 @@ class GitCommand(object):
     else:
       stderr = None
 
-    if TRACE:
+    if IsTrace():
       global LAST_CWD
       global LAST_GITDIR
 
@@ -127,7 +152,7 @@ class GitCommand(object):
         dbg += ' 1>|'
       if stderr == subprocess.PIPE:
         dbg += ' 2>|'
-      print >>sys.stderr, dbg
+      Trace('%s', dbg)
 
     try:
       p = subprocess.Popen(command,

git_config.py

@@ -13,20 +13,34 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import cPickle
 import os
 import re
+import subprocess
 import sys
+import time
+from signal import SIGTERM
 from urllib2 import urlopen, HTTPError
 from error import GitError, UploadError
-from git_command import GitCommand
+from trace import Trace
+from git_command import GitCommand, _ssh_sock
 
 R_HEADS = 'refs/heads/'
 R_TAGS  = 'refs/tags/'
 ID_RE = re.compile('^[0-9a-f]{40}$')
 
+REVIEW_CACHE = dict()
+
 def IsId(rev):
   return ID_RE.match(rev)
 
+def _key(name):
+  parts = name.split('.')
+  if len(parts) < 2:
+    return name.lower()
+  parts[ 0] = parts[ 0].lower()
+  parts[-1] = parts[-1].lower()
+  return '.'.join(parts)
 
 class GitConfig(object):
   _ForUser = None
@@ -46,14 +60,17 @@ class GitConfig(object):
     self.file = file
     self.defaults = defaults
     self._cache_dict = None
+    self._section_dict = None
     self._remotes = {}
     self._branches = {}
+    self._pickle = os.path.join(
+      os.path.dirname(self.file),
+      '.repopickle_' + os.path.basename(self.file))
 
   def Has(self, name, include_defaults = True):
     """Return true if this configuration file has the key.
     """
-    name = name.lower()
+    if _key(name) in self._cache:
-    if name in self._cache:
       return True
     if include_defaults and self.defaults:
       return self.defaults.Has(name, include_defaults = True)
@@ -81,10 +98,8 @@ class GitConfig(object):
        This configuration file is used first, if the key is not
        defined or all = True then the defaults are also searched.
     """
-    name = name.lower()
-
     try:
-      v = self._cache[name]
+      v = self._cache[_key(name)]
     except KeyError:
       if self.defaults:
         return self.defaults.GetString(name, all = all)
@@ -108,16 +123,16 @@ class GitConfig(object):
        The supplied value should be either a string,
        or a list of strings (to store multiple values).
     """
-    name = name.lower()
+    key = _key(name)
 
     try:
-      old = self._cache[name]
+      old = self._cache[key]
     except KeyError:
       old = []
 
     if value is None:
       if old:
-        del self._cache[name]
+        del self._cache[key]
         self._do('--unset-all', name)
 
     elif isinstance(value, list):
@@ -128,13 +143,13 @@ class GitConfig(object):
         self.SetString(name, value[0])
 
       elif old != value:
-        self._cache[name] = list(value)
+        self._cache[key] = list(value)
         self._do('--replace-all', name, value[0])
         for i in xrange(1, len(value)):
           self._do('--add', name, value[i])
 
     elif len(old) != 1 or old[0] != value:
-      self._cache[name] = [value]
+      self._cache[key] = [value]
       self._do('--replace-all', name, value)
 
   def GetRemote(self, name):
@@ -157,6 +172,33 @@ class GitConfig(object):
       self._branches[b.name] = b
     return b
 
+  def HasSection(self, section, subsection = ''):
+    """Does at least one key in section.subsection exist?
+    """
+    try:
+      return subsection in self._sections[section]
+    except KeyError:
+      return False
+
+  @property
+  def _sections(self):
+    d = self._section_dict
+    if d is None:
+      d = {}
+      for name in self._cache.keys():
+        p = name.split('.')
+        if 2 == len(p):
+          section = p[0]
+          subsect = ''
+        else:
+          section = p[0]
+          subsect = '.'.join(p[1:-1])
+        if section not in d:
+          d[section] = set()
+        d[section].add(subsect)
+        self._section_dict = d
+    return d
+
   @property
   def _cache(self):
     if self._cache_dict is None:
@@ -164,13 +206,54 @@ class GitConfig(object):
     return self._cache_dict
 
   def _Read(self):
+    d = self._ReadPickle()
+    if d is None:
+      d = self._ReadGit()
+      self._SavePickle(d)
+    return d
+
+  def _ReadPickle(self):
+    try:
+      if os.path.getmtime(self._pickle) \
+      <= os.path.getmtime(self.file):
+        os.remove(self._pickle)
+        return None
+    except OSError:
+      return None
+    try:
+      Trace(': unpickle %s', self.file)
+      fd = open(self._pickle, 'rb')
+      try:
+        return cPickle.load(fd)
+      finally:
+        fd.close()
+    except IOError:
+      os.remove(self._pickle)
+      return None
+    except cPickle.PickleError:
+      os.remove(self._pickle)
+      return None
+
+  def _SavePickle(self, cache):
+    try:
+      fd = open(self._pickle, 'wb')
+      try:
+        cPickle.dump(cache, fd, cPickle.HIGHEST_PROTOCOL)
+      finally:
+        fd.close()
+    except IOError:
+      os.remove(self._pickle)
+    except cPickle.PickleError:
+      os.remove(self._pickle)
+
+  def _ReadGit(self):
     d = self._do('--null', '--list')
     c = {}
     while d:
       lf = d.index('\n')
       nul = d.index('\0', lf + 1)
 
-      key = d[0:lf]
+      key = _key(d[0:lf])
       val = d[lf + 1:nul]
 
       if key in c:
@@ -251,6 +334,78 @@ class RefSpec(object):
     return s
 
 
+_ssh_cache = {}
+_ssh_master = True
+
+def _open_ssh(host, port):
+  global _ssh_master
+
+  key = '%s:%s' % (host, port)
+  if key in _ssh_cache:
+    return True
+
+  if not _ssh_master \
+  or 'GIT_SSH' in os.environ \
+  or sys.platform in ('win32', 'cygwin'):
+    # failed earlier, or cygwin ssh can't do this
+    #
+    return False
+
+  command = ['ssh',
+             '-o','ControlPath %s' % _ssh_sock(),
+             '-p',str(port),
+             '-M',
+             '-N',
+             host]
+  try:
+    Trace(': %s', ' '.join(command))
+    p = subprocess.Popen(command)
+  except Exception, e:
+    _ssh_master = False
+    print >>sys.stderr, \
+      '\nwarn: cannot enable ssh control master for %s:%s\n%s' \
+      % (host,port, str(e))
+    return False
+
+  _ssh_cache[key] = p
+  time.sleep(1)
+  return True
+
+def close_ssh():
+  for key,p in _ssh_cache.iteritems():
+    os.kill(p.pid, SIGTERM)
+    p.wait()
+  _ssh_cache.clear()
+
+  d = _ssh_sock(create=False)
+  if d:
+    try:
+      os.rmdir(os.path.dirname(d))
+    except OSError:
+      pass
+
+URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
+URI_ALL = re.compile(r'^([a-z][a-z+]*)://([^@/]*@?[^/]*)/')
+
+def _preconnect(url):
+  m = URI_ALL.match(url)
+  if m:
+    scheme = m.group(1)
+    host = m.group(2)
+    if ':' in host:
+      host, port = host.split(':')
+    else:
+      port = 22
+    if scheme in ('ssh', 'git+ssh', 'ssh+git'):
+      return _open_ssh(host, port)
+    return False
+
+  m = URI_SCP.match(url)
+  if m:
+    host = m.group(1)
+    return _open_ssh(host, 22)
+
+
 class Remote(object):
   """Configuration options related to a remote.
   """
@@ -264,6 +419,9 @@ class Remote(object):
                      self._Get('fetch', all=True))
     self._review_protocol = None
 
+  def PreConnectFetch(self):
+    return _preconnect(self.url)
+
   @property
   def ReviewProtocol(self):
     if self._review_protocol is None:
@@ -273,24 +431,44 @@ class Remote(object):
       u = self.review
       if not u.startswith('http:') and not u.startswith('https:'):
         u = 'http://%s' % u
-      if not u.endswith('/'):
+      if u.endswith('/Gerrit'):
-        u += '/'
+        u = u[:len(u) - len('/Gerrit')]
-      u += 'ssh_info'
+      if not u.endswith('/ssh_info'):
+        if not u.endswith('/'):
+          u += '/'
+        u += 'ssh_info'
 
-      try:
+      if u in REVIEW_CACHE:
-        info = urlopen(u).read()
+        info = REVIEW_CACHE[u]
-        if info == 'NOT_AVAILABLE':
+        self._review_protocol = info[0]
-          raise UploadError('Upload over ssh unavailable')
+        self._review_host = info[1]
+        self._review_port = info[2]
+      else:
+        try:
+          info = urlopen(u).read()
+          if info == 'NOT_AVAILABLE':
+            raise UploadError('Upload over ssh unavailable')
+          if '<' in info:
+            # Assume the server gave us some sort of HTML
+            # response back, like maybe a login page.
+            #
+            raise UploadError('Cannot read %s:\n%s' % (u, info))
 
-        self._review_protocol = 'ssh'
+          self._review_protocol = 'ssh'
-        self._review_host = info.split(" ")[0]
+          self._review_host = info.split(" ")[0]
-        self._review_port = info.split(" ")[1]
+          self._review_port = info.split(" ")[1]
+        except HTTPError, e:
+          if e.code == 404:
+            self._review_protocol = 'http-post'
+            self._review_host = None
+            self._review_port = None
+          else:
+            raise UploadError('Cannot guess Gerrit version')
 
-      except HTTPError, e:
+        REVIEW_CACHE[u] = (
-        if e.code == 404:
+          self._review_protocol,
-          self._review_protocol = 'http-post'
+          self._review_host,
-        else:
+          self._review_port)
-          raise UploadError('Cannot guess Gerrit version')
     return self._review_protocol
 
   def SshReviewUrl(self, userEmail):
@@ -377,11 +555,23 @@ class Branch(object):
   def Save(self):
     """Save this branch back into the configuration.
     """
-    self._Set('merge', self.merge)
+    if self._config.HasSection('branch', self.name):
-    if self.remote:
+      if self.remote:
-      self._Set('remote', self.remote.name)
+        self._Set('remote', self.remote.name)
+      else:
+        self._Set('remote', None)
+      self._Set('merge', self.merge)
+
     else:
-      self._Set('remote', None)
+      fd = open(self._config.file, 'ab')
+      try:
+        fd.write('[branch "%s"]\n' % self.name)
+        if self.remote:
+          fd.write('\tremote = %s\n' % self.remote.name)
+        if self.merge:
+          fd.write('\tmerge = %s\n' % self.merge)
+      finally:
+        fd.close()
 
   def _Set(self, key, value):
     key = 'branch.%s.%s' % (self.name, key)

git_refs.py

@@ -0,0 +1,160 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+from trace import Trace
+
+HEAD    = 'HEAD'
+R_HEADS = 'refs/heads/'
+R_TAGS  = 'refs/tags/'
+R_PUB   = 'refs/published/'
+R_M     = 'refs/remotes/m/'
+
+
+class GitRefs(object):
+  def __init__(self, gitdir):
+    self._gitdir = gitdir
+    self._phyref = None
+    self._symref = None
+    self._mtime = {}
+
+  @property
+  def all(self):
+    self._EnsureLoaded()
+    return self._phyref
+
+  def get(self, name):
+    try:
+      return self.all[name]
+    except KeyError:
+      return ''
+
+  def deleted(self, name):
+    if self._phyref is not None:
+      if name in self._phyref:
+        del self._phyref[name]
+
+      if name in self._symref:
+        del self._symref[name]
+
+      if name in self._mtime:
+        del self._mtime[name]
+
+  def symref(self, name):
+    try:
+      self._EnsureLoaded()
+      return self._symref[name]
+    except KeyError:
+      return ''
+
+  def _EnsureLoaded(self):
+    if self._phyref is None or self._NeedUpdate():
+      self._LoadAll()
+
+  def _NeedUpdate(self):
+    Trace(': scan refs %s', self._gitdir)
+
+    for name, mtime in self._mtime.iteritems():
+      try:
+        if mtime != os.path.getmtime(os.path.join(self._gitdir, name)):
+          return True
+      except OSError:
+        return True
+    return False
+
+  def _LoadAll(self):
+    Trace(': load refs %s', self._gitdir)
+
+    self._phyref = {}
+    self._symref = {}
+    self._mtime = {}
+
+    self._ReadPackedRefs()
+    self._ReadLoose('refs/')
+    self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD)
+
+    scan = self._symref
+    attempts = 0
+    while scan and attempts < 5:
+      scan_next = {}
+      for name, dest in scan.iteritems():
+        if dest in self._phyref:
+          self._phyref[name] = self._phyref[dest]
+        else:
+          scan_next[name] = dest
+      scan = scan_next
+      attempts += 1
+
+  def _ReadPackedRefs(self):
+    path = os.path.join(self._gitdir, 'packed-refs')
+    try:
+      fd = open(path, 'rb')
+      mtime = os.path.getmtime(path)
+    except IOError:
+      return
+    except OSError:
+      return
+    try:
+      for line in fd:
+        if line[0] == '#':
+          continue
+        if line[0] == '^':
+          continue
+
+        line = line[:-1]
+        p = line.split(' ')
+        id = p[0]
+        name = p[1]
+
+        self._phyref[name] = id
+    finally:
+      fd.close()
+    self._mtime['packed-refs'] = mtime
+
+  def _ReadLoose(self, prefix):
+    base = os.path.join(self._gitdir, prefix)
+    for name in os.listdir(base):
+      p = os.path.join(base, name)
+      if os.path.isdir(p):
+        self._mtime[prefix] = os.path.getmtime(base)
+        self._ReadLoose(prefix + name + '/')
+      elif name.endswith('.lock'):
+        pass
+      else:
+        self._ReadLoose1(p, prefix + name)
+
+  def _ReadLoose1(self, path, name):
+    try:
+      fd = open(path, 'rb')
+      mtime = os.path.getmtime(path)
+    except OSError:
+      return
+    except IOError:
+      return
+    try:
+      id = fd.readline()
+    finally:
+      fd.close()
+
+    if not id:
+      return
+    id = id[:-1]
+
+    if id.startswith('ref: '):
+      self._symref[name] = id[5:]
+    else:
+      self._phyref[name] = id
+    self._mtime[name] = mtime

git_ssh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec ssh -o "ControlPath $REPO_SSH_SOCK" "$@"

main.py

@@ -27,6 +27,8 @@ import os
 import re
 import sys
 
+from trace import SetTrace
+from git_config import close_ssh
 from command import InteractiveCommand
 from command import MirrorSafeCommand
 from command import PagedCommand
@@ -48,6 +50,9 @@ global_options.add_option('-p', '--paginate',
 global_options.add_option('--no-pager',
                           dest='no_pager', action='store_true',
                           help='disable the pager')
+global_options.add_option('--trace',
+                          dest='trace', action='store_true',
+                          help='trace git command execution')
 global_options.add_option('--version',
                           dest='show_version', action='store_true',
                           help='display this version of repo')
@@ -74,6 +79,8 @@ class _Repo(object):
       argv = []
     gopts, gargs = global_options.parse_args(glob)
 
+    if gopts.trace:
+      SetTrace()
     if gopts.show_version:
       if name == 'help':
         name = 'version'
@@ -99,6 +106,8 @@ class _Repo(object):
             % name
       sys.exit(1)
 
+    copts, cargs = cmd.OptionParser.parse_args(argv)
+
     if not gopts.no_pager and not isinstance(cmd, InteractiveCommand):
       config = cmd.manifest.globalConfig
       if gopts.pager:
@@ -106,11 +115,10 @@ class _Repo(object):
       else:
         use_pager = config.GetBoolean('pager.%s' % name)
         if use_pager is None:
-          use_pager = isinstance(cmd, PagedCommand)
+          use_pager = cmd.WantPager(copts)
       if use_pager:
         RunPager(config)
 
-    copts, cargs = cmd.OptionParser.parse_args(argv)
     try:
       cmd.Execute(copts, cargs)
     except ManifestInvalidRevisionError, e:
@@ -205,7 +213,10 @@ def _Main(argv):
 
   repo = _Repo(opt.repodir)
   try:
-    repo._Run(argv)
+    try:
+      repo._Run(argv)
+    finally:
+      close_ssh()
   except KeyboardInterrupt:
     sys.exit(1)
   except RepoChangedException, rce:

pager.py

@@ -22,7 +22,7 @@ active = False
 def RunPager(globalConfig):
   global active
 
-  if not os.isatty(0):
+  if not os.isatty(0) or not os.isatty(1):
     return
   pager = _SelectPager(globalConfig)
   if pager == '' or pager == 'cat':

progress.py

@@ -0,0 +1,74 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from time import time
+from trace import IsTrace
+
+class Progress(object):
+  def __init__(self, title, total=0):
+    self._title = title
+    self._total = total
+    self._done = 0
+    self._lastp = -1
+    self._start = time()
+    self._show = False
+
+  def update(self, inc=1):
+    self._done += inc
+
+    if IsTrace():
+      return
+
+    if not self._show:
+      if 0.5 <= time() - self._start:
+        self._show = True
+      else:
+        return
+
+    if self._total <= 0:
+      sys.stderr.write('\r%s: %d, ' % (
+        self._title,
+        self._done))
+      sys.stderr.flush()
+    else:
+      p = (100 * self._done) / self._total
+
+      if self._lastp != p:
+        self._lastp = p
+        sys.stderr.write('\r%s: %3d%% (%d/%d)  ' % (
+          self._title,
+          p,
+          self._done,
+          self._total))
+        sys.stderr.flush()
+
+  def end(self):
+    if IsTrace() or not self._show:
+      return
+
+    if self._total <= 0:
+      sys.stderr.write('\r%s: %d, done.  \n' % (
+        self._title,
+        self._done))
+      sys.stderr.flush()
+    else:
+      p = (100 * self._done) / self._total
+      sys.stderr.write('\r%s: %3d%% (%d/%d), done.  \n' % (
+        self._title,
+        p,
+        self._done,
+        self._total))
+      sys.stderr.flush()

project.py

@@ -28,19 +28,26 @@ from error import GitError, ImportError, UploadError
 from error import ManifestInvalidRevisionError
 from remote import Remote
 
-HEAD    = 'HEAD'
+from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
-R_HEADS = 'refs/heads/'
-R_TAGS  = 'refs/tags/'
-R_PUB   = 'refs/published/'
-R_M     = 'refs/remotes/m/'
 
-def _warn(fmt, *args):
+def _lwrite(path, content):
-  msg = fmt % args
+  lock = '%s.lock' % path
-  print >>sys.stderr, 'warn: %s' % msg
 
-def _info(fmt, *args):
+  fd = open(lock, 'wb')
+  try:
+    fd.write(content)
+  finally:
+    fd.close()
+
+  try:
+    os.rename(lock, path)
+  except OSError:
+    os.remove(lock)
+    raise
+
+def _error(fmt, *args):
   msg = fmt % args
-  print >>sys.stderr, 'info: %s' % msg
+  print >>sys.stderr, 'error: %s' % msg
 
 def not_rev(r):
   return '^' + r
@@ -148,16 +155,6 @@ class ReviewableBranch(object):
                                  self.replace_changes,
                                  people)
 
-  @property
-  def tip_url(self):
-    me = self.project.GetBranch(self.name)
-    commit = self.project.bare_git.rev_parse(R_HEADS + self.name)
-    return 'http://%s/r/%s' % (me.remote.review, commit[0:12])
-
-  @property
-  def owner_email(self):
-    return self.project.UserEmail
-
 
 class StatusColoring(Coloring):
   def __init__(self, config):
@@ -165,6 +162,7 @@ class StatusColoring(Coloring):
     self.project   = self.printer('header',    attr = 'bold')
     self.branch    = self.printer('header',    attr = 'bold')
     self.nobranch  = self.printer('nobranch',  fg = 'red')
+    self.important = self.printer('important', fg = 'red')
 
     self.added     = self.printer('added',     fg = 'green')
     self.changed   = self.printer('changed',   fg = 'red')
@@ -199,9 +197,7 @@ class _CopyFile:
         mode = mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
         os.chmod(dest, mode)
       except IOError:
-           print >>sys.stderr, \
+        _error('Cannot copy file %s to %s', src, dest)
-              'error: Cannot copy file %s to %s' \
-              % (src, dest)
 
 
 class Project(object):
@@ -232,6 +228,7 @@ class Project(object):
     else:
       self.work_git = None
     self.bare_git = self._GitGetByExec(self, bare=True)
+    self.bare_ref = GitRefs(gitdir)
 
   @property
   def Exists(self):
@@ -243,14 +240,18 @@ class Project(object):
        The branch name omits the 'refs/heads/' prefix.
        None is returned if the project is on a detached HEAD.
     """
-    try:
+    b = self.work_git.GetHead()
-      b = self.work_git.GetHead()
-    except GitError:
-      return None
     if b.startswith(R_HEADS):
       return b[len(R_HEADS):]
     return None
 
+  def IsRebaseInProgress(self):
+    w = self.worktree
+    g = os.path.join(w, '.git')
+    return os.path.exists(os.path.join(g, 'rebase-apply')) \
+        or os.path.exists(os.path.join(g, 'rebase-merge')) \
+        or os.path.exists(os.path.join(w, '.dotest'))
+    
   def IsDirty(self, consider_untracked=True):
     """Is the working directory modified in some way?
     """
@@ -306,6 +307,32 @@ class Project(object):
     """
     return self.config.GetBranch(name)
 
+  def GetBranches(self):
+    """Get all existing local branches.
+    """
+    current = self.CurrentBranch
+    all = self._allrefs
+    heads = {}
+    pubd = {}
+
+    for name, id in all.iteritems():
+      if name.startswith(R_HEADS):
+        name = name[len(R_HEADS):]
+        b = self.GetBranch(name)
+        b.current = name == current
+        b.published = None
+        b.revision = id
+        heads[name] = b
+
+    for name, id in all.iteritems():
+      if name.startswith(R_PUB):
+        name = name[len(R_PUB):]
+        b = heads.get(name)
+        if b:
+          b.published = id
+
+    return heads
+
 
 ## Status Display ##
 
@@ -322,11 +349,12 @@ class Project(object):
                                '--unmerged',
                                '--ignore-missing',
                                '--refresh')
+    rb = self.IsRebaseInProgress()
     di = self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD)
     df = self.work_git.DiffZ('diff-files')
     do = self.work_git.LsOthers()
-    if not di and not df and not do:
+    if not rb and not di and not df and not do:
-      return
+      return 'CLEAN'
 
     out = StatusColoring(self.config)
     out.project('project %-40s', self.relpath + '/')
@@ -338,6 +366,10 @@ class Project(object):
       out.branch('branch %s', branch)
     out.nl()
 
+    if rb:
+      out.important('prior sync failed; rebase still in progress')
+      out.nl()
+
     paths = list()
     paths.extend(di.keys())
     paths.extend(df.keys())
@@ -374,6 +406,7 @@ class Project(object):
       else:
         out.write('%s', line)
       out.nl()
+    return 'DIRTY'
 
   def PrintWorkTreeDiff(self):
     """Prints the status of the repository to stdout.
@@ -401,22 +434,31 @@ class Project(object):
 
 ## Publish / Upload ##
 
-  def WasPublished(self, branch):
+  def WasPublished(self, branch, all=None):
     """Was the branch published (uploaded) for code review?
        If so, returns the SHA-1 hash of the last published
        state for the branch.
     """
-    try:
+    key = R_PUB + branch
-      return self.bare_git.rev_parse(R_PUB + branch)
+    if all is None:
-    except GitError:
+      try:
-      return None
+        return self.bare_git.rev_parse(key)
+      except GitError:
+        return None
+    else:
+      try:
+        return all[key]
+      except KeyError:
+        return None
 
-  def CleanPublishedCache(self):
+  def CleanPublishedCache(self, all=None):
     """Prunes any stale published refs.
     """
+    if all is None:
+      all = self._allrefs
     heads = set()
     canrm = {}
-    for name, id in self._allrefs.iteritems():
+    for name, id in all.iteritems():
       if name.startswith(R_HEADS):
         heads.add(name)
       elif name.startswith(R_PUB):
@@ -547,40 +589,65 @@ class Project(object):
     for file in self.copyfiles:
       file._Copy()
 
-  def Sync_LocalHalf(self):
+  def Sync_LocalHalf(self, syncbuf):
     """Perform only the local IO portion of the sync process.
        Network access is not required.
-
-       Return:
-         True:  the sync was successful
-         False: the sync requires user input
     """
     self._InitWorkTree()
-    self.CleanPublishedCache()
+    all = self.bare_ref.all
+    self.CleanPublishedCache(all)
 
     rem = self.GetRemote(self.remote.name)
     rev = rem.ToLocal(self.revision)
-    try:
+    if rev in all:
-      self.bare_git.rev_parse('--verify', '%s^0' % rev)
+      revid = all[rev]
-    except GitError:
+    elif IsId(rev):
-      raise ManifestInvalidRevisionError(
+      revid = rev
-        'revision %s in %s not found' % (self.revision, self.name))
+    else:
+      try:
+        revid = self.bare_git.rev_parse('--verify', '%s^0' % rev)
+      except GitError:
+        raise ManifestInvalidRevisionError(
+          'revision %s in %s not found' % (self.revision, self.name))
 
-    branch = self.CurrentBranch
+    head = self.work_git.GetHead()
+    if head.startswith(R_HEADS):
+      branch = head[len(R_HEADS):]
+      try:
+        head = all[head]
+      except KeyError:
+        head = None
+    else:
+      branch = None
 
-    if branch is None:
+    if branch is None or syncbuf.detach_head:
       # Currently on a detached HEAD.  The user is assumed to
       # not have any local modifications worth worrying about.
       #
+      if self.IsRebaseInProgress():
+        syncbuf.fail(self, _PriorSyncFailedError())
+        return
+
+      if head == revid:
+        # No changes; don't do anything further.
+        #
+        return
+
       lost = self._revlist(not_rev(rev), HEAD)
       if lost:
-        _info("[%s] Discarding %d commits", self.name, len(lost))
+        syncbuf.info(self, "discarding %d commits", len(lost))
       try:
         self._Checkout(rev, quiet=True)
-      except GitError:
+      except GitError, e:
-        return False
+        syncbuf.fail(self, e)
+        return
       self._CopyFiles()
-      return True
+      return
+
+    if head == revid:
+      # No changes; don't do anything further.
+      #
+      return
 
     branch = self.GetBranch(branch)
     merge = branch.LocalMerge
@@ -589,19 +656,19 @@ class Project(object):
       # The current branch has no tracking configuration.
       # Jump off it to a deatched HEAD.
       #
-      _info("[%s] Leaving %s"
+      syncbuf.info(self,
-            " (does not track any upstream)",
+                   "leaving %s; does not track upstream",
-            self.name,
+                   branch.name)
-            branch.name)
       try:
         self._Checkout(rev, quiet=True)
-      except GitError:
+      except GitError, e:
-        return False
+        syncbuf.fail(self, e)
+        return
       self._CopyFiles()
-      return True
+      return
 
     upstream_gain = self._revlist(not_rev(HEAD), rev)
-    pub = self.WasPublished(branch.name)
+    pub = self.WasPublished(branch.name, all)
     if pub:
       not_merged = self._revlist(not_rev(rev), pub)
       if not_merged:
@@ -610,25 +677,20 @@ class Project(object):
           # commits are not yet merged upstream.  We do not want
           # to rewrite the published commits so we punt.
           #
-          _info("[%s] Branch %s is published,"
+          syncbuf.info(self,
-                " but is now %d commits behind.",
+                       "branch %s is published but is now %d commits behind",
-                self.name, branch.name, len(upstream_gain))
+                       branch.name,
-          _info("[%s] Consider merging or rebasing the"
+                       len(upstream_gain))
-                " unpublished commits.", self.name)
+        return
-        return True
+      elif pub == head:
-      elif upstream_gain:
+        # All published commits are merged, and thus we are a
-        # We can fast-forward safely.
+        # strict subset.  We can fast-forward safely.
         #
-        try:
+        def _doff():
           self._FastForward(rev)
-        except GitError:
+          self._CopyFiles()
-          return False
+        syncbuf.later1(self, _doff)
-        self._CopyFiles()
+        return
-        return True
-      else:
-        # Trivially no changes in the upstream.
-        #
-        return True
 
     if merge == rev:
       try:
@@ -643,8 +705,7 @@ class Project(object):
       # and pray that the old upstream also wasn't in the habit
       # of rebasing itself.
       #
-      _info("[%s] Manifest switched from %s to %s",
+      syncbuf.info(self, "manifest switched %s...%s", merge, rev)
-            self.name, merge, rev)
       old_merge = merge
 
     if rev == old_merge:
@@ -655,19 +716,19 @@ class Project(object):
     if not upstream_lost and not upstream_gain:
       # Trivially no changes caused by the upstream.
       #
-      return True
+      return
 
     if self.IsDirty(consider_untracked=False):
-      _warn('[%s] commit (or discard) uncommitted changes'
+      syncbuf.fail(self, _DirtyError())
-            ' before sync', self.name)
+      return
-      return False
 
     if upstream_lost:
       # Upstream rebased.  Not everything in HEAD
       # may have been caused by the user.
       #
-      _info("[%s] Discarding %d commits removed from upstream",
+      syncbuf.info(self,
-            self.name, len(upstream_lost))
+                   "discarding %d commits removed from upstream",
+                   len(upstream_lost))
 
     branch.remote = rem
     branch.merge = self.revision
@@ -675,23 +736,22 @@ class Project(object):
 
     my_changes = self._revlist(not_rev(old_merge), HEAD)
     if my_changes:
-      try:
+      def _dorebase():
         self._Rebase(upstream = old_merge, onto = rev)
-      except GitError:
+        self._CopyFiles()
-        return False
+      syncbuf.later2(self, _dorebase)
     elif upstream_lost:
       try:
         self._ResetHard(rev)
-      except GitError:
+        self._CopyFiles()
-        return False
+      except GitError, e:
+        syncbuf.fail(self, e)
+        return
     else:
-      try:
+      def _doff():
         self._FastForward(rev)
-      except GitError:
+        self._CopyFiles()
-        return False
+      syncbuf.later1(self, _doff)
-
-    self._CopyFiles()
-    return True
 
   def AddCopyFile(self, src, dest, absdest):
     # dest should already be an absolute path, but src is project relative
@@ -722,32 +782,127 @@ class Project(object):
   def StartBranch(self, name):
     """Create a new branch off the manifest's revision.
     """
+    head = self.work_git.GetHead()
+    if head == (R_HEADS + name):
+      return True
+
+    all = self.bare_ref.all
+    if (R_HEADS + name) in all:
+      return GitCommand(self,
+                        ['checkout', name, '--'],
+                        capture_stdout = True,
+                        capture_stderr = True).Wait() == 0
+
     branch = self.GetBranch(name)
     branch.remote = self.GetRemote(self.remote.name)
     branch.merge = self.revision
 
     rev = branch.LocalMerge
-    cmd = ['checkout', '-b', branch.name, rev]
+    if rev in all:
-    if GitCommand(self, cmd).Wait() == 0:
+      revid = all[rev]
-      branch.Save()
+    elif IsId(rev):
+      revid = rev
     else:
-      raise GitError('%s checkout %s ' % (self.name, rev))
+      revid = None
+
+    if head.startswith(R_HEADS):
+      try:
+        head = all[head]
+      except KeyError:
+        head = None
+
+    if revid and head and revid == head:
+      ref = os.path.join(self.gitdir, R_HEADS + name)
+      try:
+        os.makedirs(os.path.dirname(ref))
+      except OSError:
+        pass
+      _lwrite(ref, '%s\n' % revid)
+      _lwrite(os.path.join(self.worktree, '.git', HEAD),
+              'ref: %s%s\n' % (R_HEADS, name))
+      branch.Save()
+      return True
+
+    if GitCommand(self,
+                  ['checkout', '-b', branch.name, rev],
+                  capture_stdout = True,
+                  capture_stderr = True).Wait() == 0:
+      branch.Save()
+      return True
+    return False
+
+  def CheckoutBranch(self, name):
+    """Checkout a local topic branch.
+    """
+    rev = R_HEADS + name
+    head = self.work_git.GetHead()
+    if head == rev:
+      # Already on the branch
+      #
+      return True
+
+    all = self.bare_ref.all
+    try:
+      revid = all[rev]
+    except KeyError:
+      # Branch does not exist in this project
+      #
+      return False
+
+    if head.startswith(R_HEADS):
+      try:
+        head = all[head]
+      except KeyError:
+        head = None
+
+    if head == revid:
+      # Same revision; just update HEAD to point to the new
+      # target branch, but otherwise take no other action.
+      #
+      _lwrite(os.path.join(self.worktree, '.git', HEAD),
+              'ref: %s%s\n' % (R_HEADS, name))
+      return True
+
+    return GitCommand(self,
+                      ['checkout', name, '--'],
+                      capture_stdout = True,
+                      capture_stderr = True).Wait() == 0
 
   def AbandonBranch(self, name):
     """Destroy a local topic branch.
     """
-    try:
+    rev = R_HEADS + name
-      tip_rev = self.bare_git.rev_parse(R_HEADS + name)
+    all = self.bare_ref.all
-    except GitError:
+    if rev not in all:
-      return
+      # Doesn't exist; assume already abandoned.
+      #
+      return True
 
-    if self.CurrentBranch == name:
+    head = self.work_git.GetHead()
-      self._Checkout(
+    if head == rev:
-        self.GetRemote(self.remote.name).ToLocal(self.revision),
+      # We can't destroy the branch while we are sitting
-        quiet=True)
+      # on it.  Switch to a detached HEAD.
+      #
+      head = all[head]
 
-    cmd = ['branch', '-D', name]
+      rev = self.GetRemote(self.remote.name).ToLocal(self.revision)
-    GitCommand(self, cmd, capture_stdout=True).Wait()
+      if rev in all:
+        revid = all[rev]
+      elif IsId(rev):
+        revid = rev
+      else:
+        revid = None
+
+      if revid and head == revid:
+        _lwrite(os.path.join(self.worktree, '.git', HEAD),
+                '%s\n' % revid)
+      else:
+        self._Checkout(rev, quiet=True)
+
+    return GitCommand(self,
+                      ['branch', '-D', name],
+                      capture_stdout = True,
+                      capture_stderr = True).Wait() == 0
 
   def PruneHeads(self):
     """Prune any topic branches already merged into upstream.
@@ -769,9 +924,8 @@ class Project(object):
       kill.append(cb)
 
     if kill:
-      try:
+      old = self.bare_git.GetHead()
-        old = self.bare_git.GetHead()
+      if old is None:
-      except GitError:
         old = 'refs/heads/please_never_use_this_as_a_branch_name'
 
       try:
@@ -812,11 +966,19 @@ class Project(object):
   def _RemoteFetch(self, name=None):
     if not name:
       name = self.remote.name
+
+    ssh_proxy = False
+    if self.GetRemote(name).PreConnectFetch():
+      ssh_proxy = True
+
     cmd = ['fetch']
     if not self.worktree:
       cmd.append('--update-head-ok')
     cmd.append(name)
-    return GitCommand(self, cmd, bare = True).Wait() == 0
+    return GitCommand(self,
+                      cmd,
+                      bare = True,
+                      ssh_proxy = ssh_proxy).Wait() == 0
 
   def _Checkout(self, rev, quiet=False):
     cmd = ['checkout']
@@ -837,11 +999,11 @@ class Project(object):
       raise GitError('%s reset --hard %s ' % (self.name, rev))
 
   def _Rebase(self, upstream, onto = None):
-    cmd = ['rebase', '-i']
+    cmd = ['rebase']
     if onto is not None:
       cmd.extend(['--onto', onto])
     cmd.append(upstream)
-    if GitCommand(self, cmd, disable_editor=True).Wait() != 0:
+    if GitCommand(self, cmd).Wait() != 0:
       raise GitError('%s rebase %s ' % (self.name, upstream))
 
   def _FastForward(self, head):
@@ -923,14 +1085,17 @@ class Project(object):
     if self.manifest.branch:
       msg = 'manifest set to %s' % self.revision
       ref = R_M + self.manifest.branch
+      cur = self.bare_ref.symref(ref)
 
       if IsId(self.revision):
-        dst = self.revision + '^0'
+        if cur != '' or self.bare_ref.get(ref) != self.revision:
-        self.bare_git.UpdateRef(ref, dst, message = msg, detach = True)
+          dst = self.revision + '^0'
+          self.bare_git.UpdateRef(ref, dst, message = msg, detach = True)
       else:
         remote = self.GetRemote(self.remote.name)
         dst = remote.ToLocal(self.revision)
-        self.bare_git.symbolic_ref('-m', msg, ref, dst)
+        if cur != dst:
+          self.bare_git.symbolic_ref('-m', msg, ref, dst)
 
   def _InitMirrorHead(self):
     dst = self.GetRemote(self.remote.name).ToLocal(self.revision)
@@ -965,15 +1130,14 @@ class Project(object):
       rev = self.GetRemote(self.remote.name).ToLocal(self.revision)
       rev = self.bare_git.rev_parse('%s^0' % rev)
 
-      f = open(os.path.join(dotgit, HEAD), 'wb')
+      _lwrite(os.path.join(dotgit, HEAD), '%s\n' % rev)
-      f.write("%s\n" % rev)
-      f.close()
 
       cmd = ['read-tree', '--reset', '-u']
       cmd.append('-v')
       cmd.append('HEAD')
       if GitCommand(self, cmd).Wait() != 0:
         raise GitError("cannot initialize work tree")
+      self._CopyFiles()
 
   def _gitdir_path(self, path):
     return os.path.join(self.gitdir, path)
@@ -986,32 +1150,13 @@ class Project(object):
 
   @property
   def _allrefs(self):
-    return self.bare_git.ListRefs()
+    return self.bare_ref.all
 
   class _GitGetByExec(object):
     def __init__(self, project, bare):
       self._project = project
       self._bare = bare
 
-    def ListRefs(self, *args):
-      cmdv = ['for-each-ref', '--format=%(objectname) %(refname)']
-      cmdv.extend(args)
-      p = GitCommand(self._project,
-                     cmdv,
-                     bare = self._bare,
-                     capture_stdout = True,
-                     capture_stderr = True)
-      r = {}
-      for line in p.process.stdout:
-        id, name = line[:-1].split(' ', 2)
-        r[name] = id
-      if p.Wait() != 0:
-        raise GitError('%s for-each-ref %s: %s' % (
-                       self._project.name,
-                       str(args),
-                       p.stderr))
-      return r
-
     def LsOthers(self):
       p = GitCommand(self._project,
                      ['ls-files',
@@ -1077,7 +1222,18 @@ class Project(object):
         p.Wait()
 
     def GetHead(self):
-      return self.symbolic_ref(HEAD)
+      if self._bare:
+        path = os.path.join(self._project.gitdir, HEAD)
+      else:
+        path = os.path.join(self._project.worktree, '.git', HEAD)
+      fd = open(path, 'rb')
+      try:
+        line = fd.read()
+      finally:
+        fd.close()
+      if line.startswith('ref: '):
+        return line[5:-1]
+      return line[:-1]
 
     def SetHead(self, ref, message=None):
       cmdv = []
@@ -1113,6 +1269,7 @@ class Project(object):
       if not old:
         old = self.rev_parse(name)
       self.update_ref('-d', name, old)
+      self._project.bare_ref.deleted(name)
 
     def rev_list(self, *args):
       cmdv = ['rev-list']
@@ -1154,6 +1311,113 @@ class Project(object):
       return runner
 
 
+class _PriorSyncFailedError(Exception):
+  def __str__(self):
+    return 'prior sync failed; rebase still in progress'
+
+class _DirtyError(Exception):
+  def __str__(self):
+    return 'contains uncommitted changes'
+
+class _InfoMessage(object):
+  def __init__(self, project, text):
+    self.project = project
+    self.text = text
+
+  def Print(self, syncbuf):
+    syncbuf.out.info('%s/: %s', self.project.relpath, self.text)
+    syncbuf.out.nl()
+
+class _Failure(object):
+  def __init__(self, project, why):
+    self.project = project
+    self.why = why
+
+  def Print(self, syncbuf):
+    syncbuf.out.fail('error: %s/: %s',
+                     self.project.relpath,
+                     str(self.why))
+    syncbuf.out.nl()
+
+class _Later(object):
+  def __init__(self, project, action):
+    self.project = project
+    self.action = action
+
+  def Run(self, syncbuf):
+    out = syncbuf.out
+    out.project('project %s/', self.project.relpath)
+    out.nl()
+    try:
+      self.action()
+      out.nl()
+      return True
+    except GitError, e:
+      out.nl()
+      return False
+
+class _SyncColoring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, 'reposync')
+    self.project   = self.printer('header', attr = 'bold')
+    self.info      = self.printer('info')
+    self.fail      = self.printer('fail', fg='red')
+
+class SyncBuffer(object):
+  def __init__(self, config, detach_head=False):
+    self._messages = []
+    self._failures = []
+    self._later_queue1 = []
+    self._later_queue2 = []
+
+    self.out = _SyncColoring(config)
+    self.out.redirect(sys.stderr)
+
+    self.detach_head = detach_head
+    self.clean = True
+
+  def info(self, project, fmt, *args):
+    self._messages.append(_InfoMessage(project, fmt % args))
+
+  def fail(self, project, err=None):
+    self._failures.append(_Failure(project, err))
+    self.clean = False
+
+  def later1(self, project, what):
+    self._later_queue1.append(_Later(project, what))
+
+  def later2(self, project, what):
+    self._later_queue2.append(_Later(project, what))
+
+  def Finish(self):
+    self._PrintMessages()
+    self._RunLater()
+    self._PrintMessages()
+    return self.clean
+
+  def _RunLater(self):
+    for q in ['_later_queue1', '_later_queue2']:
+      if not self._RunQueue(q):
+        return
+
+  def _RunQueue(self, queue):
+    for m in getattr(self, queue):
+      if not m.Run(self):
+        self.clean = False
+        return False
+    setattr(self, queue, [])
+    return True
+
+  def _PrintMessages(self):
+    for m in self._messages:
+      m.Print(self)
+    for m in self._failures:
+      m.Print(self)
+
+    self._messages = []
+    self._failures = []
+
+
 class MetaProject(Project):
   """A special project housed under .repo.
   """
@@ -1176,11 +1440,37 @@ class MetaProject(Project):
         if base:
           self.revision = base
 
+  @property
+  def LastFetch(self):
+    try:
+      fh = os.path.join(self.gitdir, 'FETCH_HEAD')
+      return os.path.getmtime(fh)
+    except OSError:
+      return 0
+
   @property
   def HasChanges(self):
     """Has the remote received new commits not yet checked out?
     """
+    if not self.remote or not self.revision:
+      return False
+
+    all = self.bare_ref.all
     rev = self.GetRemote(self.remote.name).ToLocal(self.revision)
-    if self._revlist(not_rev(HEAD), rev):
+    if rev in all:
+      revid = all[rev]
+    else:
+      revid = rev
+
+    head = self.work_git.GetHead()
+    if head.startswith(R_HEADS):
+      try:
+        head = all[head]
+      except KeyError:
+        head = None
+
+    if revid == head:
+      return False
+    elif self._revlist(not_rev(HEAD), rev):
       return True
     return False

repo

@@ -120,7 +120,7 @@ group.add_option('--mirror',
                  help='mirror the forrest')
 
 # Tool
-group = init_optparse.add_option_group('Version options')
+group = init_optparse.add_option_group('repo Version options')
 group.add_option('--repo-url',
                  dest='repo_url',
                  help='repo repository location', metavar='URL')

subcmds/abandon.py

@@ -16,6 +16,7 @@
 import sys
 from command import Command
 from git_command import git
+from progress import Progress
 
 class Abandon(Command):
   common = True
@@ -38,5 +39,23 @@ It is equivalent to "git branch -D <branchname>".
       print >>sys.stderr, "error: '%s' is not a valid name" % nb
       sys.exit(1)
 
-    for project in self.GetProjects(args[1:]):
+    nb = args[0]
-      project.AbandonBranch(nb)
+    err = []
+    all = self.GetProjects(args[1:])
+
+    pm = Progress('Abandon %s' % nb, len(all))
+    for project in all:
+      pm.update()
+      if not project.AbandonBranch(nb):
+        err.append(project)
+    pm.end()
+
+    if err:
+      if len(err) == len(all):
+        print >>sys.stderr, 'error: no project has branch %s' % nb
+      else:
+        for p in err:
+          print >>sys.stderr,\
+            "error: %s/: cannot abandon %s" \
+            % (p.relpath, nb)
+      sys.exit(1)

subcmds/branches.py

@@ -0,0 +1,154 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from color import Coloring
+from command import Command
+
+class BranchColoring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, 'branch')
+    self.current = self.printer('current', fg='green')
+    self.local   = self.printer('local')
+    self.notinproject = self.printer('notinproject', fg='red')
+
+class BranchInfo(object):
+  def __init__(self, name):
+    self.name = name
+    self.current = 0
+    self.published = 0
+    self.published_equal = 0
+    self.projects = []
+
+  def add(self, b):
+    if b.current:
+      self.current += 1
+    if b.published:
+      self.published += 1
+    if b.revision == b.published:
+      self.published_equal += 1
+    self.projects.append(b)
+
+  @property
+  def IsCurrent(self):
+    return self.current > 0
+
+  @property
+  def IsPublished(self):
+    return self.published > 0
+
+  @property
+  def IsPublishedEqual(self):
+    return self.published_equal == len(self.projects)
+
+
+class Branches(Command):
+  common = True
+  helpSummary = "View current topic branches"
+  helpUsage = """
+%prog [<project>...]
+
+Summarizes the currently available topic branches.
+"""
+
+  def _Options(self, p):
+    p.add_option('-a', '--all',
+                 dest='all', action='store_true',
+                 help='show all branches, not just the majority')
+
+  def Execute(self, opt, args):
+    projects = self.GetProjects(args)
+    out = BranchColoring(self.manifest.manifestProject.config)
+    all = {}
+    project_cnt = len(projects)
+
+    for project in projects:
+      for name, b in project.GetBranches().iteritems():
+        b.project = project
+        if name not in all:
+          all[name] = BranchInfo(name)
+        all[name].add(b)
+
+    names = all.keys()
+    names.sort()
+
+    if not opt.all and not args:
+      # No -a and no specific projects listed; try to filter the
+      # results down to only the majority of projects.
+      #
+      n = []
+      for name in names:
+        i = all[name]
+        if i.IsCurrent \
+        or 80 <= (100 * len(i.projects)) / project_cnt:
+          n.append(name)
+      names = n
+
+    if not names:
+      print >>sys.stderr, '   (no branches)'
+      return
+
+    width = 25
+    for name in names:
+      if width < len(name):
+        width = len(name)
+
+    for name in names:
+      i = all[name]
+      in_cnt = len(i.projects)
+
+      if i.IsCurrent:
+        current = '*'
+        hdr = out.current
+      else:
+        current = ' '
+        hdr = out.local
+
+      if i.IsPublishedEqual:
+        published = 'P'
+      elif i.IsPublished:
+        published = 'p'
+      else:
+        published = ' '
+
+      hdr('%c%c %-*s' % (current, published, width, name))
+      out.write(' |')
+
+      if in_cnt < project_cnt and (in_cnt == 1 or opt.all):
+        fmt = out.write
+        paths = []
+        if in_cnt < project_cnt - in_cnt: 
+          type = 'in'
+          for b in i.projects:
+            paths.append(b.project.relpath)
+        else:
+          fmt = out.notinproject
+          type = 'not in'
+          have = set()
+          for b in i.projects:
+            have.add(b.project)
+          for p in projects:
+            paths.append(p.relpath)
+
+        s = ' %s %s' % (type, ', '.join(paths))
+        if width + 7 + len(s) < 80:
+          fmt(s)
+        else:
+          out.nl()
+          fmt('    %s:' % type)
+          for p in paths:
+            out.nl()
+            fmt('      %s' % p)
+      out.nl()

subcmds/checkout.py

@@ -0,0 +1,58 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from command import Command
+from progress import Progress
+
+class Checkout(Command):
+  common = True
+  helpSummary = "Checkout a branch for development"
+  helpUsage = """
+%prog <branchname> [<project>...]
+"""
+  helpDescription = """
+The '%prog' command checks out an existing branch that was previously
+created by 'repo start'.
+
+The command is equivalent to:
+
+  repo forall [<project>...] -c git checkout <branchname>
+"""
+
+  def Execute(self, opt, args):
+    if not args:
+      self.Usage()
+
+    nb = args[0]
+    err = []
+    all = self.GetProjects(args[1:])
+
+    pm = Progress('Checkout %s' % nb, len(all))
+    for project in all:
+      pm.update()
+      if not project.CheckoutBranch(nb):
+        err.append(project)
+    pm.end()
+
+    if err:
+      if len(err) == len(all):
+        print >>sys.stderr, 'error: no project has branch %s' % nb
+      else:
+        for p in err:
+          print >>sys.stderr,\
+            "error: %s/: cannot checkout %s" \
+            % (p.relpath, nb)
+      sys.exit(1)

subcmds/forall.py

@@ -13,12 +13,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import fcntl
 import re
 import os
+import select
 import sys
 import subprocess
+
+from color import Coloring
 from command import Command, MirrorSafeCommand
 
+_CAN_COLOR = [
+  'branch',
+  'diff',
+  'grep',
+  'log',
+]
+
+class ForallColoring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, 'forall')
+    self.project = self.printer('project', attr='bold')
+
+
 class Forall(Command, MirrorSafeCommand):
   common = False
   helpSummary = "Run a shell command in each project"
@@ -28,8 +45,27 @@ class Forall(Command, MirrorSafeCommand):
   helpDescription = """
 Executes the same shell command in each project.
 
+Output Formatting
+-----------------
+
+The -p option causes '%prog' to bind pipes to the command's stdin,
+stdout and stderr streams, and pipe all output into a continuous
+stream that is displayed in a single pager session.  Project headings
+are inserted before the output of each command is displayed.  If the
+command produces no output in a project, no heading is displayed.
+
+The formatting convention used by -p is very suitable for some
+types of searching, e.g. `repo forall -p -c git log -SFoo` will
+print all commits that add or remove references to Foo.
+
+The -v option causes '%prog' to display stderr messages if a
+command produces output only on stderr.  Normally the -p option
+causes command output to be suppressed until the command produces
+at least one byte of output on stdout.
+
 Environment
 -----------
+
 pwd is the project's working directory.  If the current client is
 a mirror client, then pwd is the Git repository.
 
@@ -49,8 +85,8 @@ as written in the manifest.
 shell positional arguments ($1, $2, .., $#) are set to any arguments
 following <command>.
 
-stdin, stdout, stderr are inherited from the terminal and are
+Unless -p is used, stdin, stdout, stderr are inherited from the
-not redirected.
+terminal and are not redirected.
 """
 
   def _Options(self, p):
@@ -64,6 +100,17 @@ not redirected.
                  action='callback',
                  callback=cmd)
 
+    g = p.add_option_group('Output')
+    g.add_option('-p',
+                 dest='project_header', action='store_true',
+                 help='Show project headers before output')
+    g.add_option('-v', '--verbose',
+                 dest='verbose', action='store_true',
+                 help='Show command error messages')
+
+  def WantPager(self, opt):
+    return opt.project_header
+
   def Execute(self, opt, args):
     if not opt.command:
       self.Usage()
@@ -78,28 +125,122 @@ not redirected.
       cmd.append(cmd[0])
     cmd.extend(opt.command[1:])
 
+    if  opt.project_header \
+    and not shell \
+    and cmd[0] == 'git':
+      # If this is a direct git command that can enable colorized
+      # output and the user prefers coloring, add --color into the
+      # command line because we are going to wrap the command into
+      # a pipe and git won't know coloring should activate.
+      #
+      for cn in cmd[1:]:
+        if not cn.startswith('-'):
+          break
+      if cn in _CAN_COLOR:
+        class ColorCmd(Coloring):
+          def __init__(self, config, cmd):
+            Coloring.__init__(self, config, cmd)
+        if ColorCmd(self.manifest.manifestProject.config, cn).is_on:
+          cmd.insert(cmd.index(cn) + 1, '--color')
+
     mirror = self.manifest.IsMirror
+    out = ForallColoring(self.manifest.manifestProject.config)
+    out.redirect(sys.stdout)
+
     rc = 0
+    first = True
+
     for project in self.GetProjects(args):
       env = dict(os.environ.iteritems())
-      env['REPO_PROJECT'] = project.name
+      def setenv(name, val):
-      env['REPO_PATH'] = project.relpath
+        if val is None:
-      env['REPO_REMOTE'] = project.remote.name
+          val = ''
-      env['REPO_LREV'] = project\
+        env[name] = val
+
+      setenv('REPO_PROJECT', project.name)
+      setenv('REPO_PATH', project.relpath)
+      setenv('REPO_REMOTE', project.remote.name)
+      setenv('REPO_LREV', project\
         .GetRemote(project.remote.name)\
-        .ToLocal(project.revision)
+        .ToLocal(project.revision))
-      env['REPO_RREV'] = project.revision
+      setenv('REPO_RREV', project.revision)
 
       if mirror:
-        env['GIT_DIR'] = project.gitdir
+        setenv('GIT_DIR', project.gitdir)
         cwd = project.gitdir
       else:
         cwd = project.worktree
 
+      if opt.project_header:
+        stdin = subprocess.PIPE
+        stdout = subprocess.PIPE
+        stderr = subprocess.PIPE
+      else:
+        stdin = None
+        stdout = None
+        stderr = None
+
       p = subprocess.Popen(cmd,
                            cwd = cwd,
                            shell = shell,
-                           env = env)
+                           env = env,
+                           stdin = stdin,
+                           stdout = stdout,
+                           stderr = stderr)
+
+      if opt.project_header:
+        class sfd(object):
+          def __init__(self, fd, dest):
+            self.fd = fd
+            self.dest = dest
+          def fileno(self):
+            return self.fd.fileno()
+
+        empty = True
+        didout = False
+        errbuf = ''
+
+        p.stdin.close()
+        s_in = [sfd(p.stdout, sys.stdout),
+                sfd(p.stderr, sys.stderr)]
+
+        for s in s_in:
+          flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
+          fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+        while s_in:
+          in_ready, out_ready, err_ready = select.select(s_in, [], [])
+          for s in in_ready:
+            buf = s.fd.read(4096)
+            if not buf:
+              s.fd.close()
+              s_in.remove(s)
+              continue
+
+            if not opt.verbose:
+              if s.fd == p.stdout:
+                didout = True
+              else:
+                errbuf += buf
+                continue
+
+            if empty:
+              if first:
+                first = False
+              else:
+                out.nl()
+              out.project('project %s/', project.relpath)
+              out.nl()
+              out.flush()
+              if errbuf:
+                sys.stderr.write(errbuf)
+                sys.stderr.flush()
+                errbuf = ''
+              empty = False
+
+            s.dest.write(buf)
+            s.dest.flush()
+
       r = p.wait()
       if r != 0 and r != rc:
         rc = r

subcmds/grep.py

@@ -0,0 +1,243 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from optparse import SUPPRESS_HELP
+from color import Coloring
+from command import PagedCommand
+from git_command import GitCommand
+
+class GrepColoring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, 'grep')
+    self.project = self.printer('project', attr='bold')
+
+class Grep(PagedCommand):
+  common = True
+  helpSummary = "Print lines matching a pattern"
+  helpUsage = """
+%prog {pattern | -e pattern} [<project>...]
+"""
+  helpDescription = """
+Search for the specified patterns in all project files.
+
+Boolean Options
+---------------
+
+The following options can appear as often as necessary to express
+the pattern to locate:
+
+ -e PATTERN
+ --and, --or, --not, -(, -)
+
+Further, the -r/--revision option may be specified multiple times
+in order to scan multiple trees.  If the same file matches in more
+than one tree, only the first result is reported, prefixed by the
+revision name it was found under.
+
+Examples
+-------
+
+Look for a line that has '#define' and either 'MAX_PATH or 'PATH_MAX':
+
+  repo grep -e '#define' --and -\( -e MAX_PATH -e PATH_MAX \)
+
+Look for a line that has 'NODE' or 'Unexpected' in files that
+contain a line that matches both expressions:
+
+  repo grep --all-match -e NODE -e Unexpected
+
+"""
+
+  def _Options(self, p):
+    def carry(option,
+              opt_str,
+              value,
+              parser):
+      pt = getattr(parser.values, 'cmd_argv', None)
+      if pt is None:
+        pt = []
+        setattr(parser.values, 'cmd_argv', pt)
+
+      if opt_str == '-(':
+        pt.append('(')
+      elif opt_str == '-)':
+        pt.append(')')
+      else:
+        pt.append(opt_str)
+
+      if value is not None:
+        pt.append(value)
+
+    g = p.add_option_group('Sources')
+    g.add_option('--cached',
+                 action='callback', callback=carry,
+                 help='Search the index, instead of the work tree')
+    g.add_option('-r','--revision',
+                 dest='revision', action='append', metavar='TREEish',
+                 help='Search TREEish, instead of the work tree')
+
+    g = p.add_option_group('Pattern')
+    g.add_option('-e',
+                 action='callback', callback=carry,
+                 metavar='PATTERN', type='str',
+                 help='Pattern to search for')
+    g.add_option('-i', '--ignore-case',
+                 action='callback', callback=carry,
+                 help='Ignore case differences')
+    g.add_option('-a','--text',
+                 action='callback', callback=carry,
+                 help="Process binary files as if they were text")
+    g.add_option('-I',
+                 action='callback', callback=carry,
+                 help="Don't match the pattern in binary files")
+    g.add_option('-w', '--word-regexp',
+                 action='callback', callback=carry,
+                 help='Match the pattern only at word boundaries')
+    g.add_option('-v', '--invert-match',
+                 action='callback', callback=carry,
+                 help='Select non-matching lines')
+    g.add_option('-G', '--basic-regexp',
+                 action='callback', callback=carry,
+                 help='Use POSIX basic regexp for patterns (default)')
+    g.add_option('-E', '--extended-regexp',
+                 action='callback', callback=carry,
+                 help='Use POSIX extended regexp for patterns')
+    g.add_option('-F', '--fixed-strings',
+                 action='callback', callback=carry,
+                 help='Use fixed strings (not regexp) for pattern')
+
+    g = p.add_option_group('Pattern Grouping')
+    g.add_option('--all-match',
+                 action='callback', callback=carry,
+                 help='Limit match to lines that have all patterns')
+    g.add_option('--and', '--or', '--not',
+                 action='callback', callback=carry,
+                 help='Boolean operators to combine patterns')
+    g.add_option('-(','-)',
+                 action='callback', callback=carry,
+                 help='Boolean operator grouping')
+
+    g = p.add_option_group('Output')
+    g.add_option('-n',
+                 action='callback', callback=carry,
+                 help='Prefix the line number to matching lines')
+    g.add_option('-C',
+                 action='callback', callback=carry,
+                 metavar='CONTEXT', type='str',
+                 help='Show CONTEXT lines around match')
+    g.add_option('-B',
+                 action='callback', callback=carry,
+                 metavar='CONTEXT', type='str',
+                 help='Show CONTEXT lines before match')
+    g.add_option('-A',
+                 action='callback', callback=carry,
+                 metavar='CONTEXT', type='str',
+                 help='Show CONTEXT lines after match')
+    g.add_option('-l','--name-only','--files-with-matches',
+                 action='callback', callback=carry,
+                 help='Show only file names containing matching lines')
+    g.add_option('-L','--files-without-match',
+                 action='callback', callback=carry,
+                 help='Show only file names not containing matching lines')
+
+
+  def Execute(self, opt, args):
+    out = GrepColoring(self.manifest.manifestProject.config)
+
+    cmd_argv = ['grep']
+    if out.is_on:
+      cmd_argv.append('--color')
+    cmd_argv.extend(getattr(opt,'cmd_argv',[]))
+
+    if '-e' not in cmd_argv:
+      if not args:
+        self.Usage()
+      cmd_argv.append('-e')
+      cmd_argv.append(args[0])
+      args = args[1:]
+
+    projects = self.GetProjects(args)
+
+    full_name = False
+    if len(projects) > 1:
+      cmd_argv.append('--full-name')
+      full_name = True
+
+    have_rev = False
+    if opt.revision:
+      if '--cached' in cmd_argv:
+        print >>sys.stderr,\
+          'fatal: cannot combine --cached and --revision'
+        sys.exit(1)
+      have_rev = True
+      cmd_argv.extend(opt.revision)
+    cmd_argv.append('--')
+
+    bad_rev = False
+    have_match = False
+
+    for project in projects:
+      p = GitCommand(project,
+                     cmd_argv,
+                     bare = False,
+                     capture_stdout = True,
+                     capture_stderr = True)
+      if p.Wait() != 0:
+        # no results
+        #
+        if p.stderr:
+          if have_rev and 'fatal: ambiguous argument' in p.stderr:
+            bad_rev = True
+          else:
+            out.project('--- project %s ---' % project.relpath)
+            out.nl()
+            out.write(p.stderr)
+            out.nl()
+        continue
+      have_match = True
+
+      # We cut the last element, to avoid a blank line.
+      #
+      r = p.stdout.split('\n')
+      r = r[0:-1]
+
+      if have_rev and full_name:
+        for line in r:
+          rev, line = line.split(':', 1)
+          out.write(rev)
+          out.write(':')
+          out.project(project.relpath)
+          out.write('/')
+          out.write(line)
+          out.nl()
+      elif full_name:
+        for line in r:
+          out.project(project.relpath)
+          out.write('/')
+          out.write(line)
+          out.nl()
+      else:
+        for line in r:
+          print line
+
+    if have_match:
+      sys.exit(0)
+    elif have_rev and bad_rev:
+      for r in opt.revision:
+        print >>sys.stderr, "error: can't search revision %s" % r
+      sys.exit(1)
+    else:
+      sys.exit(1)

subcmds/help.py

@@ -107,7 +107,7 @@ See 'repo help --all' for a complete list of recognized commands.
         body = body.strip()
         body = body.replace('%prog', me)
 
-        asciidoc_hdr = re.compile(r'^\n?([^\n]{1,})\n(={2,}|-{2,})$')
+        asciidoc_hdr = re.compile(r'^\n?([^\n]{1,})\n([=~-]{2,})$')
         for para in body.split("\n\n"):
           if para.startswith(' '):
             self.write('%s', para)
@@ -117,9 +117,19 @@ See 'repo help --all' for a complete list of recognized commands.
 
           m = asciidoc_hdr.match(para)
           if m:
-            self.heading('%s', m.group(1))
+            title = m.group(1)
+            type = m.group(2)
+            if type[0] in ('=', '-'):
+              p = self.heading
+            else:
+              def _p(fmt, *args):
+                self.write('  ')
+                self.heading(fmt, *args)
+              p = _p
+
+            p('%s', title)
             self.nl()
-            self.heading('%s', ''.ljust(len(m.group(1)),'-'))
+            p('%s', ''.ljust(len(title),type[0]))
             self.nl()
             continue
 
@@ -128,8 +138,8 @@ See 'repo help --all' for a complete list of recognized commands.
         self.wrap.end_paragraph(0)
 
     out = _Out(self.manifest.globalConfig)
-    cmd.OptionParser.print_help()
     out._PrintSection('Summary', 'helpSummary')
+    cmd.OptionParser.print_help()
     out._PrintSection('Description', 'helpDescription')
 
   def _Options(self, p):

subcmds/init.py

@@ -20,6 +20,7 @@ from color import Coloring
 from command import InteractiveCommand, MirrorSafeCommand
 from error import ManifestParseError
 from remote import Remote
+from project import SyncBuffer
 from git_command import git, MIN_GIT_VERSION
 
 class Init(InteractiveCommand, MirrorSafeCommand):
@@ -34,9 +35,20 @@ The latest repo source code and manifest collection is downloaded
 from the server and is installed in the .repo/ directory in the
 current working directory.
 
-The optional <manifest> argument can be used to specify an alternate
+The optional -b argument can be used to select the manifest branch
-manifest to be used. If no manifest is specified, the manifest
+to checkout and use.  If no branch is specified, master is assumed.
-default.xml will be used.
+
+The optional -m argument can be used to specify an alternate manifest
+to be used. If no manifest is specified, the manifest default.xml
+will be used.
+
+Switching Manifest Branches
+---------------------------
+
+To switch to another manifest branch, `repo init -b otherbranch`
+may be used in an existing client.  However, as this only updates the
+manifest, a subsequent `repo sync` (or `repo sync -d`) is necessary
+to update the working directory files.
 """
 
   def _Options(self, p):
@@ -63,7 +75,7 @@ default.xml will be used.
 
 
     # Tool
-    g = p.add_option_group('Version options')
+    g = p.add_option_group('repo Version options')
     g.add_option('--repo-url',
                  dest='repo_url',
                  help='repo repository location', metavar='URL')
@@ -89,8 +101,9 @@ default.xml will be used.
 
   def _SyncManifest(self, opt):
     m = self.manifest.manifestProject
+    is_new = not m.Exists
 
-    if not m.Exists:
+    if is_new:
       if not opt.manifest_url:
         print >>sys.stderr, 'fatal: manifest url (-u) is required.'
         sys.exit(1)
@@ -117,11 +130,25 @@ default.xml will be used.
       r.Save()
 
     if opt.mirror:
-      m.config.SetString('repo.mirror', 'true')
+      if is_new:
+        m.config.SetString('repo.mirror', 'true')
+      else:
+        print >>sys.stderr, 'fatal: --mirror not supported on existing client'
+        sys.exit(1)
 
-    m.Sync_NetworkHalf()
+    if not m.Sync_NetworkHalf():
-    m.Sync_LocalHalf()
+      r = m.GetRemote(m.remote.name)
-    m.StartBranch('default')
+      print >>sys.stderr, 'fatal: cannot obtain manifest %s' % r.url
+      sys.exit(1)
+
+    syncbuf = SyncBuffer(m.config)
+    m.Sync_LocalHalf(syncbuf)
+    syncbuf.Finish()
+
+    if is_new or m.CurrentBranch is None:
+      if not m.StartBranch('default'):
+        print >>sys.stderr, 'fatal: cannot create default in manifest'
+        sys.exit(1)
 
   def _LinkManifest(self, name):
     if not name:
@@ -192,11 +219,11 @@ default.xml will be used.
     self._SyncManifest(opt)
     self._LinkManifest(opt.manifest_name)
 
-    if os.isatty(0) and os.isatty(1) and not opt.mirror:
+    if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
       self._ConfigureUser()
       self._ConfigureColor()
 
-    if opt.mirror:
+    if self.manifest.IsMirror:
       type = 'mirror '
     else:
       type = ''

subcmds/selfupdate.py

@@ -0,0 +1,60 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from optparse import SUPPRESS_HELP
+import sys
+
+from command import Command, MirrorSafeCommand
+from subcmds.sync import _PostRepoUpgrade
+from subcmds.sync import _PostRepoFetch
+
+class Selfupdate(Command, MirrorSafeCommand):
+  common = False
+  helpSummary = "Update repo to the latest version"
+  helpUsage = """
+%prog
+"""
+  helpDescription = """
+The '%prog' command upgrades repo to the latest version, if a
+newer version is available.
+
+Normally this is done automatically by 'repo sync' and does not
+need to be performed by an end-user.
+"""
+
+  def _Options(self, p):
+    g = p.add_option_group('repo Version options')
+    g.add_option('--no-repo-verify',
+                 dest='no_repo_verify', action='store_true',
+                 help='do not verify repo source code')
+    g.add_option('--repo-upgraded',
+                 dest='repo_upgraded', action='store_true',
+                 help=SUPPRESS_HELP)
+
+  def Execute(self, opt, args):
+    rp = self.manifest.repoProject
+    rp.PreSync()
+
+    if opt.repo_upgraded:
+      _PostRepoUpgrade(self.manifest)
+
+    else:
+      if not rp.Sync_NetworkHalf():
+        print >>sys.stderr, "error: can't update repo"
+        sys.exit(1)
+
+      _PostRepoFetch(rp,
+                     no_repo_verify = opt.no_repo_verify,
+                     verbose = True)

subcmds/stage.py

@@ -55,12 +55,12 @@ The '%prog' command stages files to prepare the next commit.
 
     out = _ProjectList(self.manifest.manifestProject.config)
     while True:
-      out.header('        %-20s %s', 'project', 'path')
+      out.header('        %s', 'project')
       out.nl()
 
       for i in xrange(0, len(all)):
         p = all[i]
-        out.write('%3d:    %-20s %s', i + 1, p.name, p.relpath + '/')
+        out.write('%3d:    %s', i + 1, p.relpath + '/')
         out.nl()
       out.nl()
 

subcmds/start.py

@@ -16,27 +16,23 @@
 import sys
 from command import Command
 from git_command import git
+from progress import Progress
 
 class Start(Command):
   common = True
   helpSummary = "Start a new branch for development"
   helpUsage = """
-%prog <newbranchname> [<project>...]
+%prog <newbranchname> [--all | <project>...]
-
-This subcommand starts a new branch of development that is automatically
-pulled from a remote branch.
-
-It is equivalent to the following git commands:
-
-"git branch --track <newbranchname> m/<codeline>",
-or 
-"git checkout --track -b <newbranchname> m/<codeline>".
-
-All three forms set up the config entries that repo bases some of its
-processing on.  Use %prog or git branch or checkout with --track to ensure
-the configuration data is set up properly.
-
 """
+  helpDescription = """
+'%prog' begins a new branch of development, starting from the
+revision specified in the manifest.
+"""
+
+  def _Options(self, p):
+    p.add_option('--all',
+                 dest='all', action='store_true',
+                 help='begin branch in all projects')
 
   def Execute(self, opt, args):
     if not args:
@@ -47,5 +43,26 @@ the configuration data is set up properly.
       print >>sys.stderr, "error: '%s' is not a valid name" % nb
       sys.exit(1)
 
-    for project in self.GetProjects(args[1:]):
+    err = []
-      project.StartBranch(nb)
+    projects = []
+    if not opt.all:
+      projects = args[1:]
+      if len(projects) < 1:
+        print >>sys.stderr, "error: at least one project must be specified"
+        sys.exit(1)
+
+    all = self.GetProjects(projects)
+
+    pm = Progress('Starting %s' % nb, len(all))
+    for project in all:
+      pm.update()
+      if not project.StartBranch(nb):
+        err.append(project)
+    pm.end()
+
+    if err:
+      for p in err:
+        print >>sys.stderr,\
+          "error: %s/: cannot start %s" \
+          % (p.relpath, nb)
+      sys.exit(1)

subcmds/status.py

@@ -20,8 +20,66 @@ class Status(PagedCommand):
   helpSummary = "Show the working tree status"
   helpUsage = """
 %prog [<project>...]
+"""
+  helpDescription = """
+'%prog' compares the working tree to the staging area (aka index),
+and the most recent commit on this branch (HEAD), in each project
+specified.  A summary is displayed, one line per file where there
+is a difference between these three states.
+
+Status Display
+--------------
+
+The status display is organized into three columns of information,
+for example if the file 'subcmds/status.py' is modified in the
+project 'repo' on branch 'devwork':
+
+  project repo/                                   branch devwork
+   -m     subcmds/status.py
+
+The first column explains how the staging area (index) differs from
+the last commit (HEAD).  Its values are always displayed in upper
+case and have the following meanings:
+
+ -:  no difference
+ A:  added         (not in HEAD,     in index                     )
+ M:  modified      (    in HEAD,     in index, different content  )
+ D:  deleted       (    in HEAD, not in index                     )
+ R:  renamed       (not in HEAD,     in index, path changed       )
+ C:  copied        (not in HEAD,     in index, copied from another)
+ T:  mode changed  (    in HEAD,     in index, same content       )
+ U:  unmerged; conflict resolution required
+
+The second column explains how the working directory differs from
+the index.  Its values are always displayed in lower case and have
+the following meanings:
+
+ -:  new / unknown (not in index,     in work tree                )
+ m:  modified      (    in index,     in work tree, modified      )
+ d:  deleted       (    in index, not in work tree                )
+
 """
 
   def Execute(self, opt, args):
-    for project in self.GetProjects(args):
+    all = self.GetProjects(args)
-      project.PrintWorkTreeStatus()
+    clean = 0
+
+    on = {}
+    for project in all:
+      cb = project.CurrentBranch
+      if cb:
+        if cb not in on:
+          on[cb] = []
+        on[cb].append(project)
+
+    branch_names = list(on.keys())
+    branch_names.sort()
+    for cb in branch_names:
+      print '# on branch %s' % cb
+
+    for project in all:
+      state = project.PrintWorkTreeStatus()
+      if state == 'CLEAN':
+        clean += 1
+    if len(all) == clean:
+      print 'nothing to commit (working directory clean)'

subcmds/sync.py

@@ -13,15 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from optparse import SUPPRESS_HELP
 import os
 import re
 import subprocess
 import sys
+import time
 
 from git_command import GIT
+from project import HEAD
 from command import Command, MirrorSafeCommand
 from error import RepoChangedException, GitError
 from project import R_HEADS
+from project import SyncBuffer
+from progress import Progress
 
 class Sync(Command, MirrorSafeCommand):
   common = True
@@ -43,27 +48,83 @@ line.  Projects can be specified either by name, or by a relative
 or absolute path to the project's local directory. If no projects
 are specified, '%prog' will synchronize all projects listed in
 the manifest.
+
+The -d/--detach option can be used to switch specified projects
+back to the manifest revision.  This option is especially helpful
+if the project is currently on a topic branch, but the manifest
+revision is temporarily needed.
+
+SSH Connections
+---------------
+
+If at least one project remote URL uses an SSH connection (ssh://,
+git+ssh://, or user@host:path syntax) repo will automatically
+enable the SSH ControlMaster option when connecting to that host.
+This feature permits other projects in the same '%prog' session to
+reuse the same SSH tunnel, saving connection setup overheads.
+
+To disable this behavior on UNIX platforms, set the GIT_SSH
+environment variable to 'ssh'.  For example:
+
+  export GIT_SSH=ssh
+  %prog
+
+Compatibility
+~~~~~~~~~~~~~
+
+This feature is automatically disabled on Windows, due to the lack
+of UNIX domain socket support.
+
+This feature is not compatible with url.insteadof rewrites in the
+user's ~/.gitconfig.  '%prog' is currently not able to perform the
+rewrite early enough to establish the ControlMaster tunnel.
+
+If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
+later is required to fix a server side protocol bug.
+
 """
 
   def _Options(self, p):
-    p.add_option('--no-repo-verify',
+    p.add_option('-l','--local-only',
+                 dest='local_only', action='store_true',
+                 help="only update working tree, don't fetch")
+    p.add_option('-n','--network-only',
+                 dest='network_only', action='store_true',
+                 help="fetch only, don't update working tree")
+    p.add_option('-d','--detach',
+                 dest='detach_head', action='store_true',
+                 help='detach projects back to manifest revision')
+
+    g = p.add_option_group('repo Version options')
+    g.add_option('--no-repo-verify',
                  dest='no_repo_verify', action='store_true',
                  help='do not verify repo source code')
-    p.add_option('--repo-upgraded',
+    g.add_option('--repo-upgraded',
                  dest='repo_upgraded', action='store_true',
-                 help='perform additional actions after a repo upgrade')
+                 help=SUPPRESS_HELP)
 
-  def _Fetch(self, *projects):
+  def _Fetch(self, projects):
     fetched = set()
+    pm = Progress('Fetching projects', len(projects))
     for project in projects:
+      pm.update()
+
       if project.Sync_NetworkHalf():
         fetched.add(project.gitdir)
       else:
         print >>sys.stderr, 'error: Cannot fetch %s' % project.name
         sys.exit(1)
+    pm.end()
     return fetched
 
   def Execute(self, opt, args):
+    if opt.network_only and opt.detach_head:
+      print >>sys.stderr, 'error: cannot combine -n and -d'
+      sys.exit(1)
+    if opt.network_only and opt.local_only:
+      print >>sys.stderr, 'error: cannot combine -n and -l'
+      sys.exit(1)
+
     rp = self.manifest.repoProject
     rp.PreSync()
 
@@ -71,42 +132,73 @@ the manifest.
     mp.PreSync()
 
     if opt.repo_upgraded:
-      for project in self.manifest.projects.values():
+      _PostRepoUpgrade(self.manifest)
-        if project.Exists:
-          project.PostRepoUpgrade()
 
     all = self.GetProjects(args, missing_ok=True)
-    fetched = self._Fetch(rp, mp, *all)
 
-    if rp.HasChanges:
+    if not opt.local_only:
-      print >>sys.stderr, 'info: A new version of repo is available'
+      to_fetch = []
-      print >>sys.stderr, ''
+      now = time.time()
-      if opt.no_repo_verify or _VerifyTag(rp):
+      if (24 * 60 * 60) <= (now - rp.LastFetch):
-        if not rp.Sync_LocalHalf():
+        to_fetch.append(rp)
+      to_fetch.append(mp)
+      to_fetch.extend(all)
+
+      fetched = self._Fetch(to_fetch)
+      _PostRepoFetch(rp, opt.no_repo_verify)
+      if opt.network_only:
+        # bail out now; the rest touches the working tree
+        return
+
+      if mp.HasChanges:
+        syncbuf = SyncBuffer(mp.config)
+        mp.Sync_LocalHalf(syncbuf)
+        if not syncbuf.Finish():
           sys.exit(1)
-        print >>sys.stderr, 'info: Restarting repo with latest version'
-        raise RepoChangedException(['--repo-upgraded'])
-      else:
-        print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
 
-    if mp.HasChanges:
+        self.manifest._Unload()
-      if not mp.Sync_LocalHalf():
+        all = self.GetProjects(args, missing_ok=True)
-        sys.exit(1)
+        missing = []
-
+        for project in all:
-      self.manifest._Unload()
+          if project.gitdir not in fetched:
-      all = self.GetProjects(args, missing_ok=True)
+            missing.append(project)
-      missing = []
+        self._Fetch(missing)
-      for project in all:
-        if project.gitdir not in fetched:
-          missing.append(project)
-      self._Fetch(*missing)
 
+    syncbuf = SyncBuffer(mp.config,
+                         detach_head = opt.detach_head)
+    pm = Progress('Syncing work tree', len(all))
     for project in all:
+      pm.update()
       if project.worktree:
-        if not project.Sync_LocalHalf():
+        project.Sync_LocalHalf(syncbuf)
-          sys.exit(1)
+    pm.end()
+    print >>sys.stderr
+    if not syncbuf.Finish():
+      sys.exit(1)
 
 
+def _PostRepoUpgrade(manifest):
+  for project in manifest.projects.values():
+    if project.Exists:
+      project.PostRepoUpgrade()
+
+def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
+  if rp.HasChanges:
+    print >>sys.stderr, 'info: A new version of repo is available'
+    print >>sys.stderr, ''
+    if no_repo_verify or _VerifyTag(rp):
+      syncbuf = SyncBuffer(rp.config)
+      rp.Sync_LocalHalf(syncbuf)
+      if not syncbuf.Finish():
+        sys.exit(1)
+      print >>sys.stderr, 'info: Restarting repo with latest version'
+      raise RepoChangedException(['--repo-upgraded'])
+    else:
+      print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
+  else:
+    if verbose:
+      print >>sys.stderr, 'repo version %s is current' % rp.work_git.describe(HEAD)
+
 def _VerifyTag(project):
   gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
   if not os.path.exists(gpg_dir):

subcmds/upload.py

@@ -38,22 +38,21 @@ class Upload(InteractiveCommand):
 %prog [--re --cc] {[<project>]... | --replace <project>}
 """
   helpDescription = """
-The '%prog' command is used to send changes to the Gerrit code
+The '%prog' command is used to send changes to the Gerrit Code
-review system.  It searches for changes in local projects that do
+Review system.  It searches for topic branches in local projects
-not yet exist in the corresponding remote repository.  If multiple
+that have not yet been published for review.  If multiple topic
-changes are found, '%prog' opens an editor to allow the
+branches are found, '%prog' opens an editor to allow the user to
-user to choose which change to upload.  After a successful upload,
+select which branches to upload.
-repo prints the URL for the change in the Gerrit code review system.
 
-'%prog' searches for uploadable changes in all projects listed
+'%prog' searches for uploadable changes in all projects listed at
-at the command line.  Projects can be specified either by name, or
+the command line.  Projects can be specified either by name, or by
-by a relative or absolute path to the project's local directory. If
+a relative or absolute path to the project's local directory. If no
-no projects are specified, '%prog' will search for uploadable
+projects are specified, '%prog' will search for uploadable changes
-changes in all projects listed in the manifest.
+in all projects listed in the manifest.
 
 If the --reviewers or --cc options are passed, those emails are
 added to the respective list of users, and emails are sent to any
-new users.  Users passed to --reviewers must be already registered
+new users.  Users passed as --reviewers must already be registered
 with the code review system, or the upload will fail.
 
 If the --replace option is passed the user can designate which
@@ -61,6 +60,33 @@ existing change(s) in Gerrit match up to the commits in the branch
 being uploaded.  For each matched pair of change,commit the commit
 will be added as a new patch set, completely replacing the set of
 files and description associated with the change in Gerrit.
+
+Configuration
+-------------
+
+review.URL.autoupload:
+
+To disable the "Upload ... (y/n)?" prompt, you can set a per-project
+or global Git configuration option.  If review.URL.autoupload is set
+to "true" then repo will assume you always answer "y" at the prompt,
+and will not prompt you further.  If it is set to "false" then repo
+will assume you always answer "n", and will abort.
+
+The URL must match the review URL listed in the manifest XML file,
+or in the .git/config within the project.  For example:
+
+  [remote "origin"]
+    url = git://git.example.com/project.git
+    review = http://review.example.com/
+
+  [review "http://review.example.com/"]
+    autoupload = true
+
+References
+----------
+
+Gerrit Code Review:  http://code.google.com/p/gerrit/
+
 """
 
   def _Options(self, p):
@@ -77,21 +103,32 @@ files and description associated with the change in Gerrit.
   def _SingleBranch(self, branch, people):
     project = branch.project
     name = branch.name
-    date = branch.date
+    remote = project.GetBranch(name).remote
-    list = branch.commits
 
-    print 'Upload project %s/:' % project.relpath
+    key = 'review.%s.autoupload' % remote.review
-    print '  branch %s (%2d commit%s, %s):' % (
+    answer = project.config.GetBoolean(key)
-                  name,
-                  len(list),
-                  len(list) != 1 and 's' or '',
-                  date)
-    for commit in list:
-      print '         %s' % commit
 
-    sys.stdout.write('(y/n)? ')
+    if answer is False:
-    answer = sys.stdin.readline().strip()
+      _die("upload blocked by %s = false" % key)
-    if answer in ('y', 'Y', 'yes', '1', 'true', 't'):
+
+    if answer is None:
+      date = branch.date
+      list = branch.commits
+
+      print 'Upload project %s/:' % project.relpath
+      print '  branch %s (%2d commit%s, %s):' % (
+                    name,
+                    len(list),
+                    len(list) != 1 and 's' or '',
+                    date)
+      for commit in list:
+        print '         %s' % commit
+
+      sys.stdout.write('to %s (y/n)? ' % remote.review)
+      answer = sys.stdin.readline().strip()
+      answer = answer in ('y', 'Y', 'yes', '1', 'true', 't')
+
+    if answer:
       self._UploadAndReport([branch], people)
     else:
       _die("upload aborted by user")
@@ -234,9 +271,6 @@ files and description associated with the change in Gerrit.
           print >>sys.stderr, '[OK    ] %-15s %s' % (
                  branch.project.relpath + '/',
                  branch.name)
-          print >>sys.stderr, '%s' % branch.tip_url
-          print >>sys.stderr, '(as %s)' % branch.owner_email
-          print >>sys.stderr, ''
 
     if have_errors:
       sys.exit(1)

trace.py

@@ -0,0 +1,34 @@
+#
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import os
+REPO_TRACE = 'REPO_TRACE'
+
+try:
+  _TRACE = os.environ[REPO_TRACE] == '1'
+except KeyError:
+  _TRACE = False
+
+def IsTrace():
+  return _TRACE
+
+def SetTrace():
+  global _TRACE
+  _TRACE = True
+
+def Trace(fmt, *args):
+  if IsTrace():
+    print >>sys.stderr, fmt % args