Fix 'repo sync' rebase logic on a published branch

If the current branch is published, but all published commits are
merged into the manifest revision, but there is also at least one
unpublished commit on the current branch, we should rebase the
unpublished commit, rather than creating a merge commit.

Signed-off-by: Shawn O. Pearce <sop@google.com>

color.py

@@ -100,6 +100,9 @@ class Coloring(object):
     else:
       self._on = False
 
+  def redirect(self, out):
+    self._out = out
+
   @property
   def is_on(self):
     return self._on
@@ -107,6 +110,9 @@ class Coloring(object):
   def write(self, fmt, *args):
     self._out.write(fmt % args)
 
+  def flush(self):
+    self._out.flush()
+
   def nl(self):
     self._out.write('\n')
 

command.py

@@ -27,6 +27,9 @@ class Command(object):
   manifest = None
   _optparse = None
 
+  def WantPager(self, opt):
+    return False
+
   @property
   def OptionParser(self):
     if self._optparse is None:
@@ -109,8 +112,17 @@ class InteractiveCommand(Command):
   """Command which requires user interaction on the tty and
      must not run within a pager, even if the user asks to.
   """
+  def WantPager(self, opt):
+    return False
 
 class PagedCommand(Command):
   """Command which defaults to output in a pager, as its
      display tends to be larger than one screen full.
   """
+  def WantPager(self, opt):
+    return True
+
+class MirrorSafeCommand(object):
+  """Command permits itself to run within a mirror,
+     and does not require a working directory.
+  """

docs/manifest-format.txt

@@ -19,39 +19,39 @@ XML File Format
 A manifest XML file (e.g. 'default.xml') roughly conforms to the
 following DTD:
 
-<!DOCTYPE manifest [
-  <!ELEMENT manifest (remote*,
-                      default?,
-                      remove-project*,
-                      project*,
-                      add-remote*)>
-
-  <!ELEMENT remote (EMPTY)>
-  <!ATTLIST remote name         ID    #REQUIRED>
-  <!ATTLIST remote fetch        CDATA #REQUIRED>
-  <!ATTLIST remote review       CDATA #IMPLIED>
-  <!ATTLIST remote project-name CDATA #IMPLIED>
-
-  <!ELEMENT default (EMPTY)>
-  <!ATTLIST default remote   IDREF #IMPLIED>
-  <!ATTLIST default revision CDATA #IMPLIED>
-
-  <!ELEMENT project (remote*)>
-  <!ATTLIST project name     CDATA #REQUIRED>
-  <!ATTLIST project path     CDATA #IMPLIED>
-  <!ATTLIST project remote   IDREF #IMPLIED>
-  <!ATTLIST project revision CDATA #IMPLIED>
-
-  <!ELEMENT add-remote (EMPTY)>
-  <!ATTLIST add-remote to-project   ID    #REQUIRED>
-  <!ATTLIST add-remote name         ID    #REQUIRED>
-  <!ATTLIST add-remote fetch        CDATA #REQUIRED>
-  <!ATTLIST add-remote review       CDATA #IMPLIED>
-  <!ATTLIST add-remote project-name CDATA #IMPLIED>
-
-  <!ELEMENT remove-project (EMPTY)>
-  <!ATTLIST remove-project name  CDATA #REQUIRED>
-]>
+  <!DOCTYPE manifest [
+    <!ELEMENT manifest (remote*,
+                        default?,
+                        remove-project*,
+                        project*,
+                        add-remote*)>
+  
+    <!ELEMENT remote (EMPTY)>
+    <!ATTLIST remote name         ID    #REQUIRED>
+    <!ATTLIST remote fetch        CDATA #REQUIRED>
+    <!ATTLIST remote review       CDATA #IMPLIED>
+    <!ATTLIST remote project-name CDATA #IMPLIED>
+  
+    <!ELEMENT default (EMPTY)>
+    <!ATTLIST default remote   IDREF #IMPLIED>
+    <!ATTLIST default revision CDATA #IMPLIED>
+  
+    <!ELEMENT project (remote*)>
+    <!ATTLIST project name     CDATA #REQUIRED>
+    <!ATTLIST project path     CDATA #IMPLIED>
+    <!ATTLIST project remote   IDREF #IMPLIED>
+    <!ATTLIST project revision CDATA #IMPLIED>
+  
+    <!ELEMENT add-remote (EMPTY)>
+    <!ATTLIST add-remote to-project   ID    #REQUIRED>
+    <!ATTLIST add-remote name         ID    #REQUIRED>
+    <!ATTLIST add-remote fetch        CDATA #REQUIRED>
+    <!ATTLIST add-remote review       CDATA #IMPLIED>
+    <!ATTLIST add-remote project-name CDATA #IMPLIED>
+  
+    <!ELEMENT remove-project (EMPTY)>
+    <!ATTLIST remove-project name  CDATA #REQUIRED>
+  ]>
 
 A description of the elements and their attributes follows.
 
@@ -179,22 +179,15 @@ manifest, stored in `$TOP_DIR/.repo/local_manifest.xml`.
 
 For example:
 
-----
-	$ cat .repo/local_manifest.xml
-	<?xml version="1.0" encoding="UTF-8"?>
-	<manifest>
-	  <project path="manifest"
-			   name="tools/manifest" />
-	  <project path="platform-manifest"
-			   name="platform/manifest" />
-	</manifest>
-----
+  $ cat .repo/local_manifest.xml
+  <?xml version="1.0" encoding="UTF-8"?>
+  <manifest>
+    <project path="manifest"
+             name="tools/manifest" />
+    <project path="platform-manifest"
+             name="platform/manifest" />
+  </manifest>
 
 Users may add projects to the local manifest prior to a `repo sync`
 invocation, instructing repo to automatically download and manage
 these extra projects.
-
-Currently the only supported feature of a local manifest is to
-add new remotes and/or projects.  In the future a local manifest
-may support picking different revisions of a project, or deleting
-projects specified in the default manifest.

editor.py

@@ -78,7 +78,11 @@ least one of these before using this command."""
 
       if subprocess.Popen(editor + [path]).wait() != 0:
         raise EditorError()
-      return open(path).read()
+      fd2 = open(path)
+      try:
+        return fd2.read()
+      finally:
+        fd2.close()
     finally:
       if fd:
         os.close(fd)

error.py

@@ -17,6 +17,10 @@ class ManifestParseError(Exception):
   """Failed to parse the manifest file.
   """
 
+class ManifestInvalidRevisionError(Exception):
+  """The revision value in a project is incorrect.
+  """
+
 class EditorError(Exception):
   """Unspecified error from the user's text editor.
   """

git_command.py

@@ -16,19 +16,40 @@
 import os
 import sys
 import subprocess
+import tempfile
 from error import GitError
+from trace import REPO_TRACE, IsTrace, Trace
 
 GIT = 'git'
 MIN_GIT_VERSION = (1, 5, 4)
 GIT_DIR = 'GIT_DIR'
-REPO_TRACE = 'REPO_TRACE'
 
 LAST_GITDIR = None
 LAST_CWD = None
-try:
-  TRACE = os.environ[REPO_TRACE] == '1'
-except KeyError:
-  TRACE = False
+
+_ssh_proxy_path = None
+_ssh_sock_path = None
+
+def _ssh_sock(create=True):
+  global _ssh_sock_path
+  if _ssh_sock_path is None:
+    if not create:
+      return None
+    dir = '/tmp'
+    if not os.path.exists(dir):
+      dir = tempfile.gettempdir()
+    _ssh_sock_path = os.path.join(
+      tempfile.mkdtemp('', 'ssh-', dir),
+      'master-%r@%h:%p')
+  return _ssh_sock_path
+
+def _ssh_proxy():
+  global _ssh_proxy_path
+  if _ssh_proxy_path is None:
+    _ssh_proxy_path = os.path.join(
+      os.path.dirname(__file__),
+      'git_ssh')
+  return _ssh_proxy_path
 
 
 class _GitCall(object):
@@ -56,6 +77,7 @@ class GitCommand(object):
                capture_stdout = False,
                capture_stderr = False,
                disable_editor = False,
+               ssh_proxy = False,
                cwd = None,
                gitdir = None):
     env = dict(os.environ)
@@ -72,6 +94,9 @@ class GitCommand(object):
 
     if disable_editor:
       env['GIT_EDITOR'] = ':'
+    if ssh_proxy:
+      env['REPO_SSH_SOCK'] = _ssh_sock()
+      env['GIT_SSH'] = _ssh_proxy()
 
     if project:
       if not cwd:
@@ -101,7 +126,7 @@ class GitCommand(object):
     else:
       stderr = None
 
-    if TRACE:
+    if IsTrace():
       global LAST_CWD
       global LAST_GITDIR
 
@@ -127,7 +152,7 @@ class GitCommand(object):
         dbg += ' 1>|'
       if stderr == subprocess.PIPE:
         dbg += ' 2>|'
-      print >>sys.stderr, dbg
+      Trace('%s', dbg)
 
     try:
       p = subprocess.Popen(command,

git_config.py

@@ -13,20 +13,34 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import cPickle
 import os
 import re
+import subprocess
 import sys
+import time
+from signal import SIGTERM
 from urllib2 import urlopen, HTTPError
 from error import GitError, UploadError
-from git_command import GitCommand
+from trace import Trace
+from git_command import GitCommand, _ssh_sock
 
 R_HEADS = 'refs/heads/'
 R_TAGS  = 'refs/tags/'
 ID_RE = re.compile('^[0-9a-f]{40}$')
 
+REVIEW_CACHE = dict()
+
 def IsId(rev):
   return ID_RE.match(rev)
 
+def _key(name):
+  parts = name.split('.')
+  if len(parts) < 2:
+    return name.lower()
+  parts[ 0] = parts[ 0].lower()
+  parts[-1] = parts[-1].lower()
+  return '.'.join(parts)
 
 class GitConfig(object):
   _ForUser = None
@@ -46,14 +60,17 @@ class GitConfig(object):
     self.file = file
     self.defaults = defaults
     self._cache_dict = None
+    self._section_dict = None
     self._remotes = {}
     self._branches = {}
+    self._pickle = os.path.join(
+      os.path.dirname(self.file),
+      '.repopickle_' + os.path.basename(self.file))
 
   def Has(self, name, include_defaults = True):
     """Return true if this configuration file has the key.
     """
-    name = name.lower()
-    if name in self._cache:
+    if _key(name) in self._cache:
       return True
     if include_defaults and self.defaults:
       return self.defaults.Has(name, include_defaults = True)
@@ -81,10 +98,8 @@ class GitConfig(object):
        This configuration file is used first, if the key is not
        defined or all = True then the defaults are also searched.
     """
-    name = name.lower()
-
     try:
-      v = self._cache[name]
+      v = self._cache[_key(name)]
     except KeyError:
       if self.defaults:
         return self.defaults.GetString(name, all = all)
@@ -108,16 +123,16 @@ class GitConfig(object):
        The supplied value should be either a string,
        or a list of strings (to store multiple values).
     """
-    name = name.lower()
+    key = _key(name)
 
     try:
-      old = self._cache[name]
+      old = self._cache[key]
     except KeyError:
       old = []
 
     if value is None:
       if old:
-        del self._cache[name]
+        del self._cache[key]
         self._do('--unset-all', name)
 
     elif isinstance(value, list):
@@ -128,13 +143,13 @@ class GitConfig(object):
         self.SetString(name, value[0])
 
       elif old != value:
-        self._cache[name] = list(value)
+        self._cache[key] = list(value)
         self._do('--replace-all', name, value[0])
         for i in xrange(1, len(value)):
           self._do('--add', name, value[i])
 
     elif len(old) != 1 or old[0] != value:
-      self._cache[name] = [value]
+      self._cache[key] = [value]
       self._do('--replace-all', name, value)
 
   def GetRemote(self, name):
@@ -157,6 +172,33 @@ class GitConfig(object):
       self._branches[b.name] = b
     return b
 
+  def HasSection(self, section, subsection = ''):
+    """Does at least one key in section.subsection exist?
+    """
+    try:
+      return subsection in self._sections[section]
+    except KeyError:
+      return False
+
+  @property
+  def _sections(self):
+    d = self._section_dict
+    if d is None:
+      d = {}
+      for name in self._cache.keys():
+        p = name.split('.')
+        if 2 == len(p):
+          section = p[0]
+          subsect = ''
+        else:
+          section = p[0]
+          subsect = '.'.join(p[1:-1])
+        if section not in d:
+          d[section] = set()
+        d[section].add(subsect)
+        self._section_dict = d
+    return d
+
   @property
   def _cache(self):
     if self._cache_dict is None:
@@ -164,13 +206,54 @@ class GitConfig(object):
     return self._cache_dict
 
   def _Read(self):
+    d = self._ReadPickle()
+    if d is None:
+      d = self._ReadGit()
+      self._SavePickle(d)
+    return d
+
+  def _ReadPickle(self):
+    try:
+      if os.path.getmtime(self._pickle) \
+      <= os.path.getmtime(self.file):
+        os.remove(self._pickle)
+        return None
+    except OSError:
+      return None
+    try:
+      Trace(': unpickle %s', self.file)
+      fd = open(self._pickle, 'rb')
+      try:
+        return cPickle.load(fd)
+      finally:
+        fd.close()
+    except IOError:
+      os.remove(self._pickle)
+      return None
+    except cPickle.PickleError:
+      os.remove(self._pickle)
+      return None
+
+  def _SavePickle(self, cache):
+    try:
+      fd = open(self._pickle, 'wb')
+      try:
+        cPickle.dump(cache, fd, cPickle.HIGHEST_PROTOCOL)
+      finally:
+        fd.close()
+    except IOError:
+      os.remove(self._pickle)
+    except cPickle.PickleError:
+      os.remove(self._pickle)
+
+  def _ReadGit(self):
     d = self._do('--null', '--list')
     c = {}
     while d:
       lf = d.index('\n')
       nul = d.index('\0', lf + 1)
 
-      key = d[0:lf]
+      key = _key(d[0:lf])
       val = d[lf + 1:nul]
 
       if key in c:
@@ -251,6 +334,79 @@ class RefSpec(object):
     return s
 
 
+_ssh_cache = {}
+_ssh_master = True
+
+def _open_ssh(host, port=None):
+  global _ssh_master
+
+  if port is None:
+    port = 22
+
+  key = '%s:%s' % (host, port)
+  if key in _ssh_cache:
+    return True
+
+  if not _ssh_master \
+  or 'GIT_SSH' in os.environ \
+  or sys.platform == 'win32':
+    # failed earlier, or cygwin ssh can't do this
+    #
+    return False
+
+  command = ['ssh',
+             '-o','ControlPath %s' % _ssh_sock(),
+             '-p',str(port),
+             '-M',
+             '-N',
+             host]
+  try:
+    Trace(': %s', ' '.join(command))
+    p = subprocess.Popen(command)
+  except Exception, e:
+    _ssh_master = False
+    print >>sys.stderr, \
+      '\nwarn: cannot enable ssh control master for %s:%s\n%s' \
+      % (host,port, str(e))
+    return False
+
+  _ssh_cache[key] = p
+  time.sleep(1)
+  return True
+
+def close_ssh():
+  for key,p in _ssh_cache.iteritems():
+    os.kill(p.pid, SIGTERM)
+    p.wait()
+  _ssh_cache.clear()
+
+  d = _ssh_sock(create=False)
+  if d:
+    try:
+      os.rmdir(os.path.dirname(d))
+    except OSError:
+      pass
+
+URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
+URI_ALL = re.compile(r'^([a-z][a-z+]*)://([^@/]*@?[^/])/')
+
+def _preconnect(url):
+  m = URI_ALL.match(url)
+  if m:
+    scheme = m.group(1)
+    host = m.group(2)
+    if ':' in host:
+      host, port = host.split(':')
+    if scheme in ('ssh', 'git+ssh', 'ssh+git'):
+      return _open_ssh(host, port)
+    return False
+
+  m = URI_SCP.match(url)
+  if m:
+    host = m.group(1)
+    return _open_ssh(host)
+
+
 class Remote(object):
   """Configuration options related to a remote.
   """
@@ -264,6 +420,9 @@ class Remote(object):
                      self._Get('fetch', all=True))
     self._review_protocol = None
 
+  def PreConnectFetch(self):
+    return _preconnect(self.url)
+
   @property
   def ReviewProtocol(self):
     if self._review_protocol is None:
@@ -273,24 +432,44 @@ class Remote(object):
       u = self.review
       if not u.startswith('http:') and not u.startswith('https:'):
         u = 'http://%s' % u
-      if not u.endswith('/'):
-        u += '/'
-      u += 'ssh_info'
+      if u.endswith('/Gerrit'):
+        u = u[:len(u) - len('/Gerrit')]
+      if not u.endswith('/ssh_info'):
+        if not u.endswith('/'):
+          u += '/'
+        u += 'ssh_info'
 
-      try:
-        info = urlopen(u).read()
-        if info == 'NOT_AVAILABLE':
-          raise UploadError('Upload over ssh unavailable')
+      if u in REVIEW_CACHE:
+        info = REVIEW_CACHE[u]
+        self._review_protocol = info[0]
+        self._review_host = info[1]
+        self._review_port = info[2]
+      else:
+        try:
+          info = urlopen(u).read()
+          if info == 'NOT_AVAILABLE':
+            raise UploadError('Upload over ssh unavailable')
+          if '<' in info:
+            # Assume the server gave us some sort of HTML
+            # response back, like maybe a login page.
+            #
+            raise UploadError('Cannot read %s:\n%s' % (u, info))
 
-        self._review_protocol = 'ssh'
-        self._review_host = info.split(" ")[0]
-        self._review_port = info.split(" ")[1]
+          self._review_protocol = 'ssh'
+          self._review_host = info.split(" ")[0]
+          self._review_port = info.split(" ")[1]
+        except HTTPError, e:
+          if e.code == 404:
+            self._review_protocol = 'http-post'
+            self._review_host = None
+            self._review_port = None
+          else:
+            raise UploadError('Cannot guess Gerrit version')
 
-      except HTTPError, e:
-        if e.code == 404:
-          self._review_protocol = 'http-post'
-        else:
-          raise UploadError('Cannot guess Gerrit version')
+        REVIEW_CACHE[u] = (
+          self._review_protocol,
+          self._review_host,
+          self._review_port)
     return self._review_protocol
 
   def SshReviewUrl(self, userEmail):
@@ -377,11 +556,23 @@ class Branch(object):
   def Save(self):
     """Save this branch back into the configuration.
     """
-    self._Set('merge', self.merge)
-    if self.remote:
-      self._Set('remote', self.remote.name)
+    if self._config.HasSection('branch', self.name):
+      if self.remote:
+        self._Set('remote', self.remote.name)
+      else:
+        self._Set('remote', None)
+      self._Set('merge', self.merge)
+
     else:
-      self._Set('remote', None)
+      fd = open(self._config.file, 'ab')
+      try:
+        fd.write('[branch "%s"]\n' % self.name)
+        if self.remote:
+          fd.write('\tremote = %s\n' % self.remote.name)
+        if self.merge:
+          fd.write('\tmerge = %s\n' % self.merge)
+      finally:
+        fd.close()
 
   def _Set(self, key, value):
     key = 'branch.%s.%s' % (self.name, key)

git_refs.py

@@ -0,0 +1,160 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+from trace import Trace
+
+HEAD    = 'HEAD'
+R_HEADS = 'refs/heads/'
+R_TAGS  = 'refs/tags/'
+R_PUB   = 'refs/published/'
+R_M     = 'refs/remotes/m/'
+
+
+class GitRefs(object):
+  def __init__(self, gitdir):
+    self._gitdir = gitdir
+    self._phyref = None
+    self._symref = None
+    self._mtime = {}
+
+  @property
+  def all(self):
+    self._EnsureLoaded()
+    return self._phyref
+
+  def get(self, name):
+    try:
+      return self.all[name]
+    except KeyError:
+      return ''
+
+  def deleted(self, name):
+    if self._phyref is not None:
+      if name in self._phyref:
+        del self._phyref[name]
+
+      if name in self._symref:
+        del self._symref[name]
+
+      if name in self._mtime:
+        del self._mtime[name]
+
+  def symref(self, name):
+    try:
+      self._EnsureLoaded()
+      return self._symref[name]
+    except KeyError:
+      return ''
+
+  def _EnsureLoaded(self):
+    if self._phyref is None or self._NeedUpdate():
+      self._LoadAll()
+
+  def _NeedUpdate(self):
+    Trace(': scan refs %s', self._gitdir)
+
+    for name, mtime in self._mtime.iteritems():
+      try:
+        if mtime != os.path.getmtime(os.path.join(self._gitdir, name)):
+          return True
+      except OSError:
+        return True
+    return False
+
+  def _LoadAll(self):
+    Trace(': load refs %s', self._gitdir)
+
+    self._phyref = {}
+    self._symref = {}
+    self._mtime = {}
+
+    self._ReadPackedRefs()
+    self._ReadLoose('refs/')
+    self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD)
+
+    scan = self._symref
+    attempts = 0
+    while scan and attempts < 5:
+      scan_next = {}
+      for name, dest in scan.iteritems():
+        if dest in self._phyref:
+          self._phyref[name] = self._phyref[dest]
+        else:
+          scan_next[name] = dest
+      scan = scan_next
+      attempts += 1
+
+  def _ReadPackedRefs(self):
+    path = os.path.join(self._gitdir, 'packed-refs')
+    try:
+      fd = open(path, 'rb')
+      mtime = os.path.getmtime(path)
+    except IOError:
+      return
+    except OSError:
+      return
+    try:
+      for line in fd:
+        if line[0] == '#':
+          continue
+        if line[0] == '^':
+          continue
+
+        line = line[:-1]
+        p = line.split(' ')
+        id = p[0]
+        name = p[1]
+
+        self._phyref[name] = id
+    finally:
+      fd.close()
+    self._mtime['packed-refs'] = mtime
+
+  def _ReadLoose(self, prefix):
+    base = os.path.join(self._gitdir, prefix)
+    for name in os.listdir(base):
+      p = os.path.join(base, name)
+      if os.path.isdir(p):
+        self._mtime[prefix] = os.path.getmtime(base)
+        self._ReadLoose(prefix + name + '/')
+      elif name.endswith('.lock'):
+        pass
+      else:
+        self._ReadLoose1(p, prefix + name)
+
+  def _ReadLoose1(self, path, name):
+    try:
+      fd = open(path, 'rb')
+      mtime = os.path.getmtime(path)
+    except OSError:
+      return
+    except IOError:
+      return
+    try:
+      id = fd.readline()
+    finally:
+      fd.close()
+
+    if not id:
+      return
+    id = id[:-1]
+
+    if id.startswith('ref: '):
+      self._symref[name] = id[5:]
+    else:
+      self._phyref[name] = id
+    self._mtime[name] = mtime

git_ssh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec ssh -o "ControlPath $REPO_SSH_SOCK" "$@"

main.py

@@ -27,8 +27,13 @@ import os
 import re
 import sys
 
-from command import InteractiveCommand, PagedCommand
+from trace import SetTrace
+from git_config import close_ssh
+from command import InteractiveCommand
+from command import MirrorSafeCommand
+from command import PagedCommand
 from editor import Editor
+from error import ManifestInvalidRevisionError
 from error import NoSuchProjectError
 from error import RepoChangedException
 from manifest import Manifest
@@ -45,6 +50,12 @@ global_options.add_option('-p', '--paginate',
 global_options.add_option('--no-pager',
                           dest='no_pager', action='store_true',
                           help='disable the pager')
+global_options.add_option('--trace',
+                          dest='trace', action='store_true',
+                          help='trace git command execution')
+global_options.add_option('--version',
+                          dest='show_version', action='store_true',
+                          help='display this version of repo')
 
 class _Repo(object):
   def __init__(self, repodir):
@@ -68,6 +79,15 @@ class _Repo(object):
       argv = []
     gopts, gargs = global_options.parse_args(glob)
 
+    if gopts.trace:
+      SetTrace()
+    if gopts.show_version:
+      if name == 'help':
+        name = 'version'
+      else:
+        print >>sys.stderr, 'fatal: invalid usage of --version'
+        sys.exit(1)
+
     try:
       cmd = self.commands[name]
     except KeyError:
@@ -80,6 +100,14 @@ class _Repo(object):
     cmd.manifest = Manifest(cmd.repodir)
     Editor.globalConfig = cmd.manifest.globalConfig
 
+    if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror:
+      print >>sys.stderr, \
+            "fatal: '%s' requires a working directory"\
+            % name
+      sys.exit(1)
+
+    copts, cargs = cmd.OptionParser.parse_args(argv)
+
     if not gopts.no_pager and not isinstance(cmd, InteractiveCommand):
       config = cmd.manifest.globalConfig
       if gopts.pager:
@@ -87,13 +115,15 @@ class _Repo(object):
       else:
         use_pager = config.GetBoolean('pager.%s' % name)
         if use_pager is None:
-          use_pager = isinstance(cmd, PagedCommand)
+          use_pager = cmd.WantPager(copts)
       if use_pager:
         RunPager(config)
 
-    copts, cargs = cmd.OptionParser.parse_args(argv)
     try:
       cmd.Execute(copts, cargs)
+    except ManifestInvalidRevisionError, e:
+      print >>sys.stderr, 'error: %s' % str(e)
+      sys.exit(1)
     except NoSuchProjectError, e:
       if e.name:
         print >>sys.stderr, 'error: project %s not found' % e.name
@@ -183,7 +213,10 @@ def _Main(argv):
 
   repo = _Repo(opt.repodir)
   try:
-    repo._Run(argv)
+    try:
+      repo._Run(argv)
+    finally:
+      close_ssh()
   except KeyboardInterrupt:
     sys.exit(1)
   except RepoChangedException, rce:

manifest.py

@@ -18,7 +18,7 @@ import sys
 import xml.dom.minidom
 
 from git_config import GitConfig, IsId
-from project import Project, MetaProject, R_HEADS
+from project import Project, MetaProject, R_HEADS, HEAD
 from remote import Remote
 from error import ManifestParseError
 
@@ -73,6 +73,76 @@ class Manifest(object):
     except OSError, e:
       raise ManifestParseError('cannot link manifest %s' % name)
 
+  def _RemoteToXml(self, r, doc, root):
+    e = doc.createElement('remote')
+    root.appendChild(e)
+    e.setAttribute('name', r.name)
+    e.setAttribute('fetch', r.fetchUrl)
+    if r.reviewUrl is not None:
+      e.setAttribute('review', r.reviewUrl)
+    if r.projectName is not None:
+      e.setAttribute('project-name', r.projectName)
+
+  def Save(self, fd, peg_rev=False):
+    """Write the current manifest out to the given file descriptor.
+    """
+    doc = xml.dom.minidom.Document()
+    root = doc.createElement('manifest')
+    doc.appendChild(root)
+
+    d = self.default
+    sort_remotes = list(self.remotes.keys())
+    sort_remotes.sort()
+
+    for r in sort_remotes:
+      self._RemoteToXml(self.remotes[r], doc, root)
+    if self.remotes:
+      root.appendChild(doc.createTextNode(''))
+
+    have_default = False
+    e = doc.createElement('default')
+    if d.remote:
+      have_default = True
+      e.setAttribute('remote', d.remote.name)
+    if d.revision:
+      have_default = True
+      e.setAttribute('revision', d.revision)
+    if have_default:
+      root.appendChild(e)
+      root.appendChild(doc.createTextNode(''))
+
+    sort_projects = list(self.projects.keys())
+    sort_projects.sort()
+
+    for p in sort_projects:
+      p = self.projects[p]
+      e = doc.createElement('project')
+      root.appendChild(e)
+      e.setAttribute('name', p.name)
+      if p.relpath != p.name:
+        e.setAttribute('path', p.relpath)
+      if not d.remote or p.remote.name != d.remote.name:
+        e.setAttribute('remote', p.remote.name)
+      if peg_rev:
+        if self.IsMirror:
+          e.setAttribute('revision',
+                         p.bare_git.rev_parse(p.revision + '^0'))
+        else:
+          e.setAttribute('revision',
+                         p.work_git.rev_parse(HEAD + '^0'))
+      elif not d.revision or p.revision != d.revision:
+        e.setAttribute('revision', p.revision)
+
+      for r in p.extraRemotes:
+        self._RemoteToXml(p.extraRemotes[r], doc, e)
+      for c in p.copyfiles:
+        ce = doc.createElement('copyfile')
+        ce.setAttribute('src', c.src)
+        ce.setAttribute('dest', c.dest)
+        e.appendChild(ce)
+
+    doc.writexml(fd, '', '  ', '\n', 'UTF-8')
+
   @property
   def projects(self):
     self._Load()
@@ -324,7 +394,7 @@ class Manifest(object):
     if not self.IsMirror:
       # src is project relative;
       # dest is relative to the top of the tree
-      project.AddCopyFile(src, os.path.join(self.topdir, dest))
+      project.AddCopyFile(src, dest, os.path.join(self.topdir, dest))
 
   def _get_remote(self, node):
     name = node.getAttribute('remote')

pager.py

@@ -22,7 +22,7 @@ active = False
 def RunPager(globalConfig):
   global active
 
-  if not os.isatty(0):
+  if not os.isatty(0) or not os.isatty(1):
     return
   pager = _SelectPager(globalConfig)
   if pager == '' or pager == 'cat':

progress.py

@@ -0,0 +1,74 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from time import time
+from trace import IsTrace
+
+class Progress(object):
+  def __init__(self, title, total=0):
+    self._title = title
+    self._total = total
+    self._done = 0
+    self._lastp = -1
+    self._start = time()
+    self._show = False
+
+  def update(self, inc=1):
+    self._done += inc
+
+    if IsTrace():
+      return
+
+    if not self._show:
+      if 0.5 <= time() - self._start:
+        self._show = True
+      else:
+        return
+
+    if self._total <= 0:
+      sys.stderr.write('\r%s: %d, ' % (
+        self._title,
+        self._done))
+      sys.stderr.flush()
+    else:
+      p = (100 * self._done) / self._total
+
+      if self._lastp != p:
+        self._lastp = p
+        sys.stderr.write('\r%s: %3d%% (%d/%d)  ' % (
+          self._title,
+          p,
+          self._done,
+          self._total))
+        sys.stderr.flush()
+
+  def end(self):
+    if IsTrace() or not self._show:
+      return
+
+    if self._total <= 0:
+      sys.stderr.write('\r%s: %d, done.  \n' % (
+        self._title,
+        self._done))
+      sys.stderr.flush()
+    else:
+      p = (100 * self._done) / self._total
+      sys.stderr.write('\r%s: %3d%% (%d/%d), done.  \n' % (
+        self._title,
+        p,
+        self._done,
+        self._total))
+      sys.stderr.flush()

project.py

@@ -25,21 +25,29 @@ from color import Coloring
 from git_command import GitCommand
 from git_config import GitConfig, IsId
 from error import GitError, ImportError, UploadError
+from error import ManifestInvalidRevisionError
 from remote import Remote
 
-HEAD    = 'HEAD'
-R_HEADS = 'refs/heads/'
-R_TAGS  = 'refs/tags/'
-R_PUB   = 'refs/published/'
-R_M     = 'refs/remotes/m/'
+from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M
 
-def _warn(fmt, *args):
-  msg = fmt % args
-  print >>sys.stderr, 'warn: %s' % msg
+def _lwrite(path, content):
+  lock = '%s.lock' % path
 
-def _info(fmt, *args):
+  fd = open(lock, 'wb')
+  try:
+    fd.write(content)
+  finally:
+    fd.close()
+
+  try:
+    os.rename(lock, path)
+  except OSError:
+    os.remove(lock)
+    raise
+
+def _error(fmt, *args):
   msg = fmt % args
-  print >>sys.stderr, 'info: %s' % msg
+  print >>sys.stderr, 'error: %s' % msg
 
 def not_rev(r):
   return '^' + r
@@ -147,16 +155,6 @@ class ReviewableBranch(object):
                                  self.replace_changes,
                                  people)
 
-  @property
-  def tip_url(self):
-    me = self.project.GetBranch(self.name)
-    commit = self.project.bare_git.rev_parse(R_HEADS + self.name)
-    return 'http://%s/r/%s' % (me.remote.review, commit[0:12])
-
-  @property
-  def owner_email(self):
-    return self.project.UserEmail
-
 
 class StatusColoring(Coloring):
   def __init__(self, config):
@@ -164,6 +162,7 @@ class StatusColoring(Coloring):
     self.project   = self.printer('header',    attr = 'bold')
     self.branch    = self.printer('header',    attr = 'bold')
     self.nobranch  = self.printer('nobranch',  fg = 'red')
+    self.important = self.printer('important', fg = 'red')
 
     self.added     = self.printer('added',     fg = 'green')
     self.changed   = self.printer('changed',   fg = 'red')
@@ -177,13 +176,15 @@ class DiffColoring(Coloring):
 
 
 class _CopyFile:
-  def __init__(self, src, dest):
+  def __init__(self, src, dest, abssrc, absdest):
     self.src = src
     self.dest = dest
+    self.abs_src = abssrc
+    self.abs_dest = absdest
 
   def _Copy(self):
-    src = self.src
-    dest = self.dest
+    src = self.abs_src
+    dest = self.abs_dest
     # copy file if it does not exist or is out of date
     if not os.path.exists(dest) or not filecmp.cmp(src, dest):
       try:
@@ -196,9 +197,7 @@ class _CopyFile:
         mode = mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
         os.chmod(dest, mode)
       except IOError:
-           print >>sys.stderr, \
-              'error: Cannot copy file %s to %s' \
-              % (src, dest)
+        _error('Cannot copy file %s to %s', src, dest)
 
 
 class Project(object):
@@ -229,6 +228,7 @@ class Project(object):
     else:
       self.work_git = None
     self.bare_git = self._GitGetByExec(self, bare=True)
+    self.bare_ref = GitRefs(gitdir)
 
   @property
   def Exists(self):
@@ -240,14 +240,18 @@ class Project(object):
        The branch name omits the 'refs/heads/' prefix.
        None is returned if the project is on a detached HEAD.
     """
-    try:
-      b = self.work_git.GetHead()
-    except GitError:
-      return None
+    b = self.work_git.GetHead()
     if b.startswith(R_HEADS):
       return b[len(R_HEADS):]
     return None
 
+  def IsRebaseInProgress(self):
+    w = self.worktree
+    g = os.path.join(w, '.git')
+    return os.path.exists(os.path.join(g, 'rebase-apply')) \
+        or os.path.exists(os.path.join(g, 'rebase-merge')) \
+        or os.path.exists(os.path.join(w, '.dotest'))
+    
   def IsDirty(self, consider_untracked=True):
     """Is the working directory modified in some way?
     """
@@ -303,6 +307,32 @@ class Project(object):
     """
     return self.config.GetBranch(name)
 
+  def GetBranches(self):
+    """Get all existing local branches.
+    """
+    current = self.CurrentBranch
+    all = self._allrefs
+    heads = {}
+    pubd = {}
+
+    for name, id in all.iteritems():
+      if name.startswith(R_HEADS):
+        name = name[len(R_HEADS):]
+        b = self.GetBranch(name)
+        b.current = name == current
+        b.published = None
+        b.revision = id
+        heads[name] = b
+
+    for name, id in all.iteritems():
+      if name.startswith(R_PUB):
+        name = name[len(R_PUB):]
+        b = heads.get(name)
+        if b:
+          b.published = id
+
+    return heads
+
 
 ## Status Display ##
 
@@ -319,11 +349,12 @@ class Project(object):
                                '--unmerged',
                                '--ignore-missing',
                                '--refresh')
+    rb = self.IsRebaseInProgress()
     di = self.work_git.DiffZ('diff-index', '-M', '--cached', HEAD)
     df = self.work_git.DiffZ('diff-files')
     do = self.work_git.LsOthers()
-    if not di and not df and not do:
-      return
+    if not rb and not di and not df and not do:
+      return 'CLEAN'
 
     out = StatusColoring(self.config)
     out.project('project %-40s', self.relpath + '/')
@@ -335,6 +366,10 @@ class Project(object):
       out.branch('branch %s', branch)
     out.nl()
 
+    if rb:
+      out.important('prior sync failed; rebase still in progress')
+      out.nl()
+
     paths = list()
     paths.extend(di.keys())
     paths.extend(df.keys())
@@ -357,7 +392,7 @@ class Project(object):
       else: f_status = '-'
 
       if i and i.src_path:
-        line = ' %s%s\t%s => (%s%%)' % (i_status, f_status,
+        line = ' %s%s\t%s => %s (%s%%)' % (i_status, f_status,
                                         i.src_path, p, i.level)
       else:
         line = ' %s%s\t%s' % (i_status, f_status, p)
@@ -371,6 +406,7 @@ class Project(object):
       else:
         out.write('%s', line)
       out.nl()
+    return 'DIRTY'
 
   def PrintWorkTreeDiff(self):
     """Prints the status of the repository to stdout.
@@ -398,22 +434,31 @@ class Project(object):
 
 ## Publish / Upload ##
 
-  def WasPublished(self, branch):
+  def WasPublished(self, branch, all=None):
     """Was the branch published (uploaded) for code review?
        If so, returns the SHA-1 hash of the last published
        state for the branch.
     """
-    try:
-      return self.bare_git.rev_parse(R_PUB + branch)
-    except GitError:
-      return None
+    key = R_PUB + branch
+    if all is None:
+      try:
+        return self.bare_git.rev_parse(key)
+      except GitError:
+        return None
+    else:
+      try:
+        return all[key]
+      except KeyError:
+        return None
 
-  def CleanPublishedCache(self):
+  def CleanPublishedCache(self, all=None):
     """Prunes any stale published refs.
     """
+    if all is None:
+      all = self._allrefs
     heads = set()
     canrm = {}
-    for name, id in self._allrefs.iteritems():
+    for name, id in all.iteritems():
       if name.startswith(R_HEADS):
         heads.add(name)
       elif name.startswith(R_PUB):
@@ -528,7 +573,6 @@ class Project(object):
       return False
 
     if self.worktree:
-      self._RepairAndroidImportErrors()
       self._InitMRef()
     else:
       self._InitMirrorHead()
@@ -545,58 +589,65 @@ class Project(object):
     for file in self.copyfiles:
       file._Copy()
 
-  def _RepairAndroidImportErrors(self):
-    if self.name in ['platform/external/iptables',
-                     'platform/external/libpcap',
-                     'platform/external/tcpdump',
-                     'platform/external/webkit',
-                     'platform/system/wlan/ti']:
-      # I hate myself for doing this...
-      #
-      # In the initial Android 1.0 release these projects were
-      # shipped, some users got them, and then the history had
-      # to be rewritten to correct problems with their imports.
-      # The 'android-1.0' tag may still be pointing at the old
-      # history, so we need to drop the tag and fetch it again.
-      #
-      try:
-        remote = self.GetRemote(self.remote.name)
-        relname = remote.ToLocal(R_HEADS + 'release-1.0')
-        tagname = R_TAGS + 'android-1.0'
-        if self._revlist(not_rev(relname), tagname):
-          cmd = ['fetch', remote.name, '+%s:%s' % (tagname, tagname)]
-          GitCommand(self, cmd, bare = True).Wait()
-      except GitError:
-        pass
-
-  def Sync_LocalHalf(self):
+  def Sync_LocalHalf(self, syncbuf):
     """Perform only the local IO portion of the sync process.
        Network access is not required.
-
-       Return:
-         True:  the sync was successful
-         False: the sync requires user input
     """
     self._InitWorkTree()
-    self.CleanPublishedCache()
+    all = self.bare_ref.all
+    self.CleanPublishedCache(all)
 
     rem = self.GetRemote(self.remote.name)
     rev = rem.ToLocal(self.revision)
-    branch = self.CurrentBranch
+    if rev in all:
+      revid = all[rev]
+    elif IsId(rev):
+      revid = rev
+    else:
+      try:
+        revid = self.bare_git.rev_parse('--verify', '%s^0' % rev)
+      except GitError:
+        raise ManifestInvalidRevisionError(
+          'revision %s in %s not found' % (self.revision, self.name))
 
-    if branch is None:
+    head = self.work_git.GetHead()
+    if head.startswith(R_HEADS):
+      branch = head[len(R_HEADS):]
+      try:
+        head = all[head]
+      except KeyError:
+        head = None
+    else:
+      branch = None
+
+    if branch is None or syncbuf.detach_head:
       # Currently on a detached HEAD.  The user is assumed to
       # not have any local modifications worth worrying about.
       #
+      if self.IsRebaseInProgress():
+        syncbuf.fail(self, _PriorSyncFailedError())
+        return
+
+      if head == revid:
+        # No changes; don't do anything further.
+        #
+        return
+
       lost = self._revlist(not_rev(rev), HEAD)
       if lost:
-        _info("[%s] Discarding %d commits", self.name, len(lost))
+        syncbuf.info(self, "discarding %d commits", len(lost))
       try:
         self._Checkout(rev, quiet=True)
-      except GitError:
-        return False
+      except GitError, e:
+        syncbuf.fail(self, e)
+        return
       self._CopyFiles()
-      return True
+      return
+
+    if head == revid:
+      # No changes; don't do anything further.
+      #
+      return
 
     branch = self.GetBranch(branch)
     merge = branch.LocalMerge
@@ -605,19 +656,19 @@ class Project(object):
       # The current branch has no tracking configuration.
       # Jump off it to a deatched HEAD.
       #
-      _info("[%s] Leaving %s"
-            " (does not track any upstream)",
-            self.name,
-            branch.name)
+      syncbuf.info(self,
+                   "leaving %s; does not track upstream",
+                   branch.name)
       try:
         self._Checkout(rev, quiet=True)
-      except GitError:
-        return False
+      except GitError, e:
+        syncbuf.fail(self, e)
+        return
       self._CopyFiles()
-      return True
+      return
 
     upstream_gain = self._revlist(not_rev(HEAD), rev)
-    pub = self.WasPublished(branch.name)
+    pub = self.WasPublished(branch.name, all)
     if pub:
       not_merged = self._revlist(not_rev(rev), pub)
       if not_merged:
@@ -626,25 +677,20 @@ class Project(object):
           # commits are not yet merged upstream.  We do not want
           # to rewrite the published commits so we punt.
           #
-          _info("[%s] Branch %s is published,"
-                " but is now %d commits behind.",
-                self.name, branch.name, len(upstream_gain))
-          _info("[%s] Consider merging or rebasing the"
-                " unpublished commits.", self.name)
-        return True
-      elif upstream_gain:
-        # We can fast-forward safely.
+          syncbuf.info(self,
+                       "branch %s is published but is now %d commits behind",
+                       branch.name,
+                       len(upstream_gain))
+        return
+      elif pub == head:
+        # All published commits are merged, and thus we are a
+        # strict subset.  We can fast-forward safely.
         #
-        try:
+        def _doff():
           self._FastForward(rev)
-        except GitError:
-          return False
-        self._CopyFiles()
-        return True
-      else:
-        # Trivially no changes in the upstream.
-        #
-        return True
+          self._CopyFiles()
+        syncbuf.later1(self, _doff)
+        return
 
     if merge == rev:
       try:
@@ -659,8 +705,7 @@ class Project(object):
       # and pray that the old upstream also wasn't in the habit
       # of rebasing itself.
       #
-      _info("[%s] Manifest switched from %s to %s",
-            self.name, merge, rev)
+      syncbuf.info(self, "manifest switched %s...%s", merge, rev)
       old_merge = merge
 
     if rev == old_merge:
@@ -671,19 +716,19 @@ class Project(object):
     if not upstream_lost and not upstream_gain:
       # Trivially no changes caused by the upstream.
       #
-      return True
+      return
 
     if self.IsDirty(consider_untracked=False):
-      _warn('[%s] commit (or discard) uncommitted changes'
-            ' before sync', self.name)
-      return False
+      syncbuf.fail(self, _DirtyError())
+      return
 
     if upstream_lost:
       # Upstream rebased.  Not everything in HEAD
       # may have been caused by the user.
       #
-      _info("[%s] Discarding %d commits removed from upstream",
-            self.name, len(upstream_lost))
+      syncbuf.info(self,
+                   "discarding %d commits removed from upstream",
+                   len(upstream_lost))
 
     branch.remote = rem
     branch.merge = self.revision
@@ -691,29 +736,28 @@ class Project(object):
 
     my_changes = self._revlist(not_rev(old_merge), HEAD)
     if my_changes:
-      try:
+      def _dorebase():
         self._Rebase(upstream = old_merge, onto = rev)
-      except GitError:
-        return False
+        self._CopyFiles()
+      syncbuf.later2(self, _dorebase)
     elif upstream_lost:
       try:
         self._ResetHard(rev)
-      except GitError:
-        return False
+        self._CopyFiles()
+      except GitError, e:
+        syncbuf.fail(self, e)
+        return
     else:
-      try:
+      def _doff():
         self._FastForward(rev)
-      except GitError:
-        return False
+        self._CopyFiles()
+      syncbuf.later1(self, _doff)
 
-    self._CopyFiles()
-    return True
-
-  def AddCopyFile(self, src, dest):
+  def AddCopyFile(self, src, dest, absdest):
     # dest should already be an absolute path, but src is project relative
     # make src an absolute path
-    src = os.path.join(self.worktree, src)
-    self.copyfiles.append(_CopyFile(src, dest))
+    abssrc = os.path.join(self.worktree, src)
+    self.copyfiles.append(_CopyFile(src, dest, abssrc, absdest))
 
   def DownloadPatchSet(self, change_id, patch_id):
     """Download a single patch set of a single change to FETCH_HEAD.
@@ -738,39 +782,135 @@ class Project(object):
   def StartBranch(self, name):
     """Create a new branch off the manifest's revision.
     """
+    head = self.work_git.GetHead()
+    if head == (R_HEADS + name):
+      return True
+
+    all = self.bare_ref.all
+    if (R_HEADS + name) in all:
+      return GitCommand(self,
+                        ['checkout', name, '--'],
+                        capture_stdout = True,
+                        capture_stderr = True).Wait() == 0
+
     branch = self.GetBranch(name)
     branch.remote = self.GetRemote(self.remote.name)
     branch.merge = self.revision
 
     rev = branch.LocalMerge
-    cmd = ['checkout', '-b', branch.name, rev]
-    if GitCommand(self, cmd).Wait() == 0:
-      branch.Save()
+    if rev in all:
+      revid = all[rev]
+    elif IsId(rev):
+      revid = rev
     else:
-      raise GitError('%s checkout %s ' % (self.name, rev))
+      revid = None
+
+    if head.startswith(R_HEADS):
+      try:
+        head = all[head]
+      except KeyError:
+        head = None
+
+    if revid and head and revid == head:
+      ref = os.path.join(self.gitdir, R_HEADS + name)
+      try:
+        os.makedirs(os.path.dirname(ref))
+      except OSError:
+        pass
+      _lwrite(ref, '%s\n' % revid)
+      _lwrite(os.path.join(self.worktree, '.git', HEAD),
+              'ref: %s%s\n' % (R_HEADS, name))
+      branch.Save()
+      return True
+
+    if GitCommand(self,
+                  ['checkout', '-b', branch.name, rev],
+                  capture_stdout = True,
+                  capture_stderr = True).Wait() == 0:
+      branch.Save()
+      return True
+    return False
+
+  def CheckoutBranch(self, name):
+    """Checkout a local topic branch.
+    """
+    rev = R_HEADS + name
+    head = self.work_git.GetHead()
+    if head == rev:
+      # Already on the branch
+      #
+      return True
+
+    all = self.bare_ref.all
+    try:
+      revid = all[rev]
+    except KeyError:
+      # Branch does not exist in this project
+      #
+      return False
+
+    if head.startswith(R_HEADS):
+      try:
+        head = all[head]
+      except KeyError:
+        head = None
+
+    if head == revid:
+      # Same revision; just update HEAD to point to the new
+      # target branch, but otherwise take no other action.
+      #
+      _lwrite(os.path.join(self.worktree, '.git', HEAD),
+              'ref: %s%s\n' % (R_HEADS, name))
+      return True
+
+    return GitCommand(self,
+                      ['checkout', name, '--'],
+                      capture_stdout = True,
+                      capture_stderr = True).Wait() == 0
 
   def AbandonBranch(self, name):
     """Destroy a local topic branch.
     """
-    try:
-      tip_rev = self.bare_git.rev_parse(R_HEADS + name)
-    except GitError:
-      return
+    rev = R_HEADS + name
+    all = self.bare_ref.all
+    if rev not in all:
+      # Doesn't exist; assume already abandoned.
+      #
+      return True
 
-    if self.CurrentBranch == name:
-      self._Checkout(
-        self.GetRemote(self.remote.name).ToLocal(self.revision),
-        quiet=True)
+    head = self.work_git.GetHead()
+    if head == rev:
+      # We can't destroy the branch while we are sitting
+      # on it.  Switch to a detached HEAD.
+      #
+      head = all[head]
 
-    cmd = ['branch', '-D', name]
-    GitCommand(self, cmd, capture_stdout=True).Wait()
+      rev = self.GetRemote(self.remote.name).ToLocal(self.revision)
+      if rev in all:
+        revid = all[rev]
+      elif IsId(rev):
+        revid = rev
+      else:
+        revid = None
+
+      if revid and head == revid:
+        _lwrite(os.path.join(self.worktree, '.git', HEAD),
+                '%s\n' % revid)
+      else:
+        self._Checkout(rev, quiet=True)
+
+    return GitCommand(self,
+                      ['branch', '-D', name],
+                      capture_stdout = True,
+                      capture_stderr = True).Wait() == 0
 
   def PruneHeads(self):
     """Prune any topic branches already merged into upstream.
     """
     cb = self.CurrentBranch
     kill = []
-    for name in self._allrefs.keys():
+    left = self._allrefs
+    for name in left.keys():
       if name.startswith(R_HEADS):
         name = name[len(R_HEADS):]
         if cb is None or name != cb:
@@ -783,14 +923,11 @@ class Project(object):
       self.work_git.DetachHead(HEAD)
       kill.append(cb)
 
-    deleted = set()
     if kill:
-      try:
-        old = self.bare_git.GetHead()
-      except GitError:
+      old = self.bare_git.GetHead()
+      if old is None:
         old = 'refs/heads/please_never_use_this_as_a_branch_name'
 
-      rm_re = re.compile(r"^Deleted branch (.*)\.$")
       try:
         self.bare_git.DetachHead(rev)
 
@@ -802,22 +939,20 @@ class Project(object):
         b.Wait()
       finally:
         self.bare_git.SetHead(old)
+        left = self._allrefs
 
-      for line in b.stdout.split("\n"):
-        m = rm_re.match(line)
-        if m:
-          deleted.add(m.group(1))
-
-      if deleted:
-        self.CleanPublishedCache()
+      for branch in kill:
+        if (R_HEADS + branch) not in left:
+          self.CleanPublishedCache()
+          break
 
     if cb and cb not in kill:
       kill.append(cb)
-      kill.sort()
+    kill.sort()
 
     kept = []
     for branch in kill:
-      if branch not in deleted:
+      if (R_HEADS + branch) in left:
         branch = self.GetBranch(branch)
         base = branch.LocalMerge
         if not base:
@@ -831,11 +966,19 @@ class Project(object):
   def _RemoteFetch(self, name=None):
     if not name:
       name = self.remote.name
+
+    ssh_proxy = False
+    if self.GetRemote(name).PreConnectFetch():
+      ssh_proxy = True
+
     cmd = ['fetch']
     if not self.worktree:
       cmd.append('--update-head-ok')
     cmd.append(name)
-    return GitCommand(self, cmd, bare = True).Wait() == 0
+    return GitCommand(self,
+                      cmd,
+                      bare = True,
+                      ssh_proxy = ssh_proxy).Wait() == 0
 
   def _Checkout(self, rev, quiet=False):
     cmd = ['checkout']
@@ -856,11 +999,11 @@ class Project(object):
       raise GitError('%s reset --hard %s ' % (self.name, rev))
 
   def _Rebase(self, upstream, onto = None):
-    cmd = ['rebase', '-i']
+    cmd = ['rebase']
     if onto is not None:
       cmd.extend(['--onto', onto])
     cmd.append(upstream)
-    if GitCommand(self, cmd, disable_editor=True).Wait() != 0:
+    if GitCommand(self, cmd).Wait() != 0:
       raise GitError('%s rebase %s ' % (self.name, upstream))
 
   def _FastForward(self, head):
@@ -872,7 +1015,11 @@ class Project(object):
     if not os.path.exists(self.gitdir):
       os.makedirs(self.gitdir)
       self.bare_git.init()
-      self.config.SetString('core.bare', None)
+
+      if self.manifest.IsMirror:
+        self.config.SetString('core.bare', 'true')
+      else:
+        self.config.SetString('core.bare', None)
 
       hooks = self._gitdir_path('hooks')
       try:
@@ -938,14 +1085,17 @@ class Project(object):
     if self.manifest.branch:
       msg = 'manifest set to %s' % self.revision
       ref = R_M + self.manifest.branch
+      cur = self.bare_ref.symref(ref)
 
       if IsId(self.revision):
-        dst = self.revision + '^0'
-        self.bare_git.UpdateRef(ref, dst, message = msg, detach = True)
+        if cur != '' or self.bare_ref.get(ref) != self.revision:
+          dst = self.revision + '^0'
+          self.bare_git.UpdateRef(ref, dst, message = msg, detach = True)
       else:
         remote = self.GetRemote(self.remote.name)
         dst = remote.ToLocal(self.revision)
-        self.bare_git.symbolic_ref('-m', msg, ref, dst)
+        if cur != dst:
+          self.bare_git.symbolic_ref('-m', msg, ref, dst)
 
   def _InitMirrorHead(self):
     dst = self.GetRemote(self.remote.name).ToLocal(self.revision)
@@ -980,9 +1130,7 @@ class Project(object):
       rev = self.GetRemote(self.remote.name).ToLocal(self.revision)
       rev = self.bare_git.rev_parse('%s^0' % rev)
 
-      f = open(os.path.join(dotgit, HEAD), 'wb')
-      f.write("%s\n" % rev)
-      f.close()
+      _lwrite(os.path.join(dotgit, HEAD), '%s\n' % rev)
 
       cmd = ['read-tree', '--reset', '-u']
       cmd.append('-v')
@@ -1001,32 +1149,13 @@ class Project(object):
 
   @property
   def _allrefs(self):
-    return self.bare_git.ListRefs()
+    return self.bare_ref.all
 
   class _GitGetByExec(object):
     def __init__(self, project, bare):
       self._project = project
       self._bare = bare
 
-    def ListRefs(self, *args):
-      cmdv = ['for-each-ref', '--format=%(objectname) %(refname)']
-      cmdv.extend(args)
-      p = GitCommand(self._project,
-                     cmdv,
-                     bare = self._bare,
-                     capture_stdout = True,
-                     capture_stderr = True)
-      r = {}
-      for line in p.process.stdout:
-        id, name = line[:-1].split(' ', 2)
-        r[name] = id
-      if p.Wait() != 0:
-        raise GitError('%s for-each-ref %s: %s' % (
-                       self._project.name,
-                       str(args),
-                       p.stderr))
-      return r
-
     def LsOthers(self):
       p = GitCommand(self._project,
                      ['ls-files',
@@ -1092,7 +1221,18 @@ class Project(object):
         p.Wait()
 
     def GetHead(self):
-      return self.symbolic_ref(HEAD)
+      if self._bare:
+        path = os.path.join(self._project.gitdir, HEAD)
+      else:
+        path = os.path.join(self._project.worktree, '.git', HEAD)
+      fd = open(path, 'rb')
+      try:
+        line = fd.read()
+      finally:
+        fd.close()
+      if line.startswith('ref: '):
+        return line[5:-1]
+      return line[:-1]
 
     def SetHead(self, ref, message=None):
       cmdv = []
@@ -1128,6 +1268,7 @@ class Project(object):
       if not old:
         old = self.rev_parse(name)
       self.update_ref('-d', name, old)
+      self._project.bare_ref.deleted(name)
 
     def rev_list(self, *args):
       cmdv = ['rev-list']
@@ -1169,6 +1310,113 @@ class Project(object):
       return runner
 
 
+class _PriorSyncFailedError(Exception):
+  def __str__(self):
+    return 'prior sync failed; rebase still in progress'
+
+class _DirtyError(Exception):
+  def __str__(self):
+    return 'contains uncommitted changes'
+
+class _InfoMessage(object):
+  def __init__(self, project, text):
+    self.project = project
+    self.text = text
+
+  def Print(self, syncbuf):
+    syncbuf.out.info('%s/: %s', self.project.relpath, self.text)
+    syncbuf.out.nl()
+
+class _Failure(object):
+  def __init__(self, project, why):
+    self.project = project
+    self.why = why
+
+  def Print(self, syncbuf):
+    syncbuf.out.fail('error: %s/: %s',
+                     self.project.relpath,
+                     str(self.why))
+    syncbuf.out.nl()
+
+class _Later(object):
+  def __init__(self, project, action):
+    self.project = project
+    self.action = action
+
+  def Run(self, syncbuf):
+    out = syncbuf.out
+    out.project('project %s/', self.project.relpath)
+    out.nl()
+    try:
+      self.action()
+      out.nl()
+      return True
+    except GitError, e:
+      out.nl()
+      return False
+
+class _SyncColoring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, 'reposync')
+    self.project   = self.printer('header', attr = 'bold')
+    self.info      = self.printer('info')
+    self.fail      = self.printer('fail', fg='red')
+
+class SyncBuffer(object):
+  def __init__(self, config, detach_head=False):
+    self._messages = []
+    self._failures = []
+    self._later_queue1 = []
+    self._later_queue2 = []
+
+    self.out = _SyncColoring(config)
+    self.out.redirect(sys.stderr)
+
+    self.detach_head = detach_head
+    self.clean = True
+
+  def info(self, project, fmt, *args):
+    self._messages.append(_InfoMessage(project, fmt % args))
+
+  def fail(self, project, err=None):
+    self._failures.append(_Failure(project, err))
+    self.clean = False
+
+  def later1(self, project, what):
+    self._later_queue1.append(_Later(project, what))
+
+  def later2(self, project, what):
+    self._later_queue2.append(_Later(project, what))
+
+  def Finish(self):
+    self._PrintMessages()
+    self._RunLater()
+    self._PrintMessages()
+    return self.clean
+
+  def _RunLater(self):
+    for q in ['_later_queue1', '_later_queue2']:
+      if not self._RunQueue(q):
+        return
+
+  def _RunQueue(self, queue):
+    for m in getattr(self, queue):
+      if not m.Run(self):
+        self.clean = False
+        return False
+    setattr(self, queue, [])
+    return True
+
+  def _PrintMessages(self):
+    for m in self._messages:
+      m.Print(self)
+    for m in self._failures:
+      m.Print(self)
+
+    self._messages = []
+    self._failures = []
+
+
 class MetaProject(Project):
   """A special project housed under .repo.
   """
@@ -1191,11 +1439,37 @@ class MetaProject(Project):
         if base:
           self.revision = base
 
+  @property
+  def LastFetch(self):
+    try:
+      fh = os.path.join(self.gitdir, 'FETCH_HEAD')
+      return os.path.getmtime(fh)
+    except OSError:
+      return 0
+
   @property
   def HasChanges(self):
     """Has the remote received new commits not yet checked out?
     """
+    if not self.remote or not self.revision:
+      return False
+
+    all = self.bare_ref.all
     rev = self.GetRemote(self.remote.name).ToLocal(self.revision)
-    if self._revlist(not_rev(HEAD), rev):
+    if rev in all:
+      revid = all[rev]
+    else:
+      revid = rev
+
+    head = self.work_git.GetHead()
+    if head.startswith(R_HEADS):
+      try:
+        head = all[head]
+      except KeyError:
+        head = None
+
+    if revid == head:
+      return False
+    elif self._revlist(not_rev(HEAD), rev):
       return True
     return False

repo

@@ -120,7 +120,7 @@ group.add_option('--mirror',
                  help='mirror the forrest')
 
 # Tool
-group = init_optparse.add_option_group('Version options')
+group = init_optparse.add_option_group('repo Version options')
 group.add_option('--repo-url',
                  dest='repo_url',
                  help='repo repository location', metavar='URL')

subcmds/abandon.py

@@ -16,6 +16,7 @@
 import sys
 from command import Command
 from git_command import git
+from progress import Progress
 
 class Abandon(Command):
   common = True
@@ -38,5 +39,23 @@ It is equivalent to "git branch -D <branchname>".
       print >>sys.stderr, "error: '%s' is not a valid name" % nb
       sys.exit(1)
 
-    for project in self.GetProjects(args[1:]):
-      project.AbandonBranch(nb)
+    nb = args[0]
+    err = []
+    all = self.GetProjects(args[1:])
+
+    pm = Progress('Abandon %s' % nb, len(all))
+    for project in all:
+      pm.update()
+      if not project.AbandonBranch(nb):
+        err.append(project)
+    pm.end()
+
+    if err:
+      if len(err) == len(all):
+        print >>sys.stderr, 'error: no project has branch %s' % nb
+      else:
+        for p in err:
+          print >>sys.stderr,\
+            "error: %s/: cannot abandon %s" \
+            % (p.relpath, nb)
+      sys.exit(1)

subcmds/branches.py

@@ -0,0 +1,154 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from color import Coloring
+from command import Command
+
+class BranchColoring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, 'branch')
+    self.current = self.printer('current', fg='green')
+    self.local   = self.printer('local')
+    self.notinproject = self.printer('notinproject', fg='red')
+
+class BranchInfo(object):
+  def __init__(self, name):
+    self.name = name
+    self.current = 0
+    self.published = 0
+    self.published_equal = 0
+    self.projects = []
+
+  def add(self, b):
+    if b.current:
+      self.current += 1
+    if b.published:
+      self.published += 1
+    if b.revision == b.published:
+      self.published_equal += 1
+    self.projects.append(b)
+
+  @property
+  def IsCurrent(self):
+    return self.current > 0
+
+  @property
+  def IsPublished(self):
+    return self.published > 0
+
+  @property
+  def IsPublishedEqual(self):
+    return self.published_equal == len(self.projects)
+
+
+class Branches(Command):
+  common = True
+  helpSummary = "View current topic branches"
+  helpUsage = """
+%prog [<project>...]
+
+Summarizes the currently available topic branches.
+"""
+
+  def _Options(self, p):
+    p.add_option('-a', '--all',
+                 dest='all', action='store_true',
+                 help='show all branches, not just the majority')
+
+  def Execute(self, opt, args):
+    projects = self.GetProjects(args)
+    out = BranchColoring(self.manifest.manifestProject.config)
+    all = {}
+    project_cnt = len(projects)
+
+    for project in projects:
+      for name, b in project.GetBranches().iteritems():
+        b.project = project
+        if name not in all:
+          all[name] = BranchInfo(name)
+        all[name].add(b)
+
+    names = all.keys()
+    names.sort()
+
+    if not opt.all and not args:
+      # No -a and no specific projects listed; try to filter the
+      # results down to only the majority of projects.
+      #
+      n = []
+      for name in names:
+        i = all[name]
+        if i.IsCurrent \
+        or 80 <= (100 * len(i.projects)) / project_cnt:
+          n.append(name)
+      names = n
+
+    if not names:
+      print >>sys.stderr, '   (no branches)'
+      return
+
+    width = 25
+    for name in names:
+      if width < len(name):
+        width = len(name)
+
+    for name in names:
+      i = all[name]
+      in_cnt = len(i.projects)
+
+      if i.IsCurrent:
+        current = '*'
+        hdr = out.current
+      else:
+        current = ' '
+        hdr = out.local
+
+      if i.IsPublishedEqual:
+        published = 'P'
+      elif i.IsPublished:
+        published = 'p'
+      else:
+        published = ' '
+
+      hdr('%c%c %-*s' % (current, published, width, name))
+      out.write(' |')
+
+      if in_cnt < project_cnt and (in_cnt == 1 or opt.all):
+        fmt = out.write
+        paths = []
+        if in_cnt < project_cnt - in_cnt: 
+          type = 'in'
+          for b in i.projects:
+            paths.append(b.project.relpath)
+        else:
+          fmt = out.notinproject
+          type = 'not in'
+          have = set()
+          for b in i.projects:
+            have.add(b.project)
+          for p in projects:
+            paths.append(p.relpath)
+
+        s = ' %s %s' % (type, ', '.join(paths))
+        if width + 7 + len(s) < 80:
+          fmt(s)
+        else:
+          out.nl()
+          fmt('    %s:' % type)
+          for p in paths:
+            out.nl()
+            fmt('      %s' % p)
+      out.nl()

subcmds/checkout.py

@@ -0,0 +1,58 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from command import Command
+from progress import Progress
+
+class Checkout(Command):
+  common = True
+  helpSummary = "Checkout a branch for development"
+  helpUsage = """
+%prog <branchname> [<project>...]
+"""
+  helpDescription = """
+The '%prog' command checks out an existing branch that was previously
+created by 'repo start'.
+
+The command is equivalent to:
+
+  repo forall [<project>...] -c git checkout <branchname>
+"""
+
+  def Execute(self, opt, args):
+    if not args:
+      self.Usage()
+
+    nb = args[0]
+    err = []
+    all = self.GetProjects(args[1:])
+
+    pm = Progress('Checkout %s' % nb, len(all))
+    for project in all:
+      pm.update()
+      if not project.CheckoutBranch(nb):
+        err.append(project)
+    pm.end()
+
+    if err:
+      if len(err) == len(all):
+        print >>sys.stderr, 'error: no project has branch %s' % nb
+      else:
+        for p in err:
+          print >>sys.stderr,\
+            "error: %s/: cannot checkout %s" \
+            % (p.relpath, nb)
+      sys.exit(1)

subcmds/forall.py

@@ -13,13 +13,30 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import fcntl
 import re
 import os
+import select
 import sys
 import subprocess
-from command import Command
 
-class Forall(Command):
+from color import Coloring
+from command import Command, MirrorSafeCommand
+
+_CAN_COLOR = [
+  'branch',
+  'diff',
+  'grep',
+  'log',
+]
+
+class ForallColoring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, 'forall')
+    self.project = self.printer('project', attr='bold')
+
+
+class Forall(Command, MirrorSafeCommand):
   common = False
   helpSummary = "Run a shell command in each project"
   helpUsage = """
@@ -28,17 +45,48 @@ class Forall(Command):
   helpDescription = """
 Executes the same shell command in each project.
 
+Output Formatting
+-----------------
+
+The -p option causes '%prog' to bind pipes to the command's stdin,
+stdout and stderr streams, and pipe all output into a continuous
+stream that is displayed in a single pager session.  Project headings
+are inserted before the output of each command is displayed.  If the
+command produces no output in a project, no heading is displayed.
+
+The formatting convention used by -p is very suitable for some
+types of searching, e.g. `repo forall -p -c git log -SFoo` will
+print all commits that add or remove references to Foo.
+
+The -v option causes '%prog' to display stderr messages if a
+command produces output only on stderr.  Normally the -p option
+causes command output to be suppressed until the command produces
+at least one byte of output on stdout.
+
 Environment
 -----------
-pwd is the project's working directory.
+
+pwd is the project's working directory.  If the current client is
+a mirror client, then pwd is the Git repository.
 
 REPO_PROJECT is set to the unique name of the project.
 
+REPO_PATH is the path relative the the root of the client.
+
+REPO_REMOTE is the name of the remote system from the manifest.
+
+REPO_LREV is the name of the revision from the manifest, translated
+to a local tracking branch.  If you need to pass the manifest
+revision to a locally executed git command, use REPO_LREV.
+
+REPO_RREV is the name of the revision from the manifest, exactly
+as written in the manifest.
+
 shell positional arguments ($1, $2, .., $#) are set to any arguments
 following <command>.
 
-stdin, stdout, stderr are inherited from the terminal and are
-not redirected.
+Unless -p is used, stdin, stdout, stderr are inherited from the
+terminal and are not redirected.
 """
 
   def _Options(self, p):
@@ -52,6 +100,17 @@ not redirected.
                  action='callback',
                  callback=cmd)
 
+    g = p.add_option_group('Output')
+    g.add_option('-p',
+                 dest='project_header', action='store_true',
+                 help='Show project headers before output')
+    g.add_option('-v', '--verbose',
+                 dest='verbose', action='store_true',
+                 help='Show command error messages')
+
+  def WantPager(self, opt):
+    return opt.project_header
+
   def Execute(self, opt, args):
     if not opt.command:
       self.Usage()
@@ -66,15 +125,122 @@ not redirected.
       cmd.append(cmd[0])
     cmd.extend(opt.command[1:])
 
+    if  opt.project_header \
+    and not shell \
+    and cmd[0] == 'git':
+      # If this is a direct git command that can enable colorized
+      # output and the user prefers coloring, add --color into the
+      # command line because we are going to wrap the command into
+      # a pipe and git won't know coloring should activate.
+      #
+      for cn in cmd[1:]:
+        if not cn.startswith('-'):
+          break
+      if cn in _CAN_COLOR:
+        class ColorCmd(Coloring):
+          def __init__(self, config, cmd):
+            Coloring.__init__(self, config, cmd)
+        if ColorCmd(self.manifest.manifestProject.config, cn).is_on:
+          cmd.insert(cmd.index(cn) + 1, '--color')
+
+    mirror = self.manifest.IsMirror
+    out = ForallColoring(self.manifest.manifestProject.config)
+    out.redirect(sys.stderr)
+
     rc = 0
+    first = True
+
     for project in self.GetProjects(args):
       env = dict(os.environ.iteritems())
-      env['REPO_PROJECT'] = project.name
+      def setenv(name, val):
+        if val is None:
+          val = ''
+        env[name] = val
+
+      setenv('REPO_PROJECT', project.name)
+      setenv('REPO_PATH', project.relpath)
+      setenv('REPO_REMOTE', project.remote.name)
+      setenv('REPO_LREV', project\
+        .GetRemote(project.remote.name)\
+        .ToLocal(project.revision))
+      setenv('REPO_RREV', project.revision)
+
+      if mirror:
+        setenv('GIT_DIR', project.gitdir)
+        cwd = project.gitdir
+      else:
+        cwd = project.worktree
+
+      if opt.project_header:
+        stdin = subprocess.PIPE
+        stdout = subprocess.PIPE
+        stderr = subprocess.PIPE
+      else:
+        stdin = None
+        stdout = None
+        stderr = None
 
       p = subprocess.Popen(cmd,
-                           cwd = project.worktree,
+                           cwd = cwd,
                            shell = shell,
-                           env = env)
+                           env = env,
+                           stdin = stdin,
+                           stdout = stdout,
+                           stderr = stderr)
+
+      if opt.project_header:
+        class sfd(object):
+          def __init__(self, fd, dest):
+            self.fd = fd
+            self.dest = dest
+          def fileno(self):
+            return self.fd.fileno()
+
+        empty = True
+        didout = False
+        errbuf = ''
+
+        p.stdin.close()
+        s_in = [sfd(p.stdout, sys.stdout),
+                sfd(p.stderr, sys.stderr)]
+
+        for s in s_in:
+          flags = fcntl.fcntl(s.fd, fcntl.F_GETFL)
+          fcntl.fcntl(s.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+        while s_in:
+          in_ready, out_ready, err_ready = select.select(s_in, [], [])
+          for s in in_ready:
+            buf = s.fd.read(4096)
+            if not buf:
+              s.fd.close()
+              s_in.remove(s)
+              continue
+
+            if not opt.verbose:
+              if s.fd == p.stdout:
+                didout = True
+              else:
+                errbuf += buf
+                continue
+
+            if empty:
+              if first:
+                first = False
+              else:
+                out.nl()
+              out.project('project %s/', project.relpath)
+              out.nl()
+              out.flush()
+              if errbuf:
+                sys.stderr.write(errbuf)
+                sys.stderr.flush()
+                errbuf = ''
+              empty = False
+
+            s.dest.write(buf)
+            s.dest.flush()
+
       r = p.wait()
       if r != 0 and r != rc:
         rc = r

subcmds/grep.py

@@ -0,0 +1,243 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from optparse import SUPPRESS_HELP
+from color import Coloring
+from command import PagedCommand
+from git_command import GitCommand
+
+class GrepColoring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, 'grep')
+    self.project = self.printer('project', attr='bold')
+
+class Grep(PagedCommand):
+  common = True
+  helpSummary = "Print lines matching a pattern"
+  helpUsage = """
+%prog {pattern | -e pattern} [<project>...]
+"""
+  helpDescription = """
+Search for the specified patterns in all project files.
+
+Boolean Options
+---------------
+
+The following options can appear as often as necessary to express
+the pattern to locate:
+
+ -e PATTERN
+ --and, --or, --not, -(, -)
+
+Further, the -r/--revision option may be specified multiple times
+in order to scan multiple trees.  If the same file matches in more
+than one tree, only the first result is reported, prefixed by the
+revision name it was found under.
+
+Examples
+-------
+
+Look for a line that has '#define' and either 'MAX_PATH or 'PATH_MAX':
+
+  repo grep -e '#define' --and -\( -e MAX_PATH -e PATH_MAX \)
+
+Look for a line that has 'NODE' or 'Unexpected' in files that
+contain a line that matches both expressions:
+
+  repo grep --all-match -e NODE -e Unexpected
+
+"""
+
+  def _Options(self, p):
+    def carry(option,
+              opt_str,
+              value,
+              parser):
+      pt = getattr(parser.values, 'cmd_argv', None)
+      if pt is None:
+        pt = []
+        setattr(parser.values, 'cmd_argv', pt)
+
+      if opt_str == '-(':
+        pt.append('(')
+      elif opt_str == '-)':
+        pt.append(')')
+      else:
+        pt.append(opt_str)
+
+      if value is not None:
+        pt.append(value)
+
+    g = p.add_option_group('Sources')
+    g.add_option('--cached',
+                 action='callback', callback=carry,
+                 help='Search the index, instead of the work tree')
+    g.add_option('-r','--revision',
+                 dest='revision', action='append', metavar='TREEish',
+                 help='Search TREEish, instead of the work tree')
+
+    g = p.add_option_group('Pattern')
+    g.add_option('-e',
+                 action='callback', callback=carry,
+                 metavar='PATTERN', type='str',
+                 help='Pattern to search for')
+    g.add_option('-i', '--ignore-case',
+                 action='callback', callback=carry,
+                 help='Ignore case differences')
+    g.add_option('-a','--text',
+                 action='callback', callback=carry,
+                 help="Process binary files as if they were text")
+    g.add_option('-I',
+                 action='callback', callback=carry,
+                 help="Don't match the pattern in binary files")
+    g.add_option('-w', '--word-regexp',
+                 action='callback', callback=carry,
+                 help='Match the pattern only at word boundaries')
+    g.add_option('-v', '--invert-match',
+                 action='callback', callback=carry,
+                 help='Select non-matching lines')
+    g.add_option('-G', '--basic-regexp',
+                 action='callback', callback=carry,
+                 help='Use POSIX basic regexp for patterns (default)')
+    g.add_option('-E', '--extended-regexp',
+                 action='callback', callback=carry,
+                 help='Use POSIX extended regexp for patterns')
+    g.add_option('-F', '--fixed-strings',
+                 action='callback', callback=carry,
+                 help='Use fixed strings (not regexp) for pattern')
+
+    g = p.add_option_group('Pattern Grouping')
+    g.add_option('--all-match',
+                 action='callback', callback=carry,
+                 help='Limit match to lines that have all patterns')
+    g.add_option('--and', '--or', '--not',
+                 action='callback', callback=carry,
+                 help='Boolean operators to combine patterns')
+    g.add_option('-(','-)',
+                 action='callback', callback=carry,
+                 help='Boolean operator grouping')
+
+    g = p.add_option_group('Output')
+    g.add_option('-n',
+                 action='callback', callback=carry,
+                 help='Prefix the line number to matching lines')
+    g.add_option('-C',
+                 action='callback', callback=carry,
+                 metavar='CONTEXT', type='str',
+                 help='Show CONTEXT lines around match')
+    g.add_option('-B',
+                 action='callback', callback=carry,
+                 metavar='CONTEXT', type='str',
+                 help='Show CONTEXT lines before match')
+    g.add_option('-A',
+                 action='callback', callback=carry,
+                 metavar='CONTEXT', type='str',
+                 help='Show CONTEXT lines after match')
+    g.add_option('-l','--name-only','--files-with-matches',
+                 action='callback', callback=carry,
+                 help='Show only file names containing matching lines')
+    g.add_option('-L','--files-without-match',
+                 action='callback', callback=carry,
+                 help='Show only file names not containing matching lines')
+
+
+  def Execute(self, opt, args):
+    out = GrepColoring(self.manifest.manifestProject.config)
+
+    cmd_argv = ['grep']
+    if out.is_on:
+      cmd_argv.append('--color')
+    cmd_argv.extend(getattr(opt,'cmd_argv',[]))
+
+    if '-e' not in cmd_argv:
+      if not args:
+        self.Usage()
+      cmd_argv.append('-e')
+      cmd_argv.append(args[0])
+      args = args[1:]
+
+    projects = self.GetProjects(args)
+
+    full_name = False
+    if len(projects) > 1:
+      cmd_argv.append('--full-name')
+      full_name = True
+
+    have_rev = False
+    if opt.revision:
+      if '--cached' in cmd_argv:
+        print >>sys.stderr,\
+          'fatal: cannot combine --cached and --revision'
+        sys.exit(1)
+      have_rev = True
+      cmd_argv.extend(opt.revision)
+    cmd_argv.append('--')
+
+    bad_rev = False
+    have_match = False
+
+    for project in projects:
+      p = GitCommand(project,
+                     cmd_argv,
+                     bare = False,
+                     capture_stdout = True,
+                     capture_stderr = True)
+      if p.Wait() != 0:
+        # no results
+        #
+        if p.stderr:
+          if have_rev and 'fatal: ambiguous argument' in p.stderr:
+            bad_rev = True
+          else:
+            out.project('--- project %s ---' % project.relpath)
+            out.nl()
+            out.write(p.stderr)
+            out.nl()
+        continue
+      have_match = True
+
+      # We cut the last element, to avoid a blank line.
+      #
+      r = p.stdout.split('\n')
+      r = r[0:-1]
+
+      if have_rev and full_name:
+        for line in r:
+          rev, line = line.split(':', 1)
+          out.write(rev)
+          out.write(':')
+          out.project(project.relpath)
+          out.write('/')
+          out.write(line)
+          out.nl()
+      elif full_name:
+        for line in r:
+          out.project(project.relpath)
+          out.write('/')
+          out.write(line)
+          out.nl()
+      else:
+        for line in r:
+          print line
+
+    if have_match:
+      sys.exit(0)
+    elif have_rev and bad_rev:
+      for r in opt.revision:
+        print >>sys.stderr, "error: can't search revision %s" % r
+      sys.exit(1)
+    else:
+      sys.exit(1)

subcmds/help.py

@@ -13,13 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import re
 import sys
 from formatter import AbstractFormatter, DumbWriter
 
 from color import Coloring
-from command import PagedCommand
+from command import PagedCommand, MirrorSafeCommand
 
-class Help(PagedCommand):
+class Help(PagedCommand, MirrorSafeCommand):
   common = False
   helpSummary = "Display detailed help on a command"
   helpUsage = """
@@ -77,6 +78,7 @@ The most commonly used repo commands are:
       print fmt % (name, summary)
     print """
 See 'repo help <command>' for more information on a specific command.
+See 'repo help --all' for a complete list of recognized commands.
 """
 
   def _PrintCommandHelp(self, cmd):
@@ -105,19 +107,39 @@ See 'repo help <command>' for more information on a specific command.
         body = body.strip()
         body = body.replace('%prog', me)
 
+        asciidoc_hdr = re.compile(r'^\n?([^\n]{1,})\n([=~-]{2,})$')
         for para in body.split("\n\n"):
           if para.startswith(' '):
             self.write('%s', para)
             self.nl()
             self.nl()
-          else:
-            self.wrap.add_flowing_data(para)
-            self.wrap.end_paragraph(1)
+            continue
+
+          m = asciidoc_hdr.match(para)
+          if m:
+            title = m.group(1)
+            type = m.group(2)
+            if type[0] in ('=', '-'):
+              p = self.heading
+            else:
+              def _p(fmt, *args):
+                self.write('  ')
+                self.heading(fmt, *args)
+              p = _p
+
+            p('%s', title)
+            self.nl()
+            p('%s', ''.ljust(len(title),type[0]))
+            self.nl()
+            continue
+
+          self.wrap.add_flowing_data(para)
+          self.wrap.end_paragraph(1)
         self.wrap.end_paragraph(0)
 
     out = _Out(self.manifest.globalConfig)
-    cmd.OptionParser.print_help()
     out._PrintSection('Summary', 'helpSummary')
+    cmd.OptionParser.print_help()
     out._PrintSection('Description', 'helpDescription')
 
   def _Options(self, p):

subcmds/init.py

@@ -17,12 +17,13 @@ import os
 import sys
 
 from color import Coloring
-from command import InteractiveCommand
+from command import InteractiveCommand, MirrorSafeCommand
 from error import ManifestParseError
 from remote import Remote
+from project import SyncBuffer
 from git_command import git, MIN_GIT_VERSION
 
-class Init(InteractiveCommand):
+class Init(InteractiveCommand, MirrorSafeCommand):
   common = True
   helpSummary = "Initialize repo in the current directory"
   helpUsage = """
@@ -34,9 +35,20 @@ The latest repo source code and manifest collection is downloaded
 from the server and is installed in the .repo/ directory in the
 current working directory.
 
-The optional <manifest> argument can be used to specify an alternate
-manifest to be used. If no manifest is specified, the manifest
-default.xml will be used.
+The optional -b argument can be used to select the manifest branch
+to checkout and use.  If no branch is specified, master is assumed.
+
+The optional -m argument can be used to specify an alternate manifest
+to be used. If no manifest is specified, the manifest default.xml
+will be used.
+
+Switching Manifest Branches
+---------------------------
+
+To switch to another manifest branch, `repo init -b otherbranch`
+may be used in an existing client.  However, as this only updates the
+manifest, a subsequent `repo sync` (or `repo sync -d`) is necessary
+to update the working directory files.
 """
 
   def _Options(self, p):
@@ -63,7 +75,7 @@ default.xml will be used.
 
 
     # Tool
-    g = p.add_option_group('Version options')
+    g = p.add_option_group('repo Version options')
     g.add_option('--repo-url',
                  dest='repo_url',
                  help='repo repository location', metavar='URL')
@@ -89,8 +101,9 @@ default.xml will be used.
 
   def _SyncManifest(self, opt):
     m = self.manifest.manifestProject
+    is_new = not m.Exists
 
-    if not m.Exists:
+    if is_new:
       if not opt.manifest_url:
         print >>sys.stderr, 'fatal: manifest url (-u) is required.'
         sys.exit(1)
@@ -117,11 +130,25 @@ default.xml will be used.
       r.Save()
 
     if opt.mirror:
-      m.config.SetString('repo.mirror', 'true')
+      if is_new:
+        m.config.SetString('repo.mirror', 'true')
+      else:
+        print >>sys.stderr, 'fatal: --mirror not supported on existing client'
+        sys.exit(1)
 
-    m.Sync_NetworkHalf()
-    m.Sync_LocalHalf()
-    m.StartBranch('default')
+    if not m.Sync_NetworkHalf():
+      r = m.GetRemote(m.remote.name)
+      print >>sys.stderr, 'fatal: cannot obtain manifest %s' % r.url
+      sys.exit(1)
+
+    syncbuf = SyncBuffer(m.config)
+    m.Sync_LocalHalf(syncbuf)
+    syncbuf.Finish()
+
+    if is_new or m.CurrentBranch is None:
+      if not m.StartBranch('default'):
+        print >>sys.stderr, 'fatal: cannot create default in manifest'
+        sys.exit(1)
 
   def _LinkManifest(self, name):
     if not name:
@@ -192,11 +219,11 @@ default.xml will be used.
     self._SyncManifest(opt)
     self._LinkManifest(opt.manifest_name)
 
-    if os.isatty(0) and os.isatty(1) and not opt.mirror:
+    if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
       self._ConfigureUser()
       self._ConfigureColor()
 
-    if opt.mirror:
+    if self.manifest.IsMirror:
       type = 'mirror '
     else:
       type = ''

subcmds/manifest.py

@@ -0,0 +1,77 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+from command import PagedCommand
+
+class Manifest(PagedCommand):
+  common = False
+  helpSummary = "Manifest inspection utility"
+  helpUsage = """
+%prog [-o {-|NAME.xml} [-r]]
+"""
+  _helpDescription = """
+
+With the -o option, exports the current manifest for inspection.
+The manifest and (if present) local_manifest.xml are combined
+together to produce a single manifest file.  This file can be stored
+in a Git repository for use during future 'repo init' invocations.
+
+"""
+
+  @property
+  def helpDescription(self):
+    help = self._helpDescription + '\n'
+    r = os.path.dirname(__file__)
+    r = os.path.dirname(r)
+    fd = open(os.path.join(r, 'docs', 'manifest-format.txt'))
+    for line in fd:
+      help += line
+    fd.close()
+    return help
+
+  def _Options(self, p):
+    p.add_option('-r', '--revision-as-HEAD',
+                 dest='peg_rev', action='store_true',
+                 help='Save revisions as current HEAD')
+    p.add_option('-o', '--output-file',
+                 dest='output_file',
+                 help='File to save the manifest to',
+                 metavar='-|NAME.xml')
+
+  def _Output(self, opt):
+    if opt.output_file == '-':
+      fd = sys.stdout
+    else:
+      fd = open(opt.output_file, 'w')
+    self.manifest.Save(fd,
+                       peg_rev = opt.peg_rev)
+    fd.close()
+    if opt.output_file != '-':
+      print >>sys.stderr, 'Saved manifest to %s' % opt.output_file
+
+  def Execute(self, opt, args):
+    if args:
+      self.Usage()
+
+    if opt.output_file is not None:
+      self._Output(opt)
+      return
+
+    print >>sys.stderr, 'error: no operation to perform'
+    print >>sys.stderr, 'error: see repo help manifest'
+    sys.exit(1)

subcmds/selfupdate.py

@@ -0,0 +1,60 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from optparse import SUPPRESS_HELP
+import sys
+
+from command import Command, MirrorSafeCommand
+from subcmds.sync import _PostRepoUpgrade
+from subcmds.sync import _PostRepoFetch
+
+class Selfupdate(Command, MirrorSafeCommand):
+  common = False
+  helpSummary = "Update repo to the latest version"
+  helpUsage = """
+%prog
+"""
+  helpDescription = """
+The '%prog' command upgrades repo to the latest version, if a
+newer version is available.
+
+Normally this is done automatically by 'repo sync' and does not
+need to be performed by an end-user.
+"""
+
+  def _Options(self, p):
+    g = p.add_option_group('repo Version options')
+    g.add_option('--no-repo-verify',
+                 dest='no_repo_verify', action='store_true',
+                 help='do not verify repo source code')
+    g.add_option('--repo-upgraded',
+                 dest='repo_upgraded', action='store_true',
+                 help=SUPPRESS_HELP)
+
+  def Execute(self, opt, args):
+    rp = self.manifest.repoProject
+    rp.PreSync()
+
+    if opt.repo_upgraded:
+      _PostRepoUpgrade(self.manifest)
+
+    else:
+      if not rp.Sync_NetworkHalf():
+        print >>sys.stderr, "error: can't update repo"
+        sys.exit(1)
+
+      _PostRepoFetch(rp,
+                     no_repo_verify = opt.no_repo_verify,
+                     verbose = True)

subcmds/stage.py

@@ -55,12 +55,12 @@ The '%prog' command stages files to prepare the next commit.
 
     out = _ProjectList(self.manifest.manifestProject.config)
     while True:
-      out.header('        %-20s %s', 'project', 'path')
+      out.header('        %s', 'project')
       out.nl()
 
       for i in xrange(0, len(all)):
         p = all[i]
-        out.write('%3d:    %-20s %s', i + 1, p.name, p.relpath + '/')
+        out.write('%3d:    %s', i + 1, p.relpath + '/')
         out.nl()
       out.nl()
 

subcmds/start.py

@@ -16,26 +16,17 @@
 import sys
 from command import Command
 from git_command import git
+from progress import Progress
 
 class Start(Command):
   common = True
   helpSummary = "Start a new branch for development"
   helpUsage = """
 %prog <newbranchname> [<project>...]
-
-This subcommand starts a new branch of development that is automatically
-pulled from a remote branch.
-
-It is equivalent to the following git commands:
-
-"git branch --track <newbranchname> m/<codeline>",
-or 
-"git checkout --track -b <newbranchname> m/<codeline>".
-
-All three forms set up the config entries that repo bases some of its
-processing on.  Use %prog or git branch or checkout with --track to ensure
-the configuration data is set up properly.
-
+"""
+  helpDescription = """
+'%prog' begins a new branch of development, starting from the
+revision specified in the manifest.
 """
 
   def Execute(self, opt, args):
@@ -47,5 +38,19 @@ the configuration data is set up properly.
       print >>sys.stderr, "error: '%s' is not a valid name" % nb
       sys.exit(1)
 
-    for project in self.GetProjects(args[1:]):
-      project.StartBranch(nb)
+    err = []
+    all = self.GetProjects(args[1:])
+
+    pm = Progress('Starting %s' % nb, len(all))
+    for project in all:
+      pm.update()
+      if not project.StartBranch(nb):
+        err.append(project)
+    pm.end()
+
+    if err:
+      for p in err:
+        print >>sys.stderr,\
+          "error: %s/: cannot start %s" \
+          % (p.relpath, nb)
+      sys.exit(1)

subcmds/status.py

@@ -20,8 +20,66 @@ class Status(PagedCommand):
   helpSummary = "Show the working tree status"
   helpUsage = """
 %prog [<project>...]
+"""
+  helpDescription = """
+'%prog' compares the working tree to the staging area (aka index),
+and the most recent commit on this branch (HEAD), in each project
+specified.  A summary is displayed, one line per file where there
+is a difference between these three states.
+
+Status Display
+--------------
+
+The status display is organized into three columns of information,
+for example if the file 'subcmds/status.py' is modified in the
+project 'repo' on branch 'devwork':
+
+  project repo/                                   branch devwork
+   -m     subcmds/status.py
+
+The first column explains how the staging area (index) differs from
+the last commit (HEAD).  Its values are always displayed in upper
+case and have the following meanings:
+
+ -:  no difference
+ A:  added         (not in HEAD,     in index                     )
+ M:  modified      (    in HEAD,     in index, different content  )
+ D:  deleted       (    in HEAD, not in index                     )
+ R:  renamed       (not in HEAD,     in index, path changed       )
+ C:  copied        (not in HEAD,     in index, copied from another)
+ T:  mode changed  (    in HEAD,     in index, same content       )
+ U:  unmerged; conflict resolution required
+
+The second column explains how the working directory differs from
+the index.  Its values are always displayed in lower case and have
+the following meanings:
+
+ -:  new / unknown (not in index,     in work tree                )
+ m:  modified      (    in index,     in work tree, modified      )
+ d:  deleted       (    in index, not in work tree                )
+
 """
 
   def Execute(self, opt, args):
-    for project in self.GetProjects(args):
-      project.PrintWorkTreeStatus()
+    all = self.GetProjects(args)
+    clean = 0
+
+    on = {}
+    for project in all:
+      cb = project.CurrentBranch
+      if cb:
+        if cb not in on:
+          on[cb] = []
+        on[cb].append(project)
+
+    branch_names = list(on.keys())
+    branch_names.sort()
+    for cb in branch_names:
+      print '# on branch %s' % cb
+
+    for project in all:
+      state = project.PrintWorkTreeStatus()
+      if state == 'CLEAN':
+        clean += 1
+    if len(all) == clean:
+      print 'nothing to commit (working directory clean)'

subcmds/sync.py

@@ -13,17 +13,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from optparse import SUPPRESS_HELP
 import os
 import re
 import subprocess
 import sys
+import time
 
 from git_command import GIT
-from command import Command
+from project import HEAD
+from command import Command, MirrorSafeCommand
 from error import RepoChangedException, GitError
 from project import R_HEADS
+from project import SyncBuffer
+from progress import Progress
 
-class Sync(Command):
+class Sync(Command, MirrorSafeCommand):
   common = True
   helpSummary = "Update working tree to the latest revision"
   helpUsage = """
@@ -43,27 +48,83 @@ line.  Projects can be specified either by name, or by a relative
 or absolute path to the project's local directory. If no projects
 are specified, '%prog' will synchronize all projects listed in
 the manifest.
+
+The -d/--detach option can be used to switch specified projects
+back to the manifest revision.  This option is especially helpful
+if the project is currently on a topic branch, but the manifest
+revision is temporarily needed.
+
+SSH Connections
+---------------
+
+If at least one project remote URL uses an SSH connection (ssh://,
+git+ssh://, or user@host:path syntax) repo will automatically
+enable the SSH ControlMaster option when connecting to that host.
+This feature permits other projects in the same '%prog' session to
+reuse the same SSH tunnel, saving connection setup overheads.
+
+To disable this behavior on UNIX platforms, set the GIT_SSH
+environment variable to 'ssh'.  For example:
+
+  export GIT_SSH=ssh
+  %prog
+
+Compatibility
+~~~~~~~~~~~~~
+
+This feature is automatically disabled on Windows, due to the lack
+of UNIX domain socket support.
+
+This feature is not compatible with url.insteadof rewrites in the
+user's ~/.gitconfig.  '%prog' is currently not able to perform the
+rewrite early enough to establish the ControlMaster tunnel.
+
+If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
+later is required to fix a server side protocol bug.
+
 """
 
   def _Options(self, p):
-    p.add_option('--no-repo-verify',
+    p.add_option('-l','--local-only',
+                 dest='local_only', action='store_true',
+                 help="only update working tree, don't fetch")
+    p.add_option('-n','--network-only',
+                 dest='network_only', action='store_true',
+                 help="fetch only, don't update working tree")
+    p.add_option('-d','--detach',
+                 dest='detach_head', action='store_true',
+                 help='detach projects back to manifest revision')
+
+    g = p.add_option_group('repo Version options')
+    g.add_option('--no-repo-verify',
                  dest='no_repo_verify', action='store_true',
                  help='do not verify repo source code')
-    p.add_option('--repo-upgraded',
+    g.add_option('--repo-upgraded',
                  dest='repo_upgraded', action='store_true',
-                 help='perform additional actions after a repo upgrade')
+                 help=SUPPRESS_HELP)
 
-  def _Fetch(self, *projects):
+  def _Fetch(self, projects):
     fetched = set()
+    pm = Progress('Fetching projects', len(projects))
     for project in projects:
+      pm.update()
+
       if project.Sync_NetworkHalf():
         fetched.add(project.gitdir)
       else:
         print >>sys.stderr, 'error: Cannot fetch %s' % project.name
         sys.exit(1)
+    pm.end()
     return fetched
 
   def Execute(self, opt, args):
+    if opt.network_only and opt.detach_head:
+      print >>sys.stderr, 'error: cannot combine -n and -d'
+      sys.exit(1)
+    if opt.network_only and opt.local_only:
+      print >>sys.stderr, 'error: cannot combine -n and -l'
+      sys.exit(1)
+
     rp = self.manifest.repoProject
     rp.PreSync()
 
@@ -71,42 +132,73 @@ the manifest.
     mp.PreSync()
 
     if opt.repo_upgraded:
-      for project in self.manifest.projects.values():
-        if project.Exists:
-          project.PostRepoUpgrade()
+      _PostRepoUpgrade(self.manifest)
 
     all = self.GetProjects(args, missing_ok=True)
-    fetched = self._Fetch(rp, mp, *all)
 
-    if rp.HasChanges:
-      print >>sys.stderr, 'info: A new version of repo is available'
-      print >>sys.stderr, ''
-      if opt.no_repo_verify or _VerifyTag(rp):
-        if not rp.Sync_LocalHalf():
+    if not opt.local_only:
+      to_fetch = []
+      now = time.time()
+      if (24 * 60 * 60) <= (now - rp.LastFetch):
+        to_fetch.append(rp)
+      to_fetch.append(mp)
+      to_fetch.extend(all)
+
+      fetched = self._Fetch(to_fetch)
+      _PostRepoFetch(rp, opt.no_repo_verify)
+      if opt.network_only:
+        # bail out now; the rest touches the working tree
+        return
+
+      if mp.HasChanges:
+        syncbuf = SyncBuffer(mp.config)
+        mp.Sync_LocalHalf(syncbuf)
+        if not syncbuf.Finish():
           sys.exit(1)
-        print >>sys.stderr, 'info: Restarting repo with latest version'
-        raise RepoChangedException(['--repo-upgraded'])
-      else:
-        print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
 
-    if mp.HasChanges:
-      if not mp.Sync_LocalHalf():
-        sys.exit(1)
-
-      self.manifest._Unload()
-      all = self.GetProjects(args, missing_ok=True)
-      missing = []
-      for project in all:
-        if project.gitdir not in fetched:
-          missing.append(project)
-      self._Fetch(*missing)
+        self.manifest._Unload()
+        all = self.GetProjects(args, missing_ok=True)
+        missing = []
+        for project in all:
+          if project.gitdir not in fetched:
+            missing.append(project)
+        self._Fetch(missing)
 
+    syncbuf = SyncBuffer(mp.config,
+                         detach_head = opt.detach_head)
+    pm = Progress('Syncing work tree', len(all))
     for project in all:
+      pm.update()
       if project.worktree:
-        if not project.Sync_LocalHalf():
-          sys.exit(1)
+        project.Sync_LocalHalf(syncbuf)
+    pm.end()
+    print >>sys.stderr
+    if not syncbuf.Finish():
+      sys.exit(1)
 
 
+def _PostRepoUpgrade(manifest):
+  for project in manifest.projects.values():
+    if project.Exists:
+      project.PostRepoUpgrade()
+
+def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
+  if rp.HasChanges:
+    print >>sys.stderr, 'info: A new version of repo is available'
+    print >>sys.stderr, ''
+    if no_repo_verify or _VerifyTag(rp):
+      syncbuf = SyncBuffer(rp.config)
+      rp.Sync_LocalHalf(syncbuf)
+      if not syncbuf.Finish():
+        sys.exit(1)
+      print >>sys.stderr, 'info: Restarting repo with latest version'
+      raise RepoChangedException(['--repo-upgraded'])
+    else:
+      print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
+  else:
+    if verbose:
+      print >>sys.stderr, 'repo version %s is current' % rp.work_git.describe(HEAD)
+
 def _VerifyTag(project):
   gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
   if not os.path.exists(gpg_dir):

subcmds/upload.py

@@ -38,22 +38,21 @@ class Upload(InteractiveCommand):
 %prog [--re --cc] {[<project>]... | --replace <project>}
 """
   helpDescription = """
-The '%prog' command is used to send changes to the Gerrit code
-review system.  It searches for changes in local projects that do
-not yet exist in the corresponding remote repository.  If multiple
-changes are found, '%prog' opens an editor to allow the
-user to choose which change to upload.  After a successful upload,
-repo prints the URL for the change in the Gerrit code review system.
+The '%prog' command is used to send changes to the Gerrit Code
+Review system.  It searches for topic branches in local projects
+that have not yet been published for review.  If multiple topic
+branches are found, '%prog' opens an editor to allow the user to
+select which branches to upload.
 
-'%prog' searches for uploadable changes in all projects listed
-at the command line.  Projects can be specified either by name, or
-by a relative or absolute path to the project's local directory. If
-no projects are specified, '%prog' will search for uploadable
-changes in all projects listed in the manifest.
+'%prog' searches for uploadable changes in all projects listed at
+the command line.  Projects can be specified either by name, or by
+a relative or absolute path to the project's local directory. If no
+projects are specified, '%prog' will search for uploadable changes
+in all projects listed in the manifest.
 
 If the --reviewers or --cc options are passed, those emails are
 added to the respective list of users, and emails are sent to any
-new users.  Users passed to --reviewers must be already registered
+new users.  Users passed as --reviewers must already be registered
 with the code review system, or the upload will fail.
 
 If the --replace option is passed the user can designate which
@@ -61,6 +60,33 @@ existing change(s) in Gerrit match up to the commits in the branch
 being uploaded.  For each matched pair of change,commit the commit
 will be added as a new patch set, completely replacing the set of
 files and description associated with the change in Gerrit.
+
+Configuration
+-------------
+
+review.URL.autoupload:
+
+To disable the "Upload ... (y/n)?" prompt, you can set a per-project
+or global Git configuration option.  If review.URL.autoupload is set
+to "true" then repo will assume you always answer "y" at the prompt,
+and will not prompt you further.  If it is set to "false" then repo
+will assume you always answer "n", and will abort.
+
+The URL must match the review URL listed in the manifest XML file,
+or in the .git/config within the project.  For example:
+
+  [remote "origin"]
+    url = git://git.example.com/project.git
+    review = http://review.example.com/
+
+  [review "http://review.example.com/"]
+    autoupload = true
+
+References
+----------
+
+Gerrit Code Review:  http://code.google.com/p/gerrit/
+
 """
 
   def _Options(self, p):
@@ -77,21 +103,32 @@ files and description associated with the change in Gerrit.
   def _SingleBranch(self, branch, people):
     project = branch.project
     name = branch.name
-    date = branch.date
-    list = branch.commits
+    remote = project.GetBranch(name).remote
 
-    print 'Upload project %s/:' % project.relpath
-    print '  branch %s (%2d commit%s, %s):' % (
-                  name,
-                  len(list),
-                  len(list) != 1 and 's' or '',
-                  date)
-    for commit in list:
-      print '         %s' % commit
+    key = 'review.%s.autoupload' % remote.review
+    answer = project.config.GetBoolean(key)
 
-    sys.stdout.write('(y/n)? ')
-    answer = sys.stdin.readline().strip()
-    if answer in ('y', 'Y', 'yes', '1', 'true', 't'):
+    if answer is False:
+      _die("upload blocked by %s = false" % key)
+
+    if answer is None:
+      date = branch.date
+      list = branch.commits
+
+      print 'Upload project %s/:' % project.relpath
+      print '  branch %s (%2d commit%s, %s):' % (
+                    name,
+                    len(list),
+                    len(list) != 1 and 's' or '',
+                    date)
+      for commit in list:
+        print '         %s' % commit
+
+      sys.stdout.write('to %s (y/n)? ' % remote.review)
+      answer = sys.stdin.readline().strip()
+      answer = answer in ('y', 'Y', 'yes', '1', 'true', 't')
+
+    if answer:
       self._UploadAndReport([branch], people)
     else:
       _die("upload aborted by user")
@@ -234,9 +271,6 @@ files and description associated with the change in Gerrit.
           print >>sys.stderr, '[OK    ] %-15s %s' % (
                  branch.project.relpath + '/',
                  branch.name)
-          print >>sys.stderr, '%s' % branch.tip_url
-          print >>sys.stderr, '(as %s)' % branch.owner_email
-          print >>sys.stderr, ''
 
     if have_errors:
       sys.exit(1)

subcmds/version.py

@@ -0,0 +1,35 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from command import Command, MirrorSafeCommand
+from git_command import git
+from project import HEAD
+
+class Version(Command, MirrorSafeCommand):
+  common = False
+  helpSummary = "Display the version of repo"
+  helpUsage = """
+%prog
+"""
+
+  def Execute(self, opt, args):
+    rp = self.manifest.repoProject
+    rem = rp.GetRemote(rp.remote.name)
+
+    print 'repo version %s' % rp.work_git.describe(HEAD)
+    print '       (from %s)' % rem.url
+    print git.version().strip()
+    print 'Python %s' % sys.version

trace.py

@@ -0,0 +1,34 @@
+#
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import os
+REPO_TRACE = 'REPO_TRACE'
+
+try:
+  _TRACE = os.environ[REPO_TRACE] == '1'
+except KeyError:
+  _TRACE = False
+
+def IsTrace():
+  return _TRACE
+
+def SetTrace():
+  global _TRACE
+  _TRACE = True
+
+def Trace(fmt, *args):
+  if IsTrace():
+    print >>sys.stderr, fmt % args