Change repo sync to be more friendly when updating the tree

We now try to sync all projects that can be done safely first, before
we start rebasing user commits over the upstream.  This has the nice
effect of making the local tree as close to the upstream as possible
before the user has to start resolving merge conflicts, as that extra
information in other projects may aid in the conflict resolution.

Informational output is buffered and delayed until calculation for
all projects has been done, so that the user gets one concise list
of notice messages, rather than it interrupting the progress meter.

Fast-forward output is now prefixed with the project header, so the
user can see which project that update is taking place in, and make
some relation of the diffstat back to the project name.

Rebase output is now prefixed with the project header, so that if
the rebase fails, the user can see which project we were operating
on and can try to address the failure themselves.

Since rebase sits on a detached HEAD, we now look for an in-progress
rebase during sync, so we can alert the user that the given project
is in a state we cannot handle.

Signed-off-by: Shawn O. Pearce <sop@google.com>

color.py

@@ -100,6 +100,9 @@ class Coloring(object):
     else:
       self._on = False
 
+  def redirect(self, out):
+    self._out = out
+
   @property
   def is_on(self):
     return self._on

command.py

@@ -114,3 +114,8 @@ class PagedCommand(Command):
   """Command which defaults to output in a pager, as its
      display tends to be larger than one screen full.
   """
+
+class MirrorSafeCommand(object):
+  """Command permits itself to run within a mirror,
+     and does not require a working directory.
+  """

docs/manifest-format.txt

@@ -19,39 +19,39 @@ XML File Format
 A manifest XML file (e.g. 'default.xml') roughly conforms to the
 following DTD:
 
-<!DOCTYPE manifest [
-  <!ELEMENT manifest (remote*,
-                      default?,
-                      remove-project*,
-                      project*,
-                      add-remote*)>
-
-  <!ELEMENT remote (EMPTY)>
-  <!ATTLIST remote name         ID    #REQUIRED>
-  <!ATTLIST remote fetch        CDATA #REQUIRED>
-  <!ATTLIST remote review       CDATA #IMPLIED>
-  <!ATTLIST remote project-name CDATA #IMPLIED>
-
-  <!ELEMENT default (EMPTY)>
-  <!ATTLIST default remote   IDREF #IMPLIED>
-  <!ATTLIST default revision CDATA #IMPLIED>
-
-  <!ELEMENT project (remote*)>
-  <!ATTLIST project name     CDATA #REQUIRED>
-  <!ATTLIST project path     CDATA #IMPLIED>
-  <!ATTLIST project remote   IDREF #IMPLIED>
-  <!ATTLIST project revision CDATA #IMPLIED>
-
-  <!ELEMENT add-remote (EMPTY)>
-  <!ATTLIST add-remote to-project   ID    #REQUIRED>
-  <!ATTLIST add-remote name         ID    #REQUIRED>
-  <!ATTLIST add-remote fetch        CDATA #REQUIRED>
-  <!ATTLIST add-remote review       CDATA #IMPLIED>
-  <!ATTLIST add-remote project-name CDATA #IMPLIED>
-
-  <!ELEMENT remove-project (EMPTY)>
-  <!ATTLIST remove-project name  CDATA #REQUIRED>
-]>
+  <!DOCTYPE manifest [
+    <!ELEMENT manifest (remote*,
+                        default?,
+                        remove-project*,
+                        project*,
+                        add-remote*)>
+  
+    <!ELEMENT remote (EMPTY)>
+    <!ATTLIST remote name         ID    #REQUIRED>
+    <!ATTLIST remote fetch        CDATA #REQUIRED>
+    <!ATTLIST remote review       CDATA #IMPLIED>
+    <!ATTLIST remote project-name CDATA #IMPLIED>
+  
+    <!ELEMENT default (EMPTY)>
+    <!ATTLIST default remote   IDREF #IMPLIED>
+    <!ATTLIST default revision CDATA #IMPLIED>
+  
+    <!ELEMENT project (remote*)>
+    <!ATTLIST project name     CDATA #REQUIRED>
+    <!ATTLIST project path     CDATA #IMPLIED>
+    <!ATTLIST project remote   IDREF #IMPLIED>
+    <!ATTLIST project revision CDATA #IMPLIED>
+  
+    <!ELEMENT add-remote (EMPTY)>
+    <!ATTLIST add-remote to-project   ID    #REQUIRED>
+    <!ATTLIST add-remote name         ID    #REQUIRED>
+    <!ATTLIST add-remote fetch        CDATA #REQUIRED>
+    <!ATTLIST add-remote review       CDATA #IMPLIED>
+    <!ATTLIST add-remote project-name CDATA #IMPLIED>
+  
+    <!ELEMENT remove-project (EMPTY)>
+    <!ATTLIST remove-project name  CDATA #REQUIRED>
+  ]>
 
 A description of the elements and their attributes follows.
 
@@ -179,16 +179,14 @@ manifest, stored in `$TOP_DIR/.repo/local_manifest.xml`.
 
 For example:
 
-----
-	$ cat .repo/local_manifest.xml
-	<?xml version="1.0" encoding="UTF-8"?>
-	<manifest>
-	  <project path="manifest"
-			   name="tools/manifest" />
-	  <project path="platform-manifest"
-			   name="platform/manifest" />
-	</manifest>
-----
+  $ cat .repo/local_manifest.xml
+  <?xml version="1.0" encoding="UTF-8"?>
+  <manifest>
+    <project path="manifest"
+             name="tools/manifest" />
+    <project path="platform-manifest"
+             name="platform/manifest" />
+  </manifest>
 
 Users may add projects to the local manifest prior to a `repo sync`
 invocation, instructing repo to automatically download and manage

error.py

@@ -17,6 +17,10 @@ class ManifestParseError(Exception):
   """Failed to parse the manifest file.
   """
 
+class ManifestInvalidRevisionError(Exception):
+  """The revision value in a project is incorrect.
+  """
+
 class EditorError(Exception):
   """Unspecified error from the user's text editor.
   """

git_config.py

@@ -24,6 +24,8 @@ R_HEADS = 'refs/heads/'
 R_TAGS  = 'refs/tags/'
 ID_RE = re.compile('^[0-9a-f]{40}$')
 
+REVIEW_CACHE = dict()
+
 def IsId(rev):
   return ID_RE.match(rev)
 
@@ -273,24 +275,44 @@ class Remote(object):
       u = self.review
       if not u.startswith('http:') and not u.startswith('https:'):
         u = 'http://%s' % u
-      if not u.endswith('/'):
-        u += '/'
-      u += 'ssh_info'
+      if u.endswith('/Gerrit'):
+        u = u[:len(u) - len('/Gerrit')]
+      if not u.endswith('/ssh_info'):
+        if not u.endswith('/'):
+          u += '/'
+        u += 'ssh_info'
 
-      try:
-        info = urlopen(u).read()
-        if info == 'NOT_AVAILABLE':
-          raise UploadError('Upload over ssh unavailable')
+      if u in REVIEW_CACHE:
+        info = REVIEW_CACHE[u]
+        self._review_protocol = info[0]
+        self._review_host = info[1]
+        self._review_port = info[2]
+      else:
+        try:
+          info = urlopen(u).read()
+          if info == 'NOT_AVAILABLE':
+            raise UploadError('Upload over ssh unavailable')
+          if '<' in info:
+            # Assume the server gave us some sort of HTML
+            # response back, like maybe a login page.
+            #
+            raise UploadError('Cannot read %s:\n%s' % (u, info))
 
-        self._review_protocol = 'ssh'
-        self._review_host = info.split(" ")[0]
-        self._review_port = info.split(" ")[1]
+          self._review_protocol = 'ssh'
+          self._review_host = info.split(" ")[0]
+          self._review_port = info.split(" ")[1]
+        except HTTPError, e:
+          if e.code == 404:
+            self._review_protocol = 'http-post'
+            self._review_host = None
+            self._review_port = None
+          else:
+            raise UploadError('Cannot guess Gerrit version')
 
-      except HTTPError, e:
-        if e.code == 404:
-          self._review_protocol = 'http-post'
-        else:
-          raise UploadError('Cannot guess Gerrit version')
+        REVIEW_CACHE[u] = (
+          self._review_protocol,
+          self._review_host,
+          self._review_port)
     return self._review_protocol
 
   def SshReviewUrl(self, userEmail):

main.py

@@ -27,8 +27,12 @@ import os
 import re
 import sys
 
-from command import InteractiveCommand, PagedCommand
+import git_command
+from command import InteractiveCommand
+from command import MirrorSafeCommand
+from command import PagedCommand
 from editor import Editor
+from error import ManifestInvalidRevisionError
 from error import NoSuchProjectError
 from error import RepoChangedException
 from manifest import Manifest
@@ -45,6 +49,12 @@ global_options.add_option('-p', '--paginate',
 global_options.add_option('--no-pager',
                           dest='no_pager', action='store_true',
                           help='disable the pager')
+global_options.add_option('--trace',
+                          dest='trace', action='store_true',
+                          help='trace git command execution')
+global_options.add_option('--version',
+                          dest='show_version', action='store_true',
+                          help='display this version of repo')
 
 class _Repo(object):
   def __init__(self, repodir):
@@ -68,6 +78,15 @@ class _Repo(object):
       argv = []
     gopts, gargs = global_options.parse_args(glob)
 
+    if gopts.trace:
+      git_command.TRACE = True
+    if gopts.show_version:
+      if name == 'help':
+        name = 'version'
+      else:
+        print >>sys.stderr, 'fatal: invalid usage of --version'
+        sys.exit(1)
+
     try:
       cmd = self.commands[name]
     except KeyError:
@@ -80,6 +99,12 @@ class _Repo(object):
     cmd.manifest = Manifest(cmd.repodir)
     Editor.globalConfig = cmd.manifest.globalConfig
 
+    if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror:
+      print >>sys.stderr, \
+            "fatal: '%s' requires a working directory"\
+            % name
+      sys.exit(1)
+
     if not gopts.no_pager and not isinstance(cmd, InteractiveCommand):
       config = cmd.manifest.globalConfig
       if gopts.pager:
@@ -94,6 +119,9 @@ class _Repo(object):
     copts, cargs = cmd.OptionParser.parse_args(argv)
     try:
       cmd.Execute(copts, cargs)
+    except ManifestInvalidRevisionError, e:
+      print >>sys.stderr, 'error: %s' % str(e)
+      sys.exit(1)
     except NoSuchProjectError, e:
       if e.name:
         print >>sys.stderr, 'error: project %s not found' % e.name

manifest.py

@@ -18,7 +18,7 @@ import sys
 import xml.dom.minidom
 
 from git_config import GitConfig, IsId
-from project import Project, MetaProject, R_HEADS
+from project import Project, MetaProject, R_HEADS, HEAD
 from remote import Remote
 from error import ManifestParseError
 
@@ -73,6 +73,76 @@ class Manifest(object):
     except OSError, e:
       raise ManifestParseError('cannot link manifest %s' % name)
 
+  def _RemoteToXml(self, r, doc, root):
+    e = doc.createElement('remote')
+    root.appendChild(e)
+    e.setAttribute('name', r.name)
+    e.setAttribute('fetch', r.fetchUrl)
+    if r.reviewUrl is not None:
+      e.setAttribute('review', r.reviewUrl)
+    if r.projectName is not None:
+      e.setAttribute('project-name', r.projectName)
+
+  def Save(self, fd, peg_rev=False):
+    """Write the current manifest out to the given file descriptor.
+    """
+    doc = xml.dom.minidom.Document()
+    root = doc.createElement('manifest')
+    doc.appendChild(root)
+
+    d = self.default
+    sort_remotes = list(self.remotes.keys())
+    sort_remotes.sort()
+
+    for r in sort_remotes:
+      self._RemoteToXml(self.remotes[r], doc, root)
+    if self.remotes:
+      root.appendChild(doc.createTextNode(''))
+
+    have_default = False
+    e = doc.createElement('default')
+    if d.remote:
+      have_default = True
+      e.setAttribute('remote', d.remote.name)
+    if d.revision:
+      have_default = True
+      e.setAttribute('revision', d.revision)
+    if have_default:
+      root.appendChild(e)
+      root.appendChild(doc.createTextNode(''))
+
+    sort_projects = list(self.projects.keys())
+    sort_projects.sort()
+
+    for p in sort_projects:
+      p = self.projects[p]
+      e = doc.createElement('project')
+      root.appendChild(e)
+      e.setAttribute('name', p.name)
+      if p.relpath != p.name:
+        e.setAttribute('path', p.relpath)
+      if not d.remote or p.remote.name != d.remote.name:
+        e.setAttribute('remote', p.remote.name)
+      if peg_rev:
+        if self.IsMirror:
+          e.setAttribute('revision',
+                         p.bare_git.rev_parse(p.revision + '^0'))
+        else:
+          e.setAttribute('revision',
+                         p.work_git.rev_parse(HEAD + '^0'))
+      elif not d.revision or p.revision != d.revision:
+        e.setAttribute('revision', p.revision)
+
+      for r in p.extraRemotes:
+        self._RemoteToXml(p.extraRemotes[r], doc, e)
+      for c in p.copyfiles:
+        ce = doc.createElement('copyfile')
+        ce.setAttribute('src', c.src)
+        ce.setAttribute('dest', c.dest)
+        e.appendChild(ce)
+
+    doc.writexml(fd, '', '  ', '\n', 'UTF-8')
+
   @property
   def projects(self):
     self._Load()
@@ -324,7 +394,7 @@ class Manifest(object):
     if not self.IsMirror:
       # src is project relative;
       # dest is relative to the top of the tree
-      project.AddCopyFile(src, os.path.join(self.topdir, dest))
+      project.AddCopyFile(src, dest, os.path.join(self.topdir, dest))
 
   def _get_remote(self, node):
     name = node.getAttribute('remote')

pager.py

@@ -22,7 +22,7 @@ active = False
 def RunPager(globalConfig):
   global active
 
-  if not os.isatty(0):
+  if not os.isatty(0) or not os.isatty(1):
     return
   pager = _SelectPager(globalConfig)
   if pager == '' or pager == 'cat':

progress.py

@@ -0,0 +1,58 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+class Progress(object):
+  def __init__(self, title, total=0):
+    self._title = title
+    self._total = total
+    self._done = 0
+    self._lastp = -1
+
+  def update(self, inc=1):
+    self._done += inc
+
+    if self._total <= 0:
+      sys.stderr.write('\r%s: %d, ' % (
+        self._title,
+        self._done))
+      sys.stderr.flush()
+    else:
+      p = (100 * self._done) / self._total
+
+      if self._lastp != p:
+        self._lastp = p
+        sys.stderr.write('\r%s: %3d%% (%d/%d)  ' % (
+          self._title,
+          p,
+          self._done,
+          self._total))
+        sys.stderr.flush()
+
+  def end(self):
+    if self._total <= 0:
+      sys.stderr.write('\r%s: %d, done.  \n' % (
+        self._title,
+        self._done))
+      sys.stderr.flush()
+    else:
+      p = (100 * self._done) / self._total
+      sys.stderr.write('\r%s: %3d%% (%d/%d), done.  \n' % (
+        self._title,
+        p,
+        self._done,
+        self._total))
+      sys.stderr.flush()

project.py

@@ -25,6 +25,7 @@ from color import Coloring
 from git_command import GitCommand
 from git_config import GitConfig, IsId
 from error import GitError, ImportError, UploadError
+from error import ManifestInvalidRevisionError
 from remote import Remote
 
 HEAD    = 'HEAD'
@@ -33,13 +34,9 @@ R_TAGS  = 'refs/tags/'
 R_PUB   = 'refs/published/'
 R_M     = 'refs/remotes/m/'
 
-def _warn(fmt, *args):
+def _error(fmt, *args):
   msg = fmt % args
-  print >>sys.stderr, 'warn: %s' % msg
-
-def _info(fmt, *args):
-  msg = fmt % args
-  print >>sys.stderr, 'info: %s' % msg
+  print >>sys.stderr, 'error: %s' % msg
 
 def not_rev(r):
   return '^' + r
@@ -177,13 +174,15 @@ class DiffColoring(Coloring):
 
 
 class _CopyFile:
-  def __init__(self, src, dest):
+  def __init__(self, src, dest, abssrc, absdest):
     self.src = src
     self.dest = dest
+    self.abs_src = abssrc
+    self.abs_dest = absdest
 
   def _Copy(self):
-    src = self.src
-    dest = self.dest
+    src = self.abs_src
+    dest = self.abs_dest
     # copy file if it does not exist or is out of date
     if not os.path.exists(dest) or not filecmp.cmp(src, dest):
       try:
@@ -196,9 +195,7 @@ class _CopyFile:
         mode = mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
         os.chmod(dest, mode)
       except IOError:
-           print >>sys.stderr, \
-              'error: Cannot copy file %s to %s' \
-              % (src, dest)
+        _error('Cannot copy file %s to %s', src, dest)
 
 
 class Project(object):
@@ -303,6 +300,32 @@ class Project(object):
     """
     return self.config.GetBranch(name)
 
+  def GetBranches(self):
+    """Get all existing local branches.
+    """
+    current = self.CurrentBranch
+    all = self.bare_git.ListRefs()
+    heads = {}
+    pubd = {}
+
+    for name, id in all.iteritems():
+      if name.startswith(R_HEADS):
+        name = name[len(R_HEADS):]
+        b = self.GetBranch(name)
+        b.current = name == current
+        b.published = None
+        b.revision = id
+        heads[name] = b
+
+    for name, id in all.iteritems():
+      if name.startswith(R_PUB):
+        name = name[len(R_PUB):]
+        b = heads.get(name)
+        if b:
+          b.published = id
+
+    return heads
+
 
 ## Status Display ##
 
@@ -323,7 +346,7 @@ class Project(object):
     df = self.work_git.DiffZ('diff-files')
     do = self.work_git.LsOthers()
     if not di and not df and not do:
-      return
+      return 'CLEAN'
 
     out = StatusColoring(self.config)
     out.project('project %-40s', self.relpath + '/')
@@ -357,7 +380,7 @@ class Project(object):
       else: f_status = '-'
 
       if i and i.src_path:
-        line = ' %s%s\t%s => (%s%%)' % (i_status, f_status,
+        line = ' %s%s\t%s => %s (%s%%)' % (i_status, f_status,
                                         i.src_path, p, i.level)
       else:
         line = ' %s%s\t%s' % (i_status, f_status, p)
@@ -371,6 +394,7 @@ class Project(object):
       else:
         out.write('%s', line)
       out.nl()
+    return 'DIRTY'
 
   def PrintWorkTreeDiff(self):
     """Prints the status of the repository to stdout.
@@ -528,7 +552,6 @@ class Project(object):
       return False
 
     if self.worktree:
-      self._RepairAndroidImportErrors()
       self._InitMRef()
     else:
       self._InitMirrorHead()
@@ -545,58 +568,42 @@ class Project(object):
     for file in self.copyfiles:
       file._Copy()
 
-  def _RepairAndroidImportErrors(self):
-    if self.name in ['platform/external/iptables',
-                     'platform/external/libpcap',
-                     'platform/external/tcpdump',
-                     'platform/external/webkit',
-                     'platform/system/wlan/ti']:
-      # I hate myself for doing this...
-      #
-      # In the initial Android 1.0 release these projects were
-      # shipped, some users got them, and then the history had
-      # to be rewritten to correct problems with their imports.
-      # The 'android-1.0' tag may still be pointing at the old
-      # history, so we need to drop the tag and fetch it again.
-      #
-      try:
-        remote = self.GetRemote(self.remote.name)
-        relname = remote.ToLocal(R_HEADS + 'release-1.0')
-        tagname = R_TAGS + 'android-1.0'
-        if self._revlist(not_rev(relname), tagname):
-          cmd = ['fetch', remote.name, '+%s:%s' % (tagname, tagname)]
-          GitCommand(self, cmd, bare = True).Wait()
-      except GitError:
-        pass
-
-  def Sync_LocalHalf(self):
+  def Sync_LocalHalf(self, syncbuf):
     """Perform only the local IO portion of the sync process.
        Network access is not required.
-
-       Return:
-         True:  the sync was successful
-         False: the sync requires user input
     """
     self._InitWorkTree()
     self.CleanPublishedCache()
 
     rem = self.GetRemote(self.remote.name)
     rev = rem.ToLocal(self.revision)
+    try:
+      self.bare_git.rev_parse('--verify', '%s^0' % rev)
+    except GitError:
+      raise ManifestInvalidRevisionError(
+        'revision %s in %s not found' % (self.revision, self.name))
+
     branch = self.CurrentBranch
 
-    if branch is None:
+    if branch is None or syncbuf.detach_head:
       # Currently on a detached HEAD.  The user is assumed to
       # not have any local modifications worth worrying about.
       #
+      if os.path.exists(os.path.join(self.worktree, '.dotest')) \
+      or os.path.exists(os.path.join(self.worktree, '.git', 'rebase-apply')):
+        syncbuf.fail(self, _PriorSyncFailedError())
+        return
+
       lost = self._revlist(not_rev(rev), HEAD)
       if lost:
-        _info("[%s] Discarding %d commits", self.name, len(lost))
+        syncbuf.info(self, "discarding %d commits", len(lost))
       try:
         self._Checkout(rev, quiet=True)
-      except GitError:
-        return False
+      except GitError, e:
+        syncbuf.fail(self, e)
+        return
       self._CopyFiles()
-      return True
+      return
 
     branch = self.GetBranch(branch)
     merge = branch.LocalMerge
@@ -605,16 +612,16 @@ class Project(object):
       # The current branch has no tracking configuration.
       # Jump off it to a deatched HEAD.
       #
-      _info("[%s] Leaving %s"
-            " (does not track any upstream)",
-            self.name,
-            branch.name)
+      syncbuf.info(self,
+                   "leaving %s; does not track upstream",
+                   branch.name)
       try:
         self._Checkout(rev, quiet=True)
-      except GitError:
-        return False
+      except GitError, e:
+        syncbuf.fail(self, e)
+        return
       self._CopyFiles()
-      return True
+      return
 
     upstream_gain = self._revlist(not_rev(HEAD), rev)
     pub = self.WasPublished(branch.name)
@@ -626,25 +633,24 @@ class Project(object):
           # commits are not yet merged upstream.  We do not want
           # to rewrite the published commits so we punt.
           #
-          _info("[%s] Branch %s is published,"
-                " but is now %d commits behind.",
-                self.name, branch.name, len(upstream_gain))
-          _info("[%s] Consider merging or rebasing the"
-                " unpublished commits.", self.name)
-        return True
+          syncbuf.info(self,
+                       "branch %s is published but is now %d commits behind",
+                       branch.name,
+                       len(upstream_gain))
+          syncbuf.info(self, "consider merging or rebasing the unpublished commits")
+        return
       elif upstream_gain:
         # We can fast-forward safely.
         #
-        try:
+        def _doff():
           self._FastForward(rev)
-        except GitError:
-          return False
-        self._CopyFiles()
-        return True
+          self._CopyFiles()
+        syncbuf.later1(self, _doff)
+        return
       else:
         # Trivially no changes in the upstream.
         #
-        return True
+        return
 
     if merge == rev:
       try:
@@ -659,8 +665,7 @@ class Project(object):
       # and pray that the old upstream also wasn't in the habit
       # of rebasing itself.
       #
-      _info("[%s] Manifest switched from %s to %s",
-            self.name, merge, rev)
+      syncbuf.info(self, "manifest switched %s...%s", merge, rev)
       old_merge = merge
 
     if rev == old_merge:
@@ -671,19 +676,19 @@ class Project(object):
     if not upstream_lost and not upstream_gain:
       # Trivially no changes caused by the upstream.
       #
-      return True
+      return
 
     if self.IsDirty(consider_untracked=False):
-      _warn('[%s] commit (or discard) uncommitted changes'
-            ' before sync', self.name)
-      return False
+      syncbuf.fail(self, _DirtyError())
+      return
 
     if upstream_lost:
       # Upstream rebased.  Not everything in HEAD
       # may have been caused by the user.
       #
-      _info("[%s] Discarding %d commits removed from upstream",
-            self.name, len(upstream_lost))
+      syncbuf.info(self,
+                   "discarding %d commits removed from upstream",
+                   len(upstream_lost))
 
     branch.remote = rem
     branch.merge = self.revision
@@ -691,29 +696,28 @@ class Project(object):
 
     my_changes = self._revlist(not_rev(old_merge), HEAD)
     if my_changes:
-      try:
+      def _dorebase():
         self._Rebase(upstream = old_merge, onto = rev)
-      except GitError:
-        return False
+        self._CopyFiles()
+      syncbuf.later2(self, _dorebase)
     elif upstream_lost:
       try:
         self._ResetHard(rev)
-      except GitError:
-        return False
+        self._CopyFiles()
+      except GitError, e:
+        syncbuf.fail(self, e)
+        return
     else:
-      try:
+      def _doff():
         self._FastForward(rev)
-      except GitError:
-        return False
+        self._CopyFiles()
+      syncbuf.later1(self, _doff)
 
-    self._CopyFiles()
-    return True
-
-  def AddCopyFile(self, src, dest):
+  def AddCopyFile(self, src, dest, absdest):
     # dest should already be an absolute path, but src is project relative
     # make src an absolute path
-    src = os.path.join(self.worktree, src)
-    self.copyfiles.append(_CopyFile(src, dest))
+    abssrc = os.path.join(self.worktree, src)
+    self.copyfiles.append(_CopyFile(src, dest, abssrc, absdest))
 
   def DownloadPatchSet(self, change_id, patch_id):
     """Download a single patch set of a single change to FETCH_HEAD.
@@ -738,16 +742,45 @@ class Project(object):
   def StartBranch(self, name):
     """Create a new branch off the manifest's revision.
     """
-    branch = self.GetBranch(name)
-    branch.remote = self.GetRemote(self.remote.name)
-    branch.merge = self.revision
+    try:
+      self.bare_git.rev_parse(R_HEADS + name)
+      exists = True
+    except GitError:
+      exists = False;
+
+    if exists:
+      if name == self.CurrentBranch:
+        return True
+      else:
+        cmd = ['checkout', name, '--']
+        return GitCommand(self, cmd).Wait() == 0
 
-    rev = branch.LocalMerge
-    cmd = ['checkout', '-b', branch.name, rev]
-    if GitCommand(self, cmd).Wait() == 0:
-      branch.Save()
     else:
-      raise GitError('%s checkout %s ' % (self.name, rev))
+      branch = self.GetBranch(name)
+      branch.remote = self.GetRemote(self.remote.name)
+      branch.merge = self.revision
+
+      rev = branch.LocalMerge
+      cmd = ['checkout', '-b', branch.name, rev]
+      if GitCommand(self, cmd).Wait() == 0:
+        branch.Save()
+        return True
+      else:
+        return False
+
+  def CheckoutBranch(self, name):
+    """Checkout a local topic branch.
+    """
+
+    # Be sure the branch exists
+    try:
+      tip_rev = self.bare_git.rev_parse(R_HEADS + name)
+    except GitError:
+      return False;
+
+    # Do the checkout
+    cmd = ['checkout', name, '--']
+    return GitCommand(self, cmd).Wait() == 0
 
   def AbandonBranch(self, name):
     """Destroy a local topic branch.
@@ -770,7 +803,8 @@ class Project(object):
     """
     cb = self.CurrentBranch
     kill = []
-    for name in self._allrefs.keys():
+    left = self._allrefs
+    for name in left.keys():
       if name.startswith(R_HEADS):
         name = name[len(R_HEADS):]
         if cb is None or name != cb:
@@ -783,14 +817,12 @@ class Project(object):
       self.work_git.DetachHead(HEAD)
       kill.append(cb)
 
-    deleted = set()
     if kill:
       try:
         old = self.bare_git.GetHead()
       except GitError:
         old = 'refs/heads/please_never_use_this_as_a_branch_name'
 
-      rm_re = re.compile(r"^Deleted branch (.*)\.$")
       try:
         self.bare_git.DetachHead(rev)
 
@@ -802,22 +834,20 @@ class Project(object):
         b.Wait()
       finally:
         self.bare_git.SetHead(old)
+        left = self._allrefs
 
-      for line in b.stdout.split("\n"):
-        m = rm_re.match(line)
-        if m:
-          deleted.add(m.group(1))
-
-      if deleted:
-        self.CleanPublishedCache()
+      for branch in kill:
+        if (R_HEADS + branch) not in left:
+          self.CleanPublishedCache()
+          break
 
     if cb and cb not in kill:
       kill.append(cb)
-      kill.sort()
+    kill.sort()
 
     kept = []
     for branch in kill:
-      if branch not in deleted:
+      if (R_HEADS + branch) in left:
         branch = self.GetBranch(branch)
         base = branch.LocalMerge
         if not base:
@@ -856,11 +886,11 @@ class Project(object):
       raise GitError('%s reset --hard %s ' % (self.name, rev))
 
   def _Rebase(self, upstream, onto = None):
-    cmd = ['rebase', '-i']
+    cmd = ['rebase']
     if onto is not None:
       cmd.extend(['--onto', onto])
     cmd.append(upstream)
-    if GitCommand(self, cmd, disable_editor=True).Wait() != 0:
+    if GitCommand(self, cmd).Wait() != 0:
       raise GitError('%s rebase %s ' % (self.name, upstream))
 
   def _FastForward(self, head):
@@ -872,7 +902,11 @@ class Project(object):
     if not os.path.exists(self.gitdir):
       os.makedirs(self.gitdir)
       self.bare_git.init()
-      self.config.SetString('core.bare', None)
+
+      if self.manifest.IsMirror:
+        self.config.SetString('core.bare', 'true')
+      else:
+        self.config.SetString('core.bare', None)
 
       hooks = self._gitdir_path('hooks')
       try:
@@ -1169,6 +1203,113 @@ class Project(object):
       return runner
 
 
+class _PriorSyncFailedError(Exception):
+  def __str__(self):
+    return 'prior sync failed; rebase still in progress'
+
+class _DirtyError(Exception):
+  def __str__(self):
+    return 'contains uncommitted changes'
+
+class _InfoMessage(object):
+  def __init__(self, project, text):
+    self.project = project
+    self.text = text
+
+  def Print(self, syncbuf):
+    syncbuf.out.info('%s/: %s', self.project.relpath, self.text)
+    syncbuf.out.nl()
+
+class _Failure(object):
+  def __init__(self, project, why):
+    self.project = project
+    self.why = why
+
+  def Print(self, syncbuf):
+    syncbuf.out.fail('error: %s/: %s',
+                     self.project.relpath,
+                     str(self.why))
+    syncbuf.out.nl()
+
+class _Later(object):
+  def __init__(self, project, action):
+    self.project = project
+    self.action = action
+
+  def Run(self, syncbuf):
+    out = syncbuf.out
+    out.project('project %s/', self.project.relpath)
+    out.nl()
+    try:
+      self.action()
+      out.nl()
+      return True
+    except GitError, e:
+      out.nl()
+      return False
+
+class _SyncColoring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, 'reposync')
+    self.project   = self.printer('header', attr = 'bold')
+    self.info      = self.printer('info')
+    self.fail      = self.printer('fail', fg='red')
+
+class SyncBuffer(object):
+  def __init__(self, config, detach_head=False):
+    self._messages = []
+    self._failures = []
+    self._later_queue1 = []
+    self._later_queue2 = []
+
+    self.out = _SyncColoring(config)
+    self.out.redirect(sys.stderr)
+
+    self.detach_head = detach_head
+    self.clean = True
+
+  def info(self, project, fmt, *args):
+    self._messages.append(_InfoMessage(project, fmt % args))
+
+  def fail(self, project, err=None):
+    self._failures.append(_Failure(project, err))
+    self.clean = False
+
+  def later1(self, project, what):
+    self._later_queue1.append(_Later(project, what))
+
+  def later2(self, project, what):
+    self._later_queue2.append(_Later(project, what))
+
+  def Finish(self):
+    self._PrintMessages()
+    self._RunLater()
+    self._PrintMessages()
+    return self.clean
+
+  def _RunLater(self):
+    for q in ['_later_queue1', '_later_queue2']:
+      if not self._RunQueue(q):
+        return
+
+  def _RunQueue(self, queue):
+    for m in getattr(self, queue):
+      if not m.Run(self):
+        self.clean = False
+        return False
+    setattr(self, queue, [])
+    return True
+
+  def _PrintMessages(self):
+    for m in self._messages:
+      m.Print(self)
+    for m in self._failures:
+      m.Print(self)
+
+    self._messages = []
+    self._failures = []
+
+
 class MetaProject(Project):
   """A special project housed under .repo.
   """

subcmds/branches.py

@@ -0,0 +1,150 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from color import Coloring
+from command import Command
+
+class BranchColoring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, 'branch')
+    self.current = self.printer('current', fg='green')
+    self.local   = self.printer('local')
+    self.notinproject = self.printer('notinproject', fg='red')
+
+class BranchInfo(object):
+  def __init__(self, name):
+    self.name = name
+    self.current = 0
+    self.published = 0
+    self.published_equal = 0
+    self.projects = []
+
+  def add(self, b):
+    if b.current:
+      self.current += 1
+    if b.published:
+      self.published += 1
+    if b.revision == b.published:
+      self.published_equal += 1
+    self.projects.append(b)
+
+  @property
+  def IsCurrent(self):
+    return self.current > 0
+
+  @property
+  def IsPublished(self):
+    return self.published > 0
+
+  @property
+  def IsPublishedEqual(self):
+    return self.published_equal == len(self.projects)
+
+
+class Branches(Command):
+  common = True
+  helpSummary = "View current topic branches"
+  helpUsage = """
+%prog [<project>...]
+
+Summarizes the currently available topic branches.
+"""
+
+  def _Options(self, p):
+    p.add_option('-a', '--all',
+                 dest='all', action='store_true',
+                 help='show all branches, not just the majority')
+
+  def Execute(self, opt, args):
+    projects = self.GetProjects(args)
+    out = BranchColoring(self.manifest.manifestProject.config)
+    all = {}
+    project_cnt = len(projects)
+
+    for project in projects:
+      for name, b in project.GetBranches().iteritems():
+        b.project = project
+        if name not in all:
+          all[name] = BranchInfo(name)
+        all[name].add(b)
+
+    names = all.keys()
+    names.sort()
+
+    if not opt.all and not args:
+      # No -a and no specific projects listed; try to filter the
+      # results down to only the majority of projects.
+      #
+      n = []
+      for name in names:
+        i = all[name]
+        if i.IsCurrent \
+        or 80 <= (100 * len(i.projects)) / project_cnt:
+          n.append(name)
+      names = n
+
+    width = 25
+    for name in names:
+      if width < len(name):
+        width = len(name)
+
+    for name in names:
+      i = all[name]
+      in_cnt = len(i.projects)
+
+      if i.IsCurrent:
+        current = '*'
+        hdr = out.current
+      else:
+        current = ' '
+        hdr = out.local
+
+      if i.IsPublishedEqual:
+        published = 'P'
+      elif i.IsPublished:
+        published = 'p'
+      else:
+        published = ' '
+
+      hdr('%c%c %-*s' % (current, published, width, name))
+      out.write(' |')
+
+      if in_cnt < project_cnt and (in_cnt == 1 or opt.all):
+        fmt = out.write
+        paths = []
+        if in_cnt < project_cnt - in_cnt: 
+          type = 'in'
+          for b in i.projects:
+            paths.append(b.project.relpath)
+        else:
+          fmt = out.notinproject
+          type = 'not in'
+          have = set()
+          for b in i.projects:
+            have.add(b.project)
+          for p in projects:
+            paths.append(p.relpath)
+
+        s = ' %s %s' % (type, ', '.join(paths))
+        if width + 7 + len(s) < 80:
+          fmt(s)
+        else:
+          out.nl()
+          fmt('    %s:' % type)
+          for p in paths:
+            out.nl()
+            fmt('      %s' % p)
+      out.nl()

subcmds/checkout.py

@@ -0,0 +1,47 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from command import Command
+
+class Checkout(Command):
+  common = True
+  helpSummary = "Checkout a branch for development"
+  helpUsage = """
+%prog <branchname> [<project>...]
+"""
+  helpDescription = """
+The '%prog' command checks out an existing branch that was previously
+created by 'repo start'.
+
+The command is equivalent to:
+
+  repo forall [<project>...] -c git checkout <branchname>
+"""
+
+  def Execute(self, opt, args):
+    if not args:
+      self.Usage()
+
+    retValue = 0;
+
+    branch = args[0]
+    for project in self.GetProjects(args[1:]):
+      if not project.CheckoutBranch(branch):
+        retValue = 1;
+        print >>sys.stderr, "error: checking out branch '%s' in %s failed" % (branch, project.name)
+
+    if (retValue != 0):
+      sys.exit(retValue);

subcmds/forall.py

@@ -17,9 +17,9 @@ import re
 import os
 import sys
 import subprocess
-from command import Command
+from command import Command, MirrorSafeCommand
 
-class Forall(Command):
+class Forall(Command, MirrorSafeCommand):
   common = False
   helpSummary = "Run a shell command in each project"
   helpUsage = """
@@ -30,10 +30,23 @@ Executes the same shell command in each project.
 
 Environment
 -----------
-pwd is the project's working directory.
+
+pwd is the project's working directory.  If the current client is
+a mirror client, then pwd is the Git repository.
 
 REPO_PROJECT is set to the unique name of the project.
 
+REPO_PATH is the path relative the the root of the client.
+
+REPO_REMOTE is the name of the remote system from the manifest.
+
+REPO_LREV is the name of the revision from the manifest, translated
+to a local tracking branch.  If you need to pass the manifest
+revision to a locally executed git command, use REPO_LREV.
+
+REPO_RREV is the name of the revision from the manifest, exactly
+as written in the manifest.
+
 shell positional arguments ($1, $2, .., $#) are set to any arguments
 following <command>.
 
@@ -66,13 +79,31 @@ not redirected.
       cmd.append(cmd[0])
     cmd.extend(opt.command[1:])
 
+    mirror = self.manifest.IsMirror
     rc = 0
     for project in self.GetProjects(args):
       env = dict(os.environ.iteritems())
-      env['REPO_PROJECT'] = project.name
+      def setenv(name, val):
+        if val is None:
+          val = ''
+        env[name] = val
+
+      setenv('REPO_PROJECT', project.name)
+      setenv('REPO_PATH', project.relpath)
+      setenv('REPO_REMOTE', project.remote.name)
+      setenv('REPO_LREV', project\
+        .GetRemote(project.remote.name)\
+        .ToLocal(project.revision))
+      setenv('REPO_RREV', project.revision)
+
+      if mirror:
+        setenv('GIT_DIR', project.gitdir)
+        cwd = project.gitdir
+      else:
+        cwd = project.worktree
 
       p = subprocess.Popen(cmd,
-                           cwd = project.worktree,
+                           cwd = cwd,
                            shell = shell,
                            env = env)
       r = p.wait()

subcmds/grep.py

@@ -0,0 +1,243 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from optparse import SUPPRESS_HELP
+from color import Coloring
+from command import PagedCommand
+from git_command import GitCommand
+
+class GrepColoring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, 'grep')
+    self.project = self.printer('project', attr='bold')
+
+class Grep(PagedCommand):
+  common = True
+  helpSummary = "Print lines matching a pattern"
+  helpUsage = """
+%prog {pattern | -e pattern} [<project>...]
+"""
+  helpDescription = """
+Search for the specified patterns in all project files.
+
+Options
+-------
+
+The following options can appear as often as necessary to express
+the pattern to locate:
+
+ -e PATTERN
+ --and, --or, --not, -(, -)
+
+Further, the -r/--revision option may be specified multiple times
+in order to scan multiple trees.  If the same file matches in more
+than one tree, only the first result is reported, prefixed by the
+revision name it was found under.
+
+Examples
+-------
+
+Look for a line that has '#define' and either 'MAX_PATH or 'PATH_MAX':
+
+  repo grep -e '#define' --and -\( -e MAX_PATH -e PATH_MAX \)
+
+Look for a line that has 'NODE' or 'Unexpected' in files that
+contain a line that matches both expressions:
+
+  repo grep --all-match -e NODE -e Unexpected
+
+"""
+
+  def _Options(self, p):
+    def carry(option,
+              opt_str,
+              value,
+              parser):
+      pt = getattr(parser.values, 'cmd_argv', None)
+      if pt is None:
+        pt = []
+        setattr(parser.values, 'cmd_argv', pt)
+
+      if opt_str == '-(':
+        pt.append('(')
+      elif opt_str == '-)':
+        pt.append(')')
+      else:
+        pt.append(opt_str)
+
+      if value is not None:
+        pt.append(value)
+
+    g = p.add_option_group('Sources')
+    g.add_option('--cached',
+                 action='callback', callback=carry,
+                 help='Search the index, instead of the work tree')
+    g.add_option('-r','--revision',
+                 dest='revision', action='append', metavar='TREEish',
+                 help='Search TREEish, instead of the work tree')
+
+    g = p.add_option_group('Pattern')
+    g.add_option('-e',
+                 action='callback', callback=carry,
+                 metavar='PATTERN', type='str',
+                 help='Pattern to search for')
+    g.add_option('-i', '--ignore-case',
+                 action='callback', callback=carry,
+                 help='Ignore case differences')
+    g.add_option('-a','--text',
+                 action='callback', callback=carry,
+                 help="Process binary files as if they were text")
+    g.add_option('-I',
+                 action='callback', callback=carry,
+                 help="Don't match the pattern in binary files")
+    g.add_option('-w', '--word-regexp',
+                 action='callback', callback=carry,
+                 help='Match the pattern only at word boundaries')
+    g.add_option('-v', '--invert-match',
+                 action='callback', callback=carry,
+                 help='Select non-matching lines')
+    g.add_option('-G', '--basic-regexp',
+                 action='callback', callback=carry,
+                 help='Use POSIX basic regexp for patterns (default)')
+    g.add_option('-E', '--extended-regexp',
+                 action='callback', callback=carry,
+                 help='Use POSIX extended regexp for patterns')
+    g.add_option('-F', '--fixed-strings',
+                 action='callback', callback=carry,
+                 help='Use fixed strings (not regexp) for pattern')
+
+    g = p.add_option_group('Pattern Grouping')
+    g.add_option('--all-match',
+                 action='callback', callback=carry,
+                 help='Limit match to lines that have all patterns')
+    g.add_option('--and', '--or', '--not',
+                 action='callback', callback=carry,
+                 help='Boolean operators to combine patterns')
+    g.add_option('-(','-)',
+                 action='callback', callback=carry,
+                 help='Boolean operator grouping')
+
+    g = p.add_option_group('Output')
+    g.add_option('-n',
+                 action='callback', callback=carry,
+                 help='Prefix the line number to matching lines')
+    g.add_option('-C',
+                 action='callback', callback=carry,
+                 metavar='CONTEXT', type='str',
+                 help='Show CONTEXT lines around match')
+    g.add_option('-B',
+                 action='callback', callback=carry,
+                 metavar='CONTEXT', type='str',
+                 help='Show CONTEXT lines before match')
+    g.add_option('-A',
+                 action='callback', callback=carry,
+                 metavar='CONTEXT', type='str',
+                 help='Show CONTEXT lines after match')
+    g.add_option('-l','--name-only','--files-with-matches',
+                 action='callback', callback=carry,
+                 help='Show only file names containing matching lines')
+    g.add_option('-L','--files-without-match',
+                 action='callback', callback=carry,
+                 help='Show only file names not containing matching lines')
+
+
+  def Execute(self, opt, args):
+    out = GrepColoring(self.manifest.manifestProject.config)
+
+    cmd_argv = ['grep']
+    if out.is_on:
+      cmd_argv.append('--color')
+    cmd_argv.extend(getattr(opt,'cmd_argv',[]))
+
+    if '-e' not in cmd_argv:
+      if not args:
+        self.Usage()
+      cmd_argv.append('-e')
+      cmd_argv.append(args[0])
+      args = args[1:]
+
+    projects = self.GetProjects(args)
+
+    full_name = False
+    if len(projects) > 1:
+      cmd_argv.append('--full-name')
+      full_name = True
+
+    have_rev = False
+    if opt.revision:
+      if '--cached' in cmd_argv:
+        print >>sys.stderr,\
+          'fatal: cannot combine --cached and --revision'
+        sys.exit(1)
+      have_rev = True
+      cmd_argv.extend(opt.revision)
+    cmd_argv.append('--')
+
+    bad_rev = False
+    have_match = False
+
+    for project in projects:
+      p = GitCommand(project,
+                     cmd_argv,
+                     bare = False,
+                     capture_stdout = True,
+                     capture_stderr = True)
+      if p.Wait() != 0:
+        # no results
+        #
+        if p.stderr:
+          if have_rev and 'fatal: ambiguous argument' in p.stderr:
+            bad_rev = True
+          else:
+            out.project('--- project %s ---' % project.relpath)
+            out.nl()
+            out.write(p.stderr)
+            out.nl()
+        continue
+      have_match = True
+
+      # We cut the last element, to avoid a blank line.
+      #
+      r = p.stdout.split('\n')
+      r = r[0:-1]
+
+      if have_rev and full_name:
+        for line in r:
+          rev, line = line.split(':', 1)
+          out.write(rev)
+          out.write(':')
+          out.project(project.relpath)
+          out.write('/')
+          out.write(line)
+          out.nl()
+      elif full_name:
+        for line in r:
+          out.project(project.relpath)
+          out.write('/')
+          out.write(line)
+          out.nl()
+      else:
+        for line in r:
+          print line
+
+    if have_match:
+      sys.exit(0)
+    elif have_rev and bad_rev:
+      for r in opt.revision:
+        print >>sys.stderr, "error: can't search revision %s" % r
+      sys.exit(1)
+    else:
+      sys.exit(1)

subcmds/help.py

@@ -13,13 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import re
 import sys
 from formatter import AbstractFormatter, DumbWriter
 
 from color import Coloring
-from command import PagedCommand
+from command import PagedCommand, MirrorSafeCommand
 
-class Help(PagedCommand):
+class Help(PagedCommand, MirrorSafeCommand):
   common = False
   helpSummary = "Display detailed help on a command"
   helpUsage = """
@@ -77,6 +78,7 @@ The most commonly used repo commands are:
       print fmt % (name, summary)
     print """
 See 'repo help <command>' for more information on a specific command.
+See 'repo help --all' for a complete list of recognized commands.
 """
 
   def _PrintCommandHelp(self, cmd):
@@ -105,14 +107,24 @@ See 'repo help <command>' for more information on a specific command.
         body = body.strip()
         body = body.replace('%prog', me)
 
+        asciidoc_hdr = re.compile(r'^\n?([^\n]{1,})\n(={2,}|-{2,})$')
         for para in body.split("\n\n"):
           if para.startswith(' '):
             self.write('%s', para)
             self.nl()
             self.nl()
-          else:
-            self.wrap.add_flowing_data(para)
-            self.wrap.end_paragraph(1)
+            continue
+
+          m = asciidoc_hdr.match(para)
+          if m:
+            self.heading('%s', m.group(1))
+            self.nl()
+            self.heading('%s', ''.ljust(len(m.group(1)),'-'))
+            self.nl()
+            continue
+
+          self.wrap.add_flowing_data(para)
+          self.wrap.end_paragraph(1)
         self.wrap.end_paragraph(0)
 
     out = _Out(self.manifest.globalConfig)

subcmds/init.py

@@ -17,12 +17,13 @@ import os
 import sys
 
 from color import Coloring
-from command import InteractiveCommand
+from command import InteractiveCommand, MirrorSafeCommand
 from error import ManifestParseError
 from remote import Remote
+from project import SyncBuffer
 from git_command import git, MIN_GIT_VERSION
 
-class Init(InteractiveCommand):
+class Init(InteractiveCommand, MirrorSafeCommand):
   common = True
   helpSummary = "Initialize repo in the current directory"
   helpUsage = """
@@ -89,8 +90,9 @@ default.xml will be used.
 
   def _SyncManifest(self, opt):
     m = self.manifest.manifestProject
+    is_new = not m.Exists
 
-    if not m.Exists:
+    if is_new:
       if not opt.manifest_url:
         print >>sys.stderr, 'fatal: manifest url (-u) is required.'
         sys.exit(1)
@@ -117,11 +119,25 @@ default.xml will be used.
       r.Save()
 
     if opt.mirror:
-      m.config.SetString('repo.mirror', 'true')
+      if is_new:
+        m.config.SetString('repo.mirror', 'true')
+      else:
+        print >>sys.stderr, 'fatal: --mirror not supported on existing client'
+        sys.exit(1)
 
-    m.Sync_NetworkHalf()
-    m.Sync_LocalHalf()
-    m.StartBranch('default')
+    if not m.Sync_NetworkHalf():
+      r = m.GetRemote(m.remote.name)
+      print >>sys.stderr, 'fatal: cannot obtain manifest %s' % r.url
+      sys.exit(1)
+
+    syncbuf = SyncBuffer(m.config)
+    m.Sync_LocalHalf(syncbuf)
+    syncbuf.Finish()
+
+    if is_new or m.CurrentBranch is None:
+      if not m.StartBranch('default'):
+        print >>sys.stderr, 'fatal: cannot create default in manifest'
+        sys.exit(1)
 
   def _LinkManifest(self, name):
     if not name:
@@ -192,11 +208,11 @@ default.xml will be used.
     self._SyncManifest(opt)
     self._LinkManifest(opt.manifest_name)
 
-    if os.isatty(0) and os.isatty(1) and not opt.mirror:
+    if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
       self._ConfigureUser()
       self._ConfigureColor()
 
-    if opt.mirror:
+    if self.manifest.IsMirror:
       type = 'mirror '
     else:
       type = ''

subcmds/manifest.py

@@ -0,0 +1,77 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+from command import PagedCommand
+
+class Manifest(PagedCommand):
+  common = False
+  helpSummary = "Manifest inspection utility"
+  helpUsage = """
+%prog [-o {-|NAME.xml} [-r]]
+"""
+  _helpDescription = """
+
+With the -o option, exports the current manifest for inspection.
+The manifest and (if present) local_manifest.xml are combined
+together to produce a single manifest file.  This file can be stored
+in a Git repository for use during future 'repo init' invocations.
+
+"""
+
+  @property
+  def helpDescription(self):
+    help = self._helpDescription + '\n'
+    r = os.path.dirname(__file__)
+    r = os.path.dirname(r)
+    fd = open(os.path.join(r, 'docs', 'manifest-format.txt'))
+    for line in fd:
+      help += line
+    fd.close()
+    return help
+
+  def _Options(self, p):
+    p.add_option('-r', '--revision-as-HEAD',
+                 dest='peg_rev', action='store_true',
+                 help='Save revisions as current HEAD')
+    p.add_option('-o', '--output-file',
+                 dest='output_file',
+                 help='File to save the manifest to',
+                 metavar='-|NAME.xml')
+
+  def _Output(self, opt):
+    if opt.output_file == '-':
+      fd = sys.stdout
+    else:
+      fd = open(opt.output_file, 'w')
+    self.manifest.Save(fd,
+                       peg_rev = opt.peg_rev)
+    fd.close()
+    if opt.output_file != '-':
+      print >>sys.stderr, 'Saved manifest to %s' % opt.output_file
+
+  def Execute(self, opt, args):
+    if args:
+      self.Usage()
+
+    if opt.output_file is not None:
+      self._Output(opt)
+      return
+
+    print >>sys.stderr, 'error: no operation to perform'
+    print >>sys.stderr, 'error: see repo help manifest'
+    sys.exit(1)

subcmds/selfupdate.py

@@ -0,0 +1,59 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from optparse import SUPPRESS_HELP
+import sys
+
+from command import Command, MirrorSafeCommand
+from subcmds.sync import _PostRepoUpgrade
+from subcmds.sync import _PostRepoFetch
+
+class Selfupdate(Command, MirrorSafeCommand):
+  common = False
+  helpSummary = "Update repo to the latest version"
+  helpUsage = """
+%prog
+"""
+  helpDescription = """
+The '%prog' command upgrades repo to the latest version, if a
+newer version is available.
+
+Normally this is done automatically by 'repo sync' and does not
+need to be performed by an end-user.
+"""
+
+  def _Options(self, p):
+    p.add_option('--no-repo-verify',
+                 dest='no_repo_verify', action='store_true',
+                 help='do not verify repo source code')
+    p.add_option('--repo-upgraded',
+                 dest='repo_upgraded', action='store_true',
+                 help=SUPPRESS_HELP)
+
+  def Execute(self, opt, args):
+    rp = self.manifest.repoProject
+    rp.PreSync()
+
+    if opt.repo_upgraded:
+      _PostRepoUpgrade(self.manifest)
+
+    else:
+      if not rp.Sync_NetworkHalf():
+        print >>sys.stderr, "error: can't update repo"
+        sys.exit(1)
+
+      _PostRepoFetch(rp,
+                     no_repo_verify = opt.no_repo_verify,
+                     verbose = True)

subcmds/start.py

@@ -47,5 +47,13 @@ the configuration data is set up properly.
       print >>sys.stderr, "error: '%s' is not a valid name" % nb
       sys.exit(1)
 
+    err = []
     for project in self.GetProjects(args[1:]):
-      project.StartBranch(nb)
+      if not project.StartBranch(nb):
+        err.append(project)
+
+    if err:
+      err.sort()
+      for p in err:
+        print >>sys.stderr, "error: cannot start in %s" % p.relpath
+      sys.exit(1)

subcmds/status.py

@@ -20,8 +20,53 @@ class Status(PagedCommand):
   helpSummary = "Show the working tree status"
   helpUsage = """
 %prog [<project>...]
+"""
+  helpDescription = """
+'%prog' compares the working tree to the staging area (aka index),
+and the most recent commit on this branch (HEAD), in each project
+specified.  A summary is displayed, one line per file where there
+is a difference between these three states.
+
+Status Display
+--------------
+
+The status display is organized into three columns of information,
+for example if the file 'subcmds/status.py' is modified in the
+project 'repo' on branch 'devwork':
+
+  project repo/                                   branch devwork
+   -m     subcmds/status.py
+
+The first column explains how the staging area (index) differs from
+the last commit (HEAD).  Its values are always displayed in upper
+case and have the following meanings:
+
+ -:  no difference
+ A:  added         (not in HEAD,     in index                     )
+ M:  modified      (    in HEAD,     in index, different content  )
+ D:  deleted       (    in HEAD, not in index                     )
+ R:  renamed       (not in HEAD,     in index, path changed       )
+ C:  copied        (not in HEAD,     in index, copied from another)
+ T:  mode changed  (    in HEAD,     in index, same content       )
+ U:  unmerged; conflict resolution required
+
+The second column explains how the working directory differs from
+the index.  Its values are always displayed in lower case and have
+the following meanings:
+
+ -:  new / unknown (not in index,     in work tree                )
+ m:  modified      (    in index,     in work tree, modified      )
+ d:  deleted       (    in index, not in work tree                )
+
 """
 
   def Execute(self, opt, args):
-    for project in self.GetProjects(args):
-      project.PrintWorkTreeStatus()
+    all = self.GetProjects(args)
+    clean = 0
+
+    for project in all:
+      state = project.PrintWorkTreeStatus()
+      if state == 'CLEAN':
+        clean += 1
+    if len(all) == clean:
+      print 'nothing to commit (working directory clean)'

subcmds/sync.py

@@ -13,17 +13,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from optparse import SUPPRESS_HELP
 import os
 import re
 import subprocess
 import sys
 
 from git_command import GIT
-from command import Command
+from project import HEAD
+from command import Command, MirrorSafeCommand
 from error import RepoChangedException, GitError
 from project import R_HEADS
+from project import SyncBuffer
+from progress import Progress
 
-class Sync(Command):
+class Sync(Command, MirrorSafeCommand):
   common = True
   helpSummary = "Update working tree to the latest revision"
   helpUsage = """
@@ -43,27 +47,53 @@ line.  Projects can be specified either by name, or by a relative
 or absolute path to the project's local directory. If no projects
 are specified, '%prog' will synchronize all projects listed in
 the manifest.
+
+The -d/--detach option can be used to switch specified projects
+back to the manifest revision.  This option is especially helpful
+if the project is currently on a topic branch, but the manifest
+revision is temporarily needed.
 """
 
   def _Options(self, p):
+    p.add_option('-l','--local-only',
+                 dest='local_only', action='store_true',
+                 help="only update working tree, don't fetch")
+    p.add_option('-n','--network-only',
+                 dest='network_only', action='store_true',
+                 help="fetch only, don't update working tree")
+    p.add_option('-d','--detach',
+                 dest='detach_head', action='store_true',
+                 help='detach projects back to manifest revision')
+
     p.add_option('--no-repo-verify',
                  dest='no_repo_verify', action='store_true',
                  help='do not verify repo source code')
     p.add_option('--repo-upgraded',
                  dest='repo_upgraded', action='store_true',
-                 help='perform additional actions after a repo upgrade')
+                 help=SUPPRESS_HELP)
 
   def _Fetch(self, *projects):
     fetched = set()
+    pm = Progress('Fetching projects', len(projects))
     for project in projects:
+      pm.update()
+
       if project.Sync_NetworkHalf():
         fetched.add(project.gitdir)
       else:
         print >>sys.stderr, 'error: Cannot fetch %s' % project.name
         sys.exit(1)
+    pm.end()
     return fetched
 
   def Execute(self, opt, args):
+    if opt.network_only and opt.detach_head:
+      print >>sys.stderr, 'error: cannot combine -n and -d'
+      sys.exit(1)
+    if opt.network_only and opt.local_only:
+      print >>sys.stderr, 'error: cannot combine -n and -l'
+      sys.exit(1)
+
     rp = self.manifest.repoProject
     rp.PreSync()
 
@@ -71,42 +101,66 @@ the manifest.
     mp.PreSync()
 
     if opt.repo_upgraded:
-      for project in self.manifest.projects.values():
-        if project.Exists:
-          project.PostRepoUpgrade()
+      _PostRepoUpgrade(self.manifest)
 
     all = self.GetProjects(args, missing_ok=True)
-    fetched = self._Fetch(rp, mp, *all)
 
-    if rp.HasChanges:
-      print >>sys.stderr, 'info: A new version of repo is available'
-      print >>sys.stderr, ''
-      if opt.no_repo_verify or _VerifyTag(rp):
-        if not rp.Sync_LocalHalf():
+    if not opt.local_only:
+      fetched = self._Fetch(rp, mp, *all)
+      _PostRepoFetch(rp, opt.no_repo_verify)
+      if opt.network_only:
+        # bail out now; the rest touches the working tree
+        return
+
+      if mp.HasChanges:
+        syncbuf = SyncBuffer(mp.config)
+        mp.Sync_LocalHalf(syncbuf)
+        if not syncbuf.Finish():
           sys.exit(1)
-        print >>sys.stderr, 'info: Restarting repo with latest version'
-        raise RepoChangedException(['--repo-upgraded'])
-      else:
-        print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
 
-    if mp.HasChanges:
-      if not mp.Sync_LocalHalf():
-        sys.exit(1)
-
-      self.manifest._Unload()
-      all = self.GetProjects(args, missing_ok=True)
-      missing = []
-      for project in all:
-        if project.gitdir not in fetched:
-          missing.append(project)
-      self._Fetch(*missing)
+        self.manifest._Unload()
+        all = self.GetProjects(args, missing_ok=True)
+        missing = []
+        for project in all:
+          if project.gitdir not in fetched:
+            missing.append(project)
+        self._Fetch(*missing)
 
+    syncbuf = SyncBuffer(mp.config,
+                         detach_head = opt.detach_head)
+    pm = Progress('Syncing work tree', len(all))
     for project in all:
+      pm.update()
       if project.worktree:
-        if not project.Sync_LocalHalf():
-          sys.exit(1)
+        project.Sync_LocalHalf(syncbuf)
+    pm.end()
+    print >>sys.stderr
+    if not syncbuf.Finish():
+      sys.exit(1)
 
 
+def _PostRepoUpgrade(manifest):
+  for project in manifest.projects.values():
+    if project.Exists:
+      project.PostRepoUpgrade()
+
+def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
+  if rp.HasChanges:
+    print >>sys.stderr, 'info: A new version of repo is available'
+    print >>sys.stderr, ''
+    if no_repo_verify or _VerifyTag(rp):
+      syncbuf = SyncBuffer(rp.config)
+      rp.Sync_LocalHalf(syncbuf)
+      if not syncbuf.Finish():
+        sys.exit(1)
+      print >>sys.stderr, 'info: Restarting repo with latest version'
+      raise RepoChangedException(['--repo-upgraded'])
+    else:
+      print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
+  else:
+    if verbose:
+      print >>sys.stderr, 'repo version %s is current' % rp.work_git.describe(HEAD)
+
 def _VerifyTag(project):
   gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
   if not os.path.exists(gpg_dir):

subcmds/upload.py

@@ -234,9 +234,6 @@ files and description associated with the change in Gerrit.
           print >>sys.stderr, '[OK    ] %-15s %s' % (
                  branch.project.relpath + '/',
                  branch.name)
-          print >>sys.stderr, '%s' % branch.tip_url
-          print >>sys.stderr, '(as %s)' % branch.owner_email
-          print >>sys.stderr, ''
 
     if have_errors:
       sys.exit(1)

subcmds/version.py

@@ -0,0 +1,35 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from command import Command, MirrorSafeCommand
+from git_command import git
+from project import HEAD
+
+class Version(Command, MirrorSafeCommand):
+  common = False
+  helpSummary = "Display the version of repo"
+  helpUsage = """
+%prog
+"""
+
+  def Execute(self, opt, args):
+    rp = self.manifest.repoProject
+    rem = rp.GetRemote(rp.remote.name)
+
+    print 'repo version %s' % rp.work_git.describe(HEAD)
+    print '       (from %s)' % rem.url
+    print git.version().strip()
+    print 'Python %s' % sys.version