Change repo sync to be more friendly when updating the tree

We now try to sync all projects that can be done safely first, before
we start rebasing user commits over the upstream.  This has the nice
effect of making the local tree as close to the upstream as possible
before the user has to start resolving merge conflicts, as that extra
information in other projects may aid in the conflict resolution.

Informational output is buffered and delayed until calculation for
all projects has been done, so that the user gets one concise list
of notice messages, rather than it interrupting the progress meter.

Fast-forward output is now prefixed with the project header, so the
user can see which project that update is taking place in, and make
some relation of the diffstat back to the project name.

Rebase output is now prefixed with the project header, so that if
the rebase fails, the user can see which project we were operating
on and can try to address the failure themselves.

Since rebase sits on a detached HEAD, we now look for an in-progress
rebase during sync, so we can alert the user that the given project
is in a state we cannot handle.

Signed-off-by: Shawn O. Pearce <sop@google.com>

color.py

@@ -100,6 +100,9 @@ class Coloring(object):
     else:
       self._on = False
 
+  def redirect(self, out):
+    self._out = out
+
   @property
   def is_on(self):
     return self._on

progress.py

@@ -16,7 +16,7 @@
 import sys
 
 class Progress(object):
-  def __init__(self, title, total):
+  def __init__(self, title, total=0):
     self._title = title
     self._total = total
     self._done = 0
@@ -24,22 +24,35 @@ class Progress(object):
 
   def update(self, inc=1):
     self._done += inc
-    p = (100 * self._done) / self._total
 
-    if self._lastp != p:
+    if self._total <= 0:
-      self._lastp = p
+      sys.stderr.write('\r%s: %d, ' % (
-      sys.stderr.write('\r%s: %3d%% (%d/%d)  ' % (
+        self._title,
+        self._done))
+      sys.stderr.flush()
+    else:
+      p = (100 * self._done) / self._total
+
+      if self._lastp != p:
+        self._lastp = p
+        sys.stderr.write('\r%s: %3d%% (%d/%d)  ' % (
+          self._title,
+          p,
+          self._done,
+          self._total))
+        sys.stderr.flush()
+
+  def end(self):
+    if self._total <= 0:
+      sys.stderr.write('\r%s: %d, done.  \n' % (
+        self._title,
+        self._done))
+      sys.stderr.flush()
+    else:
+      p = (100 * self._done) / self._total
+      sys.stderr.write('\r%s: %3d%% (%d/%d), done.  \n' % (
         self._title,
         p,
         self._done,
         self._total))
       sys.stderr.flush()
-
-  def end(self):
-    p = (100 * self._done) / self._total
-    sys.stderr.write('\r%s: %3d%% (%d/%d), done.  \n' % (
-      self._title,
-      p,
-      self._done,
-      self._total))
-    sys.stderr.flush()

project.py

@@ -34,13 +34,9 @@ R_TAGS  = 'refs/tags/'
 R_PUB   = 'refs/published/'
 R_M     = 'refs/remotes/m/'
 
-def _warn(fmt, *args):
+def _error(fmt, *args):
   msg = fmt % args
-  print >>sys.stderr, 'warn: %s' % msg
+  print >>sys.stderr, 'error: %s' % msg
-
-def _info(fmt, *args):
-  msg = fmt % args
-  print >>sys.stderr, 'info: %s' % msg
 
 def not_rev(r):
   return '^' + r
@@ -199,9 +195,7 @@ class _CopyFile:
         mode = mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
         os.chmod(dest, mode)
       except IOError:
-           print >>sys.stderr, \
+        _error('Cannot copy file %s to %s', src, dest)
-              'error: Cannot copy file %s to %s' \
-              % (src, dest)
 
 
 class Project(object):
@@ -574,13 +568,9 @@ class Project(object):
     for file in self.copyfiles:
       file._Copy()
 
-  def Sync_LocalHalf(self, detach_head=False):
+  def Sync_LocalHalf(self, syncbuf):
     """Perform only the local IO portion of the sync process.
        Network access is not required.
-
-       Return:
-         True:  the sync was successful
-         False: the sync requires user input
     """
     self._InitWorkTree()
     self.CleanPublishedCache()
@@ -595,19 +585,25 @@ class Project(object):
 
     branch = self.CurrentBranch
 
-    if branch is None or detach_head:
+    if branch is None or syncbuf.detach_head:
       # Currently on a detached HEAD.  The user is assumed to
       # not have any local modifications worth worrying about.
       #
+      if os.path.exists(os.path.join(self.worktree, '.dotest')) \
+      or os.path.exists(os.path.join(self.worktree, '.git', 'rebase-apply')):
+        syncbuf.fail(self, _PriorSyncFailedError())
+        return
+
       lost = self._revlist(not_rev(rev), HEAD)
       if lost:
-        _info("[%s] Discarding %d commits", self.name, len(lost))
+        syncbuf.info(self, "discarding %d commits", len(lost))
       try:
         self._Checkout(rev, quiet=True)
-      except GitError:
+      except GitError, e:
-        return False
+        syncbuf.fail(self, e)
+        return
       self._CopyFiles()
-      return True
+      return
 
     branch = self.GetBranch(branch)
     merge = branch.LocalMerge
@@ -616,16 +612,16 @@ class Project(object):
       # The current branch has no tracking configuration.
       # Jump off it to a deatched HEAD.
       #
-      _info("[%s] Leaving %s"
+      syncbuf.info(self,
-            " (does not track any upstream)",
+                   "leaving %s; does not track upstream",
-            self.name,
+                   branch.name)
-            branch.name)
       try:
         self._Checkout(rev, quiet=True)
-      except GitError:
+      except GitError, e:
-        return False
+        syncbuf.fail(self, e)
+        return
       self._CopyFiles()
-      return True
+      return
 
     upstream_gain = self._revlist(not_rev(HEAD), rev)
     pub = self.WasPublished(branch.name)
@@ -637,25 +633,24 @@ class Project(object):
           # commits are not yet merged upstream.  We do not want
           # to rewrite the published commits so we punt.
           #
-          _info("[%s] Branch %s is published,"
+          syncbuf.info(self,
-                " but is now %d commits behind.",
+                       "branch %s is published but is now %d commits behind",
-                self.name, branch.name, len(upstream_gain))
+                       branch.name,
-          _info("[%s] Consider merging or rebasing the"
+                       len(upstream_gain))
-                " unpublished commits.", self.name)
+          syncbuf.info(self, "consider merging or rebasing the unpublished commits")
-        return True
+        return
       elif upstream_gain:
         # We can fast-forward safely.
         #
-        try:
+        def _doff():
           self._FastForward(rev)
-        except GitError:
+          self._CopyFiles()
-          return False
+        syncbuf.later1(self, _doff)
-        self._CopyFiles()
+        return
-        return True
       else:
         # Trivially no changes in the upstream.
         #
-        return True
+        return
 
     if merge == rev:
       try:
@@ -670,8 +665,7 @@ class Project(object):
       # and pray that the old upstream also wasn't in the habit
       # of rebasing itself.
       #
-      _info("[%s] Manifest switched from %s to %s",
+      syncbuf.info(self, "manifest switched %s...%s", merge, rev)
-            self.name, merge, rev)
       old_merge = merge
 
     if rev == old_merge:
@@ -682,19 +676,19 @@ class Project(object):
     if not upstream_lost and not upstream_gain:
       # Trivially no changes caused by the upstream.
       #
-      return True
+      return
 
     if self.IsDirty(consider_untracked=False):
-      _warn('[%s] commit (or discard) uncommitted changes'
+      syncbuf.fail(self, _DirtyError())
-            ' before sync', self.name)
+      return
-      return False
 
     if upstream_lost:
       # Upstream rebased.  Not everything in HEAD
       # may have been caused by the user.
       #
-      _info("[%s] Discarding %d commits removed from upstream",
+      syncbuf.info(self,
-            self.name, len(upstream_lost))
+                   "discarding %d commits removed from upstream",
+                   len(upstream_lost))
 
     branch.remote = rem
     branch.merge = self.revision
@@ -702,23 +696,22 @@ class Project(object):
 
     my_changes = self._revlist(not_rev(old_merge), HEAD)
     if my_changes:
-      try:
+      def _dorebase():
         self._Rebase(upstream = old_merge, onto = rev)
-      except GitError:
+        self._CopyFiles()
-        return False
+      syncbuf.later2(self, _dorebase)
     elif upstream_lost:
       try:
         self._ResetHard(rev)
-      except GitError:
+        self._CopyFiles()
-        return False
+      except GitError, e:
+        syncbuf.fail(self, e)
+        return
     else:
-      try:
+      def _doff():
         self._FastForward(rev)
-      except GitError:
+        self._CopyFiles()
-        return False
+      syncbuf.later1(self, _doff)
-
-    self._CopyFiles()
-    return True
 
   def AddCopyFile(self, src, dest, absdest):
     # dest should already be an absolute path, but src is project relative
@@ -893,11 +886,11 @@ class Project(object):
       raise GitError('%s reset --hard %s ' % (self.name, rev))
 
   def _Rebase(self, upstream, onto = None):
-    cmd = ['rebase', '-i']
+    cmd = ['rebase']
     if onto is not None:
       cmd.extend(['--onto', onto])
     cmd.append(upstream)
-    if GitCommand(self, cmd, disable_editor=True).Wait() != 0:
+    if GitCommand(self, cmd).Wait() != 0:
       raise GitError('%s rebase %s ' % (self.name, upstream))
 
   def _FastForward(self, head):
@@ -1210,6 +1203,113 @@ class Project(object):
       return runner
 
 
+class _PriorSyncFailedError(Exception):
+  def __str__(self):
+    return 'prior sync failed; rebase still in progress'
+
+class _DirtyError(Exception):
+  def __str__(self):
+    return 'contains uncommitted changes'
+
+class _InfoMessage(object):
+  def __init__(self, project, text):
+    self.project = project
+    self.text = text
+
+  def Print(self, syncbuf):
+    syncbuf.out.info('%s/: %s', self.project.relpath, self.text)
+    syncbuf.out.nl()
+
+class _Failure(object):
+  def __init__(self, project, why):
+    self.project = project
+    self.why = why
+
+  def Print(self, syncbuf):
+    syncbuf.out.fail('error: %s/: %s',
+                     self.project.relpath,
+                     str(self.why))
+    syncbuf.out.nl()
+
+class _Later(object):
+  def __init__(self, project, action):
+    self.project = project
+    self.action = action
+
+  def Run(self, syncbuf):
+    out = syncbuf.out
+    out.project('project %s/', self.project.relpath)
+    out.nl()
+    try:
+      self.action()
+      out.nl()
+      return True
+    except GitError, e:
+      out.nl()
+      return False
+
+class _SyncColoring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, 'reposync')
+    self.project   = self.printer('header', attr = 'bold')
+    self.info      = self.printer('info')
+    self.fail      = self.printer('fail', fg='red')
+
+class SyncBuffer(object):
+  def __init__(self, config, detach_head=False):
+    self._messages = []
+    self._failures = []
+    self._later_queue1 = []
+    self._later_queue2 = []
+
+    self.out = _SyncColoring(config)
+    self.out.redirect(sys.stderr)
+
+    self.detach_head = detach_head
+    self.clean = True
+
+  def info(self, project, fmt, *args):
+    self._messages.append(_InfoMessage(project, fmt % args))
+
+  def fail(self, project, err=None):
+    self._failures.append(_Failure(project, err))
+    self.clean = False
+
+  def later1(self, project, what):
+    self._later_queue1.append(_Later(project, what))
+
+  def later2(self, project, what):
+    self._later_queue2.append(_Later(project, what))
+
+  def Finish(self):
+    self._PrintMessages()
+    self._RunLater()
+    self._PrintMessages()
+    return self.clean
+
+  def _RunLater(self):
+    for q in ['_later_queue1', '_later_queue2']:
+      if not self._RunQueue(q):
+        return
+
+  def _RunQueue(self, queue):
+    for m in getattr(self, queue):
+      if not m.Run(self):
+        self.clean = False
+        return False
+    setattr(self, queue, [])
+    return True
+
+  def _PrintMessages(self):
+    for m in self._messages:
+      m.Print(self)
+    for m in self._failures:
+      m.Print(self)
+
+    self._messages = []
+    self._failures = []
+
+
 class MetaProject(Project):
   """A special project housed under .repo.
   """

subcmds/checkout.py

@@ -21,12 +21,14 @@ class Checkout(Command):
   helpSummary = "Checkout a branch for development"
   helpUsage = """
 %prog <branchname> [<project>...]
+"""
+  helpDescription = """
+The '%prog' command checks out an existing branch that was previously
+created by 'repo start'.
 
-This subcommand checks out an existing branch and
+The command is equivalent to:
-is equivalent to the following git command run on
-every project or the list of specified projects:
 
-"git checkout <branchname>"
+  repo forall [<project>...] -c git checkout <branchname>
 """
 
   def Execute(self, opt, args):

subcmds/forall.py

@@ -30,6 +30,7 @@ Executes the same shell command in each project.
 
 Environment
 -----------
+
 pwd is the project's working directory.  If the current client is
 a mirror client, then pwd is the Git repository.
 

subcmds/init.py

@@ -20,6 +20,7 @@ from color import Coloring
 from command import InteractiveCommand, MirrorSafeCommand
 from error import ManifestParseError
 from remote import Remote
+from project import SyncBuffer
 from git_command import git, MIN_GIT_VERSION
 
 class Init(InteractiveCommand, MirrorSafeCommand):
@@ -129,7 +130,10 @@ default.xml will be used.
       print >>sys.stderr, 'fatal: cannot obtain manifest %s' % r.url
       sys.exit(1)
 
-    m.Sync_LocalHalf()
+    syncbuf = SyncBuffer(m.config)
+    m.Sync_LocalHalf(syncbuf)
+    syncbuf.Finish()
+
     if is_new or m.CurrentBranch is None:
       if not m.StartBranch('default'):
         print >>sys.stderr, 'fatal: cannot create default in manifest'

subcmds/selfupdate.py

@@ -0,0 +1,59 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from optparse import SUPPRESS_HELP
+import sys
+
+from command import Command, MirrorSafeCommand
+from subcmds.sync import _PostRepoUpgrade
+from subcmds.sync import _PostRepoFetch
+
+class Selfupdate(Command, MirrorSafeCommand):
+  common = False
+  helpSummary = "Update repo to the latest version"
+  helpUsage = """
+%prog
+"""
+  helpDescription = """
+The '%prog' command upgrades repo to the latest version, if a
+newer version is available.
+
+Normally this is done automatically by 'repo sync' and does not
+need to be performed by an end-user.
+"""
+
+  def _Options(self, p):
+    p.add_option('--no-repo-verify',
+                 dest='no_repo_verify', action='store_true',
+                 help='do not verify repo source code')
+    p.add_option('--repo-upgraded',
+                 dest='repo_upgraded', action='store_true',
+                 help=SUPPRESS_HELP)
+
+  def Execute(self, opt, args):
+    rp = self.manifest.repoProject
+    rp.PreSync()
+
+    if opt.repo_upgraded:
+      _PostRepoUpgrade(self.manifest)
+
+    else:
+      if not rp.Sync_NetworkHalf():
+        print >>sys.stderr, "error: can't update repo"
+        sys.exit(1)
+
+      _PostRepoFetch(rp,
+                     no_repo_verify = opt.no_repo_verify,
+                     verbose = True)

subcmds/status.py

@@ -20,6 +20,44 @@ class Status(PagedCommand):
   helpSummary = "Show the working tree status"
   helpUsage = """
 %prog [<project>...]
+"""
+  helpDescription = """
+'%prog' compares the working tree to the staging area (aka index),
+and the most recent commit on this branch (HEAD), in each project
+specified.  A summary is displayed, one line per file where there
+is a difference between these three states.
+
+Status Display
+--------------
+
+The status display is organized into three columns of information,
+for example if the file 'subcmds/status.py' is modified in the
+project 'repo' on branch 'devwork':
+
+  project repo/                                   branch devwork
+   -m     subcmds/status.py
+
+The first column explains how the staging area (index) differs from
+the last commit (HEAD).  Its values are always displayed in upper
+case and have the following meanings:
+
+ -:  no difference
+ A:  added         (not in HEAD,     in index                     )
+ M:  modified      (    in HEAD,     in index, different content  )
+ D:  deleted       (    in HEAD, not in index                     )
+ R:  renamed       (not in HEAD,     in index, path changed       )
+ C:  copied        (not in HEAD,     in index, copied from another)
+ T:  mode changed  (    in HEAD,     in index, same content       )
+ U:  unmerged; conflict resolution required
+
+The second column explains how the working directory differs from
+the index.  Its values are always displayed in lower case and have
+the following meanings:
+
+ -:  new / unknown (not in index,     in work tree                )
+ m:  modified      (    in index,     in work tree, modified      )
+ d:  deleted       (    in index, not in work tree                )
+
 """
 
   def Execute(self, opt, args):

subcmds/sync.py

@@ -20,9 +20,11 @@ import subprocess
 import sys
 
 from git_command import GIT
+from project import HEAD
 from command import Command, MirrorSafeCommand
 from error import RepoChangedException, GitError
 from project import R_HEADS
+from project import SyncBuffer
 from progress import Progress
 
 class Sync(Command, MirrorSafeCommand):
@@ -99,32 +101,21 @@ revision is temporarily needed.
     mp.PreSync()
 
     if opt.repo_upgraded:
-      for project in self.manifest.projects.values():
+      _PostRepoUpgrade(self.manifest)
-        if project.Exists:
-          project.PostRepoUpgrade()
 
     all = self.GetProjects(args, missing_ok=True)
 
     if not opt.local_only:
       fetched = self._Fetch(rp, mp, *all)
-
+      _PostRepoFetch(rp, opt.no_repo_verify)
-      if rp.HasChanges:
-        print >>sys.stderr, 'info: A new version of repo is available'
-        print >>sys.stderr, ''
-        if opt.no_repo_verify or _VerifyTag(rp):
-          if not rp.Sync_LocalHalf():
-            sys.exit(1)
-          print >>sys.stderr, 'info: Restarting repo with latest version'
-          raise RepoChangedException(['--repo-upgraded'])
-        else:
-          print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
-
       if opt.network_only:
         # bail out now; the rest touches the working tree
         return
 
       if mp.HasChanges:
-        if not mp.Sync_LocalHalf():
+        syncbuf = SyncBuffer(mp.config)
+        mp.Sync_LocalHalf(syncbuf)
+        if not syncbuf.Finish():
           sys.exit(1)
 
         self.manifest._Unload()
@@ -135,14 +126,40 @@ revision is temporarily needed.
             missing.append(project)
         self._Fetch(*missing)
 
+    syncbuf = SyncBuffer(mp.config,
+                         detach_head = opt.detach_head)
     pm = Progress('Syncing work tree', len(all))
     for project in all:
       pm.update()
       if project.worktree:
-        if not project.Sync_LocalHalf(
+        project.Sync_LocalHalf(syncbuf)
-            detach_head=opt.detach_head):
-          sys.exit(1)
     pm.end()
+    print >>sys.stderr
+    if not syncbuf.Finish():
+      sys.exit(1)
+
+
+def _PostRepoUpgrade(manifest):
+  for project in manifest.projects.values():
+    if project.Exists:
+      project.PostRepoUpgrade()
+
+def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
+  if rp.HasChanges:
+    print >>sys.stderr, 'info: A new version of repo is available'
+    print >>sys.stderr, ''
+    if no_repo_verify or _VerifyTag(rp):
+      syncbuf = SyncBuffer(rp.config)
+      rp.Sync_LocalHalf(syncbuf)
+      if not syncbuf.Finish():
+        sys.exit(1)
+      print >>sys.stderr, 'info: Restarting repo with latest version'
+      raise RepoChangedException(['--repo-upgraded'])
+    else:
+      print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
+  else:
+    if verbose:
+      print >>sys.stderr, 'repo version %s is current' % rp.work_git.describe(HEAD)
 
 def _VerifyTag(project):
   gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')