Change repo sync to be more friendly when updating the tree

We now try to sync all projects that can be done safely first, before
we start rebasing user commits over the upstream.  This has the nice
effect of making the local tree as close to the upstream as possible
before the user has to start resolving merge conflicts, as that extra
information in other projects may aid in the conflict resolution.

Informational output is buffered and delayed until calculation for
all projects has been done, so that the user gets one concise list
of notice messages, rather than it interrupting the progress meter.

Fast-forward output is now prefixed with the project header, so the
user can see which project that update is taking place in, and make
some relation of the diffstat back to the project name.

Rebase output is now prefixed with the project header, so that if
the rebase fails, the user can see which project we were operating
on and can try to address the failure themselves.

Since rebase sits on a detached HEAD, we now look for an in-progress
rebase during sync, so we can alert the user that the given project
is in a state we cannot handle.

Signed-off-by: Shawn O. Pearce <sop@google.com>

color.py

@@ -100,6 +100,9 @@ class Coloring(object):
     else:
       self._on = False
 
+  def redirect(self, out):
+    self._out = out
+
   @property
   def is_on(self):
     return self._on

git_config.py

@@ -24,6 +24,8 @@ R_HEADS = 'refs/heads/'
 R_TAGS  = 'refs/tags/'
 ID_RE = re.compile('^[0-9a-f]{40}$')
 
+REVIEW_CACHE = dict()
+
 def IsId(rev):
   return ID_RE.match(rev)
 
@@ -273,24 +275,44 @@ class Remote(object):
       u = self.review
       if not u.startswith('http:') and not u.startswith('https:'):
         u = 'http://%s' % u
-      if not u.endswith('/'):
-        u += '/'
-      u += 'ssh_info'
+      if u.endswith('/Gerrit'):
+        u = u[:len(u) - len('/Gerrit')]
+      if not u.endswith('/ssh_info'):
+        if not u.endswith('/'):
+          u += '/'
+        u += 'ssh_info'
 
-      try:
-        info = urlopen(u).read()
-        if info == 'NOT_AVAILABLE':
-          raise UploadError('Upload over ssh unavailable')
+      if u in REVIEW_CACHE:
+        info = REVIEW_CACHE[u]
+        self._review_protocol = info[0]
+        self._review_host = info[1]
+        self._review_port = info[2]
+      else:
+        try:
+          info = urlopen(u).read()
+          if info == 'NOT_AVAILABLE':
+            raise UploadError('Upload over ssh unavailable')
+          if '<' in info:
+            # Assume the server gave us some sort of HTML
+            # response back, like maybe a login page.
+            #
+            raise UploadError('Cannot read %s:\n%s' % (u, info))
 
-        self._review_protocol = 'ssh'
-        self._review_host = info.split(" ")[0]
-        self._review_port = info.split(" ")[1]
+          self._review_protocol = 'ssh'
+          self._review_host = info.split(" ")[0]
+          self._review_port = info.split(" ")[1]
+        except HTTPError, e:
+          if e.code == 404:
+            self._review_protocol = 'http-post'
+            self._review_host = None
+            self._review_port = None
+          else:
+            raise UploadError('Cannot guess Gerrit version')
 
-      except HTTPError, e:
-        if e.code == 404:
-          self._review_protocol = 'http-post'
-        else:
-          raise UploadError('Cannot guess Gerrit version')
+        REVIEW_CACHE[u] = (
+          self._review_protocol,
+          self._review_host,
+          self._review_port)
     return self._review_protocol
 
   def SshReviewUrl(self, userEmail):

pager.py

@@ -22,7 +22,7 @@ active = False
 def RunPager(globalConfig):
   global active
 
-  if not os.isatty(0):
+  if not os.isatty(0) or not os.isatty(1):
     return
   pager = _SelectPager(globalConfig)
   if pager == '' or pager == 'cat':

progress.py

@@ -0,0 +1,58 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+class Progress(object):
+  def __init__(self, title, total=0):
+    self._title = title
+    self._total = total
+    self._done = 0
+    self._lastp = -1
+
+  def update(self, inc=1):
+    self._done += inc
+
+    if self._total <= 0:
+      sys.stderr.write('\r%s: %d, ' % (
+        self._title,
+        self._done))
+      sys.stderr.flush()
+    else:
+      p = (100 * self._done) / self._total
+
+      if self._lastp != p:
+        self._lastp = p
+        sys.stderr.write('\r%s: %3d%% (%d/%d)  ' % (
+          self._title,
+          p,
+          self._done,
+          self._total))
+        sys.stderr.flush()
+
+  def end(self):
+    if self._total <= 0:
+      sys.stderr.write('\r%s: %d, done.  \n' % (
+        self._title,
+        self._done))
+      sys.stderr.flush()
+    else:
+      p = (100 * self._done) / self._total
+      sys.stderr.write('\r%s: %3d%% (%d/%d), done.  \n' % (
+        self._title,
+        p,
+        self._done,
+        self._total))
+      sys.stderr.flush()

project.py

@@ -34,13 +34,9 @@ R_TAGS  = 'refs/tags/'
 R_PUB   = 'refs/published/'
 R_M     = 'refs/remotes/m/'
 
-def _warn(fmt, *args):
+def _error(fmt, *args):
   msg = fmt % args
-  print >>sys.stderr, 'warn: %s' % msg
-
-def _info(fmt, *args):
-  msg = fmt % args
-  print >>sys.stderr, 'info: %s' % msg
+  print >>sys.stderr, 'error: %s' % msg
 
 def not_rev(r):
   return '^' + r
@@ -199,9 +195,7 @@ class _CopyFile:
         mode = mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
         os.chmod(dest, mode)
       except IOError:
-           print >>sys.stderr, \
-              'error: Cannot copy file %s to %s' \
-              % (src, dest)
+        _error('Cannot copy file %s to %s', src, dest)
 
 
 class Project(object):
@@ -306,6 +300,32 @@ class Project(object):
     """
     return self.config.GetBranch(name)
 
+  def GetBranches(self):
+    """Get all existing local branches.
+    """
+    current = self.CurrentBranch
+    all = self.bare_git.ListRefs()
+    heads = {}
+    pubd = {}
+
+    for name, id in all.iteritems():
+      if name.startswith(R_HEADS):
+        name = name[len(R_HEADS):]
+        b = self.GetBranch(name)
+        b.current = name == current
+        b.published = None
+        b.revision = id
+        heads[name] = b
+
+    for name, id in all.iteritems():
+      if name.startswith(R_PUB):
+        name = name[len(R_PUB):]
+        b = heads.get(name)
+        if b:
+          b.published = id
+
+    return heads
+
 
 ## Status Display ##
 
@@ -326,7 +346,7 @@ class Project(object):
     df = self.work_git.DiffZ('diff-files')
     do = self.work_git.LsOthers()
     if not di and not df and not do:
-      return
+      return 'CLEAN'
 
     out = StatusColoring(self.config)
     out.project('project %-40s', self.relpath + '/')
@@ -374,6 +394,7 @@ class Project(object):
       else:
         out.write('%s', line)
       out.nl()
+    return 'DIRTY'
 
   def PrintWorkTreeDiff(self):
     """Prints the status of the repository to stdout.
@@ -547,13 +568,9 @@ class Project(object):
     for file in self.copyfiles:
       file._Copy()
 
-  def Sync_LocalHalf(self):
+  def Sync_LocalHalf(self, syncbuf):
     """Perform only the local IO portion of the sync process.
        Network access is not required.
-
-       Return:
-         True:  the sync was successful
-         False: the sync requires user input
     """
     self._InitWorkTree()
     self.CleanPublishedCache()
@@ -568,19 +585,25 @@ class Project(object):
 
     branch = self.CurrentBranch
 
-    if branch is None:
+    if branch is None or syncbuf.detach_head:
       # Currently on a detached HEAD.  The user is assumed to
       # not have any local modifications worth worrying about.
       #
+      if os.path.exists(os.path.join(self.worktree, '.dotest')) \
+      or os.path.exists(os.path.join(self.worktree, '.git', 'rebase-apply')):
+        syncbuf.fail(self, _PriorSyncFailedError())
+        return
+
       lost = self._revlist(not_rev(rev), HEAD)
       if lost:
-        _info("[%s] Discarding %d commits", self.name, len(lost))
+        syncbuf.info(self, "discarding %d commits", len(lost))
       try:
         self._Checkout(rev, quiet=True)
-      except GitError:
-        return False
+      except GitError, e:
+        syncbuf.fail(self, e)
+        return
       self._CopyFiles()
-      return True
+      return
 
     branch = self.GetBranch(branch)
     merge = branch.LocalMerge
@@ -589,16 +612,16 @@ class Project(object):
       # The current branch has no tracking configuration.
       # Jump off it to a deatched HEAD.
       #
-      _info("[%s] Leaving %s"
-            " (does not track any upstream)",
-            self.name,
-            branch.name)
+      syncbuf.info(self,
+                   "leaving %s; does not track upstream",
+                   branch.name)
       try:
         self._Checkout(rev, quiet=True)
-      except GitError:
-        return False
+      except GitError, e:
+        syncbuf.fail(self, e)
+        return
       self._CopyFiles()
-      return True
+      return
 
     upstream_gain = self._revlist(not_rev(HEAD), rev)
     pub = self.WasPublished(branch.name)
@@ -610,25 +633,24 @@ class Project(object):
           # commits are not yet merged upstream.  We do not want
           # to rewrite the published commits so we punt.
           #
-          _info("[%s] Branch %s is published,"
-                " but is now %d commits behind.",
-                self.name, branch.name, len(upstream_gain))
-          _info("[%s] Consider merging or rebasing the"
-                " unpublished commits.", self.name)
-        return True
+          syncbuf.info(self,
+                       "branch %s is published but is now %d commits behind",
+                       branch.name,
+                       len(upstream_gain))
+          syncbuf.info(self, "consider merging or rebasing the unpublished commits")
+        return
       elif upstream_gain:
         # We can fast-forward safely.
         #
-        try:
+        def _doff():
           self._FastForward(rev)
-        except GitError:
-          return False
-        self._CopyFiles()
-        return True
+          self._CopyFiles()
+        syncbuf.later1(self, _doff)
+        return
       else:
         # Trivially no changes in the upstream.
         #
-        return True
+        return
 
     if merge == rev:
       try:
@@ -643,8 +665,7 @@ class Project(object):
       # and pray that the old upstream also wasn't in the habit
       # of rebasing itself.
       #
-      _info("[%s] Manifest switched from %s to %s",
-            self.name, merge, rev)
+      syncbuf.info(self, "manifest switched %s...%s", merge, rev)
       old_merge = merge
 
     if rev == old_merge:
@@ -655,19 +676,19 @@ class Project(object):
     if not upstream_lost and not upstream_gain:
       # Trivially no changes caused by the upstream.
       #
-      return True
+      return
 
     if self.IsDirty(consider_untracked=False):
-      _warn('[%s] commit (or discard) uncommitted changes'
-            ' before sync', self.name)
-      return False
+      syncbuf.fail(self, _DirtyError())
+      return
 
     if upstream_lost:
       # Upstream rebased.  Not everything in HEAD
       # may have been caused by the user.
       #
-      _info("[%s] Discarding %d commits removed from upstream",
-            self.name, len(upstream_lost))
+      syncbuf.info(self,
+                   "discarding %d commits removed from upstream",
+                   len(upstream_lost))
 
     branch.remote = rem
     branch.merge = self.revision
@@ -675,23 +696,22 @@ class Project(object):
 
     my_changes = self._revlist(not_rev(old_merge), HEAD)
     if my_changes:
-      try:
+      def _dorebase():
         self._Rebase(upstream = old_merge, onto = rev)
-      except GitError:
-        return False
+        self._CopyFiles()
+      syncbuf.later2(self, _dorebase)
     elif upstream_lost:
       try:
         self._ResetHard(rev)
-      except GitError:
-        return False
+        self._CopyFiles()
+      except GitError, e:
+        syncbuf.fail(self, e)
+        return
     else:
-      try:
+      def _doff():
         self._FastForward(rev)
-      except GitError:
-        return False
-
-    self._CopyFiles()
-    return True
+        self._CopyFiles()
+      syncbuf.later1(self, _doff)
 
   def AddCopyFile(self, src, dest, absdest):
     # dest should already be an absolute path, but src is project relative
@@ -722,16 +742,45 @@ class Project(object):
   def StartBranch(self, name):
     """Create a new branch off the manifest's revision.
     """
-    branch = self.GetBranch(name)
-    branch.remote = self.GetRemote(self.remote.name)
-    branch.merge = self.revision
+    try:
+      self.bare_git.rev_parse(R_HEADS + name)
+      exists = True
+    except GitError:
+      exists = False;
+
+    if exists:
+      if name == self.CurrentBranch:
+        return True
+      else:
+        cmd = ['checkout', name, '--']
+        return GitCommand(self, cmd).Wait() == 0
 
-    rev = branch.LocalMerge
-    cmd = ['checkout', '-b', branch.name, rev]
-    if GitCommand(self, cmd).Wait() == 0:
-      branch.Save()
     else:
-      raise GitError('%s checkout %s ' % (self.name, rev))
+      branch = self.GetBranch(name)
+      branch.remote = self.GetRemote(self.remote.name)
+      branch.merge = self.revision
+
+      rev = branch.LocalMerge
+      cmd = ['checkout', '-b', branch.name, rev]
+      if GitCommand(self, cmd).Wait() == 0:
+        branch.Save()
+        return True
+      else:
+        return False
+
+  def CheckoutBranch(self, name):
+    """Checkout a local topic branch.
+    """
+
+    # Be sure the branch exists
+    try:
+      tip_rev = self.bare_git.rev_parse(R_HEADS + name)
+    except GitError:
+      return False;
+
+    # Do the checkout
+    cmd = ['checkout', name, '--']
+    return GitCommand(self, cmd).Wait() == 0
 
   def AbandonBranch(self, name):
     """Destroy a local topic branch.
@@ -837,11 +886,11 @@ class Project(object):
       raise GitError('%s reset --hard %s ' % (self.name, rev))
 
   def _Rebase(self, upstream, onto = None):
-    cmd = ['rebase', '-i']
+    cmd = ['rebase']
     if onto is not None:
       cmd.extend(['--onto', onto])
     cmd.append(upstream)
-    if GitCommand(self, cmd, disable_editor=True).Wait() != 0:
+    if GitCommand(self, cmd).Wait() != 0:
       raise GitError('%s rebase %s ' % (self.name, upstream))
 
   def _FastForward(self, head):
@@ -1154,6 +1203,113 @@ class Project(object):
       return runner
 
 
+class _PriorSyncFailedError(Exception):
+  def __str__(self):
+    return 'prior sync failed; rebase still in progress'
+
+class _DirtyError(Exception):
+  def __str__(self):
+    return 'contains uncommitted changes'
+
+class _InfoMessage(object):
+  def __init__(self, project, text):
+    self.project = project
+    self.text = text
+
+  def Print(self, syncbuf):
+    syncbuf.out.info('%s/: %s', self.project.relpath, self.text)
+    syncbuf.out.nl()
+
+class _Failure(object):
+  def __init__(self, project, why):
+    self.project = project
+    self.why = why
+
+  def Print(self, syncbuf):
+    syncbuf.out.fail('error: %s/: %s',
+                     self.project.relpath,
+                     str(self.why))
+    syncbuf.out.nl()
+
+class _Later(object):
+  def __init__(self, project, action):
+    self.project = project
+    self.action = action
+
+  def Run(self, syncbuf):
+    out = syncbuf.out
+    out.project('project %s/', self.project.relpath)
+    out.nl()
+    try:
+      self.action()
+      out.nl()
+      return True
+    except GitError, e:
+      out.nl()
+      return False
+
+class _SyncColoring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, 'reposync')
+    self.project   = self.printer('header', attr = 'bold')
+    self.info      = self.printer('info')
+    self.fail      = self.printer('fail', fg='red')
+
+class SyncBuffer(object):
+  def __init__(self, config, detach_head=False):
+    self._messages = []
+    self._failures = []
+    self._later_queue1 = []
+    self._later_queue2 = []
+
+    self.out = _SyncColoring(config)
+    self.out.redirect(sys.stderr)
+
+    self.detach_head = detach_head
+    self.clean = True
+
+  def info(self, project, fmt, *args):
+    self._messages.append(_InfoMessage(project, fmt % args))
+
+  def fail(self, project, err=None):
+    self._failures.append(_Failure(project, err))
+    self.clean = False
+
+  def later1(self, project, what):
+    self._later_queue1.append(_Later(project, what))
+
+  def later2(self, project, what):
+    self._later_queue2.append(_Later(project, what))
+
+  def Finish(self):
+    self._PrintMessages()
+    self._RunLater()
+    self._PrintMessages()
+    return self.clean
+
+  def _RunLater(self):
+    for q in ['_later_queue1', '_later_queue2']:
+      if not self._RunQueue(q):
+        return
+
+  def _RunQueue(self, queue):
+    for m in getattr(self, queue):
+      if not m.Run(self):
+        self.clean = False
+        return False
+    setattr(self, queue, [])
+    return True
+
+  def _PrintMessages(self):
+    for m in self._messages:
+      m.Print(self)
+    for m in self._failures:
+      m.Print(self)
+
+    self._messages = []
+    self._failures = []
+
+
 class MetaProject(Project):
   """A special project housed under .repo.
   """

subcmds/branches.py

@@ -0,0 +1,150 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from color import Coloring
+from command import Command
+
+class BranchColoring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, 'branch')
+    self.current = self.printer('current', fg='green')
+    self.local   = self.printer('local')
+    self.notinproject = self.printer('notinproject', fg='red')
+
+class BranchInfo(object):
+  def __init__(self, name):
+    self.name = name
+    self.current = 0
+    self.published = 0
+    self.published_equal = 0
+    self.projects = []
+
+  def add(self, b):
+    if b.current:
+      self.current += 1
+    if b.published:
+      self.published += 1
+    if b.revision == b.published:
+      self.published_equal += 1
+    self.projects.append(b)
+
+  @property
+  def IsCurrent(self):
+    return self.current > 0
+
+  @property
+  def IsPublished(self):
+    return self.published > 0
+
+  @property
+  def IsPublishedEqual(self):
+    return self.published_equal == len(self.projects)
+
+
+class Branches(Command):
+  common = True
+  helpSummary = "View current topic branches"
+  helpUsage = """
+%prog [<project>...]
+
+Summarizes the currently available topic branches.
+"""
+
+  def _Options(self, p):
+    p.add_option('-a', '--all',
+                 dest='all', action='store_true',
+                 help='show all branches, not just the majority')
+
+  def Execute(self, opt, args):
+    projects = self.GetProjects(args)
+    out = BranchColoring(self.manifest.manifestProject.config)
+    all = {}
+    project_cnt = len(projects)
+
+    for project in projects:
+      for name, b in project.GetBranches().iteritems():
+        b.project = project
+        if name not in all:
+          all[name] = BranchInfo(name)
+        all[name].add(b)
+
+    names = all.keys()
+    names.sort()
+
+    if not opt.all and not args:
+      # No -a and no specific projects listed; try to filter the
+      # results down to only the majority of projects.
+      #
+      n = []
+      for name in names:
+        i = all[name]
+        if i.IsCurrent \
+        or 80 <= (100 * len(i.projects)) / project_cnt:
+          n.append(name)
+      names = n
+
+    width = 25
+    for name in names:
+      if width < len(name):
+        width = len(name)
+
+    for name in names:
+      i = all[name]
+      in_cnt = len(i.projects)
+
+      if i.IsCurrent:
+        current = '*'
+        hdr = out.current
+      else:
+        current = ' '
+        hdr = out.local
+
+      if i.IsPublishedEqual:
+        published = 'P'
+      elif i.IsPublished:
+        published = 'p'
+      else:
+        published = ' '
+
+      hdr('%c%c %-*s' % (current, published, width, name))
+      out.write(' |')
+
+      if in_cnt < project_cnt and (in_cnt == 1 or opt.all):
+        fmt = out.write
+        paths = []
+        if in_cnt < project_cnt - in_cnt: 
+          type = 'in'
+          for b in i.projects:
+            paths.append(b.project.relpath)
+        else:
+          fmt = out.notinproject
+          type = 'not in'
+          have = set()
+          for b in i.projects:
+            have.add(b.project)
+          for p in projects:
+            paths.append(p.relpath)
+
+        s = ' %s %s' % (type, ', '.join(paths))
+        if width + 7 + len(s) < 80:
+          fmt(s)
+        else:
+          out.nl()
+          fmt('    %s:' % type)
+          for p in paths:
+            out.nl()
+            fmt('      %s' % p)
+      out.nl()

subcmds/checkout.py

@@ -0,0 +1,47 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from command import Command
+
+class Checkout(Command):
+  common = True
+  helpSummary = "Checkout a branch for development"
+  helpUsage = """
+%prog <branchname> [<project>...]
+"""
+  helpDescription = """
+The '%prog' command checks out an existing branch that was previously
+created by 'repo start'.
+
+The command is equivalent to:
+
+  repo forall [<project>...] -c git checkout <branchname>
+"""
+
+  def Execute(self, opt, args):
+    if not args:
+      self.Usage()
+
+    retValue = 0;
+
+    branch = args[0]
+    for project in self.GetProjects(args[1:]):
+      if not project.CheckoutBranch(branch):
+        retValue = 1;
+        print >>sys.stderr, "error: checking out branch '%s' in %s failed" % (branch, project.name)
+
+    if (retValue != 0):
+      sys.exit(retValue);

subcmds/forall.py

@@ -30,6 +30,7 @@ Executes the same shell command in each project.
 
 Environment
 -----------
+
 pwd is the project's working directory.  If the current client is
 a mirror client, then pwd is the Git repository.
 

subcmds/grep.py

@@ -0,0 +1,243 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from optparse import SUPPRESS_HELP
+from color import Coloring
+from command import PagedCommand
+from git_command import GitCommand
+
+class GrepColoring(Coloring):
+  def __init__(self, config):
+    Coloring.__init__(self, config, 'grep')
+    self.project = self.printer('project', attr='bold')
+
+class Grep(PagedCommand):
+  common = True
+  helpSummary = "Print lines matching a pattern"
+  helpUsage = """
+%prog {pattern | -e pattern} [<project>...]
+"""
+  helpDescription = """
+Search for the specified patterns in all project files.
+
+Options
+-------
+
+The following options can appear as often as necessary to express
+the pattern to locate:
+
+ -e PATTERN
+ --and, --or, --not, -(, -)
+
+Further, the -r/--revision option may be specified multiple times
+in order to scan multiple trees.  If the same file matches in more
+than one tree, only the first result is reported, prefixed by the
+revision name it was found under.
+
+Examples
+-------
+
+Look for a line that has '#define' and either 'MAX_PATH or 'PATH_MAX':
+
+  repo grep -e '#define' --and -\( -e MAX_PATH -e PATH_MAX \)
+
+Look for a line that has 'NODE' or 'Unexpected' in files that
+contain a line that matches both expressions:
+
+  repo grep --all-match -e NODE -e Unexpected
+
+"""
+
+  def _Options(self, p):
+    def carry(option,
+              opt_str,
+              value,
+              parser):
+      pt = getattr(parser.values, 'cmd_argv', None)
+      if pt is None:
+        pt = []
+        setattr(parser.values, 'cmd_argv', pt)
+
+      if opt_str == '-(':
+        pt.append('(')
+      elif opt_str == '-)':
+        pt.append(')')
+      else:
+        pt.append(opt_str)
+
+      if value is not None:
+        pt.append(value)
+
+    g = p.add_option_group('Sources')
+    g.add_option('--cached',
+                 action='callback', callback=carry,
+                 help='Search the index, instead of the work tree')
+    g.add_option('-r','--revision',
+                 dest='revision', action='append', metavar='TREEish',
+                 help='Search TREEish, instead of the work tree')
+
+    g = p.add_option_group('Pattern')
+    g.add_option('-e',
+                 action='callback', callback=carry,
+                 metavar='PATTERN', type='str',
+                 help='Pattern to search for')
+    g.add_option('-i', '--ignore-case',
+                 action='callback', callback=carry,
+                 help='Ignore case differences')
+    g.add_option('-a','--text',
+                 action='callback', callback=carry,
+                 help="Process binary files as if they were text")
+    g.add_option('-I',
+                 action='callback', callback=carry,
+                 help="Don't match the pattern in binary files")
+    g.add_option('-w', '--word-regexp',
+                 action='callback', callback=carry,
+                 help='Match the pattern only at word boundaries')
+    g.add_option('-v', '--invert-match',
+                 action='callback', callback=carry,
+                 help='Select non-matching lines')
+    g.add_option('-G', '--basic-regexp',
+                 action='callback', callback=carry,
+                 help='Use POSIX basic regexp for patterns (default)')
+    g.add_option('-E', '--extended-regexp',
+                 action='callback', callback=carry,
+                 help='Use POSIX extended regexp for patterns')
+    g.add_option('-F', '--fixed-strings',
+                 action='callback', callback=carry,
+                 help='Use fixed strings (not regexp) for pattern')
+
+    g = p.add_option_group('Pattern Grouping')
+    g.add_option('--all-match',
+                 action='callback', callback=carry,
+                 help='Limit match to lines that have all patterns')
+    g.add_option('--and', '--or', '--not',
+                 action='callback', callback=carry,
+                 help='Boolean operators to combine patterns')
+    g.add_option('-(','-)',
+                 action='callback', callback=carry,
+                 help='Boolean operator grouping')
+
+    g = p.add_option_group('Output')
+    g.add_option('-n',
+                 action='callback', callback=carry,
+                 help='Prefix the line number to matching lines')
+    g.add_option('-C',
+                 action='callback', callback=carry,
+                 metavar='CONTEXT', type='str',
+                 help='Show CONTEXT lines around match')
+    g.add_option('-B',
+                 action='callback', callback=carry,
+                 metavar='CONTEXT', type='str',
+                 help='Show CONTEXT lines before match')
+    g.add_option('-A',
+                 action='callback', callback=carry,
+                 metavar='CONTEXT', type='str',
+                 help='Show CONTEXT lines after match')
+    g.add_option('-l','--name-only','--files-with-matches',
+                 action='callback', callback=carry,
+                 help='Show only file names containing matching lines')
+    g.add_option('-L','--files-without-match',
+                 action='callback', callback=carry,
+                 help='Show only file names not containing matching lines')
+
+
+  def Execute(self, opt, args):
+    out = GrepColoring(self.manifest.manifestProject.config)
+
+    cmd_argv = ['grep']
+    if out.is_on:
+      cmd_argv.append('--color')
+    cmd_argv.extend(getattr(opt,'cmd_argv',[]))
+
+    if '-e' not in cmd_argv:
+      if not args:
+        self.Usage()
+      cmd_argv.append('-e')
+      cmd_argv.append(args[0])
+      args = args[1:]
+
+    projects = self.GetProjects(args)
+
+    full_name = False
+    if len(projects) > 1:
+      cmd_argv.append('--full-name')
+      full_name = True
+
+    have_rev = False
+    if opt.revision:
+      if '--cached' in cmd_argv:
+        print >>sys.stderr,\
+          'fatal: cannot combine --cached and --revision'
+        sys.exit(1)
+      have_rev = True
+      cmd_argv.extend(opt.revision)
+    cmd_argv.append('--')
+
+    bad_rev = False
+    have_match = False
+
+    for project in projects:
+      p = GitCommand(project,
+                     cmd_argv,
+                     bare = False,
+                     capture_stdout = True,
+                     capture_stderr = True)
+      if p.Wait() != 0:
+        # no results
+        #
+        if p.stderr:
+          if have_rev and 'fatal: ambiguous argument' in p.stderr:
+            bad_rev = True
+          else:
+            out.project('--- project %s ---' % project.relpath)
+            out.nl()
+            out.write(p.stderr)
+            out.nl()
+        continue
+      have_match = True
+
+      # We cut the last element, to avoid a blank line.
+      #
+      r = p.stdout.split('\n')
+      r = r[0:-1]
+
+      if have_rev and full_name:
+        for line in r:
+          rev, line = line.split(':', 1)
+          out.write(rev)
+          out.write(':')
+          out.project(project.relpath)
+          out.write('/')
+          out.write(line)
+          out.nl()
+      elif full_name:
+        for line in r:
+          out.project(project.relpath)
+          out.write('/')
+          out.write(line)
+          out.nl()
+      else:
+        for line in r:
+          print line
+
+    if have_match:
+      sys.exit(0)
+    elif have_rev and bad_rev:
+      for r in opt.revision:
+        print >>sys.stderr, "error: can't search revision %s" % r
+      sys.exit(1)
+    else:
+      sys.exit(1)

subcmds/init.py

@@ -20,6 +20,7 @@ from color import Coloring
 from command import InteractiveCommand, MirrorSafeCommand
 from error import ManifestParseError
 from remote import Remote
+from project import SyncBuffer
 from git_command import git, MIN_GIT_VERSION
 
 class Init(InteractiveCommand, MirrorSafeCommand):
@@ -129,9 +130,14 @@ default.xml will be used.
       print >>sys.stderr, 'fatal: cannot obtain manifest %s' % r.url
       sys.exit(1)
 
-    m.Sync_LocalHalf()
+    syncbuf = SyncBuffer(m.config)
+    m.Sync_LocalHalf(syncbuf)
+    syncbuf.Finish()
+
     if is_new or m.CurrentBranch is None:
-      m.StartBranch('default')
+      if not m.StartBranch('default'):
+        print >>sys.stderr, 'fatal: cannot create default in manifest'
+        sys.exit(1)
 
   def _LinkManifest(self, name):
     if not name:
@@ -202,11 +208,11 @@ default.xml will be used.
     self._SyncManifest(opt)
     self._LinkManifest(opt.manifest_name)
 
-    if os.isatty(0) and os.isatty(1) and not opt.mirror:
+    if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
       self._ConfigureUser()
       self._ConfigureColor()
 
-    if opt.mirror:
+    if self.manifest.IsMirror:
       type = 'mirror '
     else:
       type = ''

subcmds/selfupdate.py

@@ -0,0 +1,59 @@
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from optparse import SUPPRESS_HELP
+import sys
+
+from command import Command, MirrorSafeCommand
+from subcmds.sync import _PostRepoUpgrade
+from subcmds.sync import _PostRepoFetch
+
+class Selfupdate(Command, MirrorSafeCommand):
+  common = False
+  helpSummary = "Update repo to the latest version"
+  helpUsage = """
+%prog
+"""
+  helpDescription = """
+The '%prog' command upgrades repo to the latest version, if a
+newer version is available.
+
+Normally this is done automatically by 'repo sync' and does not
+need to be performed by an end-user.
+"""
+
+  def _Options(self, p):
+    p.add_option('--no-repo-verify',
+                 dest='no_repo_verify', action='store_true',
+                 help='do not verify repo source code')
+    p.add_option('--repo-upgraded',
+                 dest='repo_upgraded', action='store_true',
+                 help=SUPPRESS_HELP)
+
+  def Execute(self, opt, args):
+    rp = self.manifest.repoProject
+    rp.PreSync()
+
+    if opt.repo_upgraded:
+      _PostRepoUpgrade(self.manifest)
+
+    else:
+      if not rp.Sync_NetworkHalf():
+        print >>sys.stderr, "error: can't update repo"
+        sys.exit(1)
+
+      _PostRepoFetch(rp,
+                     no_repo_verify = opt.no_repo_verify,
+                     verbose = True)

subcmds/start.py

@@ -47,5 +47,13 @@ the configuration data is set up properly.
       print >>sys.stderr, "error: '%s' is not a valid name" % nb
       sys.exit(1)
 
+    err = []
     for project in self.GetProjects(args[1:]):
-      project.StartBranch(nb)
+      if not project.StartBranch(nb):
+        err.append(project)
+
+    if err:
+      err.sort()
+      for p in err:
+        print >>sys.stderr, "error: cannot start in %s" % p.relpath
+      sys.exit(1)

subcmds/status.py

@@ -20,8 +20,53 @@ class Status(PagedCommand):
   helpSummary = "Show the working tree status"
   helpUsage = """
 %prog [<project>...]
+"""
+  helpDescription = """
+'%prog' compares the working tree to the staging area (aka index),
+and the most recent commit on this branch (HEAD), in each project
+specified.  A summary is displayed, one line per file where there
+is a difference between these three states.
+
+Status Display
+--------------
+
+The status display is organized into three columns of information,
+for example if the file 'subcmds/status.py' is modified in the
+project 'repo' on branch 'devwork':
+
+  project repo/                                   branch devwork
+   -m     subcmds/status.py
+
+The first column explains how the staging area (index) differs from
+the last commit (HEAD).  Its values are always displayed in upper
+case and have the following meanings:
+
+ -:  no difference
+ A:  added         (not in HEAD,     in index                     )
+ M:  modified      (    in HEAD,     in index, different content  )
+ D:  deleted       (    in HEAD, not in index                     )
+ R:  renamed       (not in HEAD,     in index, path changed       )
+ C:  copied        (not in HEAD,     in index, copied from another)
+ T:  mode changed  (    in HEAD,     in index, same content       )
+ U:  unmerged; conflict resolution required
+
+The second column explains how the working directory differs from
+the index.  Its values are always displayed in lower case and have
+the following meanings:
+
+ -:  new / unknown (not in index,     in work tree                )
+ m:  modified      (    in index,     in work tree, modified      )
+ d:  deleted       (    in index, not in work tree                )
+
 """
 
   def Execute(self, opt, args):
-    for project in self.GetProjects(args):
-      project.PrintWorkTreeStatus()
+    all = self.GetProjects(args)
+    clean = 0
+
+    for project in all:
+      state = project.PrintWorkTreeStatus()
+      if state == 'CLEAN':
+        clean += 1
+    if len(all) == clean:
+      print 'nothing to commit (working directory clean)'

subcmds/sync.py

@@ -13,15 +13,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from optparse import SUPPRESS_HELP
 import os
 import re
 import subprocess
 import sys
 
 from git_command import GIT
+from project import HEAD
 from command import Command, MirrorSafeCommand
 from error import RepoChangedException, GitError
 from project import R_HEADS
+from project import SyncBuffer
+from progress import Progress
 
 class Sync(Command, MirrorSafeCommand):
   common = True
@@ -43,27 +47,53 @@ line.  Projects can be specified either by name, or by a relative
 or absolute path to the project's local directory. If no projects
 are specified, '%prog' will synchronize all projects listed in
 the manifest.
+
+The -d/--detach option can be used to switch specified projects
+back to the manifest revision.  This option is especially helpful
+if the project is currently on a topic branch, but the manifest
+revision is temporarily needed.
 """
 
   def _Options(self, p):
+    p.add_option('-l','--local-only',
+                 dest='local_only', action='store_true',
+                 help="only update working tree, don't fetch")
+    p.add_option('-n','--network-only',
+                 dest='network_only', action='store_true',
+                 help="fetch only, don't update working tree")
+    p.add_option('-d','--detach',
+                 dest='detach_head', action='store_true',
+                 help='detach projects back to manifest revision')
+
     p.add_option('--no-repo-verify',
                  dest='no_repo_verify', action='store_true',
                  help='do not verify repo source code')
     p.add_option('--repo-upgraded',
                  dest='repo_upgraded', action='store_true',
-                 help='perform additional actions after a repo upgrade')
+                 help=SUPPRESS_HELP)
 
   def _Fetch(self, *projects):
     fetched = set()
+    pm = Progress('Fetching projects', len(projects))
     for project in projects:
+      pm.update()
+
       if project.Sync_NetworkHalf():
         fetched.add(project.gitdir)
       else:
         print >>sys.stderr, 'error: Cannot fetch %s' % project.name
         sys.exit(1)
+    pm.end()
     return fetched
 
   def Execute(self, opt, args):
+    if opt.network_only and opt.detach_head:
+      print >>sys.stderr, 'error: cannot combine -n and -d'
+      sys.exit(1)
+    if opt.network_only and opt.local_only:
+      print >>sys.stderr, 'error: cannot combine -n and -l'
+      sys.exit(1)
+
     rp = self.manifest.repoProject
     rp.PreSync()
 
@@ -71,42 +101,66 @@ the manifest.
     mp.PreSync()
 
     if opt.repo_upgraded:
-      for project in self.manifest.projects.values():
-        if project.Exists:
-          project.PostRepoUpgrade()
+      _PostRepoUpgrade(self.manifest)
 
     all = self.GetProjects(args, missing_ok=True)
-    fetched = self._Fetch(rp, mp, *all)
 
-    if rp.HasChanges:
-      print >>sys.stderr, 'info: A new version of repo is available'
-      print >>sys.stderr, ''
-      if opt.no_repo_verify or _VerifyTag(rp):
-        if not rp.Sync_LocalHalf():
+    if not opt.local_only:
+      fetched = self._Fetch(rp, mp, *all)
+      _PostRepoFetch(rp, opt.no_repo_verify)
+      if opt.network_only:
+        # bail out now; the rest touches the working tree
+        return
+
+      if mp.HasChanges:
+        syncbuf = SyncBuffer(mp.config)
+        mp.Sync_LocalHalf(syncbuf)
+        if not syncbuf.Finish():
           sys.exit(1)
-        print >>sys.stderr, 'info: Restarting repo with latest version'
-        raise RepoChangedException(['--repo-upgraded'])
-      else:
-        print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
 
-    if mp.HasChanges:
-      if not mp.Sync_LocalHalf():
-        sys.exit(1)
-
-      self.manifest._Unload()
-      all = self.GetProjects(args, missing_ok=True)
-      missing = []
-      for project in all:
-        if project.gitdir not in fetched:
-          missing.append(project)
-      self._Fetch(*missing)
+        self.manifest._Unload()
+        all = self.GetProjects(args, missing_ok=True)
+        missing = []
+        for project in all:
+          if project.gitdir not in fetched:
+            missing.append(project)
+        self._Fetch(*missing)
 
+    syncbuf = SyncBuffer(mp.config,
+                         detach_head = opt.detach_head)
+    pm = Progress('Syncing work tree', len(all))
     for project in all:
+      pm.update()
       if project.worktree:
-        if not project.Sync_LocalHalf():
-          sys.exit(1)
+        project.Sync_LocalHalf(syncbuf)
+    pm.end()
+    print >>sys.stderr
+    if not syncbuf.Finish():
+      sys.exit(1)
 
 
+def _PostRepoUpgrade(manifest):
+  for project in manifest.projects.values():
+    if project.Exists:
+      project.PostRepoUpgrade()
+
+def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
+  if rp.HasChanges:
+    print >>sys.stderr, 'info: A new version of repo is available'
+    print >>sys.stderr, ''
+    if no_repo_verify or _VerifyTag(rp):
+      syncbuf = SyncBuffer(rp.config)
+      rp.Sync_LocalHalf(syncbuf)
+      if not syncbuf.Finish():
+        sys.exit(1)
+      print >>sys.stderr, 'info: Restarting repo with latest version'
+      raise RepoChangedException(['--repo-upgraded'])
+    else:
+      print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
+  else:
+    if verbose:
+      print >>sys.stderr, 'repo version %s is current' % rp.work_git.describe(HEAD)
+
 def _VerifyTag(project):
   gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
   if not os.path.exists(gpg_dir):

subcmds/upload.py

@@ -234,9 +234,6 @@ files and description associated with the change in Gerrit.
           print >>sys.stderr, '[OK    ] %-15s %s' % (
                  branch.project.relpath + '/',
                  branch.name)
-          print >>sys.stderr, '%s' % branch.tip_url
-          print >>sys.stderr, '(as %s)' % branch.owner_email
-          print >>sys.stderr, ''
 
     if have_errors:
       sys.exit(1)