git_config: hoist Windows ssh check earlier

The ssh master logic has never worked under Windows which is why this code always returned False when running there (including cygwin). But the OS check was still done while holding the threading lock. While it might be a little slower than necessary, it still worked. The switch from the threading module to the multiprocessing module changed global behavior subtly under Windows and broke things: the globals previously would stay valid, but now they get cleared. So the lock is reset to None in children workers. We could tweak the logic to pass the lock through, but there isn't much point when the rest of the code is still disabled in Windows. So perform the platform check before we grab the lock. This fixes the crash, and probably speeds things up a few nanoseconds. This shouldn't be a problem on Linux systems as the platform fork will duplicate the existing process memory (including globals). Bug: https://crbug.com/gerrit/14480 Change-Id: I1d1da82c6d7bd6b8cdc1f03f640a520ecd047063 Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/305149 Reviewed-by: Raman Tenneti <rtenneti@google.com> Tested-by: Mike Frysinger <vapier@google.com>
sync: fix print error when handling server error
2025-06-26 20:17:52 +00:00 · 2021-05-04 19:49:58 -04:00 · 2021-05-04 12:48:42 -04:00 · 2021-04-22 18:00:32 +00:00 · 2021-04-21 15:37:16 +00:00 · 2021-04-21 04:45:59 +00:00
32 changed files with 578 additions and 585 deletions
--- a/.github/workflows/test-ci.yml
+++ b/.github/workflows/test-ci.yml
@ -14,7 +14,7 @@ jobs:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.5, 3.6, 3.7, 3.8, 3.9]
    runs-on: ${{ matrix.os }}

    steps:
--- a/command.py
+++ b/command.py
@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+import multiprocessing
 import os
 import optparse
 import platform
@ -21,6 +22,7 @@ import sys
 from event_log import EventLog
 from error import NoSuchProjectError
 from error import InvalidProjectGroupsError
+import progress


 # Number of projects to submit to a single worker process at a time.
@ -84,18 +86,34 @@ class Command(object):
        usage = 'repo %s' % self.NAME
      epilog = 'Run `repo help %s` to view the detailed manual.' % self.NAME
      self._optparse = optparse.OptionParser(usage=usage, epilog=epilog)
+      self._CommonOptions(self._optparse)
      self._Options(self._optparse)
    return self._optparse

-  def _Options(self, p):
-    """Initialize the option parser.
+  def _CommonOptions(self, p, opt_v=True):
+    """Initialize the option parser with common options.
+
+    These will show up for *all* subcommands, so use sparingly.
+    NB: Keep in sync with repo:InitParser().
    """
+    g = p.add_option_group('Logging options')
+    opts = ['-v'] if opt_v else []
+    g.add_option(*opts, '--verbose',
+                 dest='output_mode', action='store_true',
+                 help='show all output')
+    g.add_option('-q', '--quiet',
+                 dest='output_mode', action='store_false',
+                 help='only show errors')
+
    if self.PARALLEL_JOBS is not None:
      p.add_option(
          '-j', '--jobs',
          type=int, default=self.PARALLEL_JOBS,
          help='number of jobs to run in parallel (default: %s)' % self.PARALLEL_JOBS)

+  def _Options(self, p):
+    """Initialize the option parser with subcommand-specific options."""
+
  def _RegisteredEnvironmentOptions(self):
    """Get options that can be set from environment variables.

@ -120,6 +138,11 @@ class Command(object):
    self.OptionParser.print_usage()
    sys.exit(1)

+  def CommonValidateOptions(self, opt, args):
+    """Validate common options."""
+    opt.quiet = opt.output_mode is False
+    opt.verbose = opt.output_mode is True
+
  def ValidateOptions(self, opt, args):
    """Validate the user options & arguments before executing.

@ -135,6 +158,44 @@ class Command(object):
    """
    raise NotImplementedError

+  @staticmethod
+  def ExecuteInParallel(jobs, func, inputs, callback, output=None, ordered=False):
+    """Helper for managing parallel execution boiler plate.
+
+    For subcommands that can easily split their work up.
+
+    Args:
+      jobs: How many parallel processes to use.
+      func: The function to apply to each of the |inputs|.  Usually a
+          functools.partial for wrapping additional arguments.  It will be run
+          in a separate process, so it must be pickalable, so nested functions
+          won't work.  Methods on the subcommand Command class should work.
+      inputs: The list of items to process.  Must be a list.
+      callback: The function to pass the results to for processing.  It will be
+          executed in the main thread and process the results of |func| as they
+          become available.  Thus it may be a local nested function.  Its return
+          value is passed back directly.  It takes three arguments:
+          - The processing pool (or None with one job).
+          - The |output| argument.
+          - An iterator for the results.
+      output: An output manager.  May be progress.Progess or color.Coloring.
+      ordered: Whether the jobs should be processed in order.
+
+    Returns:
+      The |callback| function's results are returned.
+    """
+    try:
+      # NB: Multiprocessing is heavy, so don't spin it up for one job.
+      if len(inputs) == 1 or jobs == 1:
+        return callback(None, output, (func(x) for x in inputs))
+      else:
+        with multiprocessing.Pool(jobs) as pool:
+          submit = pool.imap if ordered else pool.imap_unordered
+          return callback(pool, output, submit(func, inputs, chunksize=WORKER_BATCH_SIZE))
+    finally:
+      if isinstance(output, progress.Progress):
+        output.end()
+
  def _ResetPathToProjectMap(self, projects):
    self._by_path = dict((p.worktree, p) for p in projects)

--- a/docs/internal-fs-layout.md
+++ b/docs/internal-fs-layout.md
@ -147,22 +147,23 @@ repo client checkout.
 Most settings use the `[repo]` section to avoid conflicts with git.
 User controlled settings are initialized when running `repo init`.

-| Setting           | `repo init` Option        | Use/Meaning |
-|-------------------|---------------------------|-------------|
-| manifest.groups   | `--groups` & `--platform` | The manifest groups to sync |
-| repo.archive      | `--archive`               | Use `git archive` for checkouts |
-| repo.clonebundle  | `--clone-bundle`          | Whether the initial sync used clone.bundle explicitly |
-| repo.clonefilter  | `--clone-filter`          | Filter setting when using [partial git clones] |
-| repo.depth        | `--depth`                 | Create shallow checkouts when cloning |
-| repo.dissociate   | `--dissociate`            | Dissociate from any reference/mirrors after initial clone |
-| repo.mirror       | `--mirror`                | Checkout is a repo mirror |
-| repo.partialclone | `--partial-clone`         | Create [partial git clones] |
-| repo.reference    | `--reference`             | Reference repo client checkout |
-| repo.submodules   | `--submodules`            | Sync git submodules |
-| repo.superproject | `--use-superproject`      | Sync [superproject] |
-| repo.worktree     | `--worktree`              | Use [git worktree] for checkouts |
-| user.email        | `--config-name`           | User's e-mail address; Copied into `.git/config` when checking out a new project |
-| user.name         | `--config-name`           | User's name; Copied into `.git/config` when checking out a new project |
+| Setting                  | `repo init` Option        | Use/Meaning |
+|-------------------       |---------------------------|-------------|
+| manifest.groups          | `--groups` & `--platform` | The manifest groups to sync |
+| repo.archive             | `--archive`               | Use `git archive` for checkouts |
+| repo.clonebundle         | `--clone-bundle`          | Whether the initial sync used clone.bundle explicitly |
+| repo.clonefilter         | `--clone-filter`          | Filter setting when using [partial git clones] |
+| repo.depth               | `--depth`                 | Create shallow checkouts when cloning |
+| repo.dissociate          | `--dissociate`            | Dissociate from any reference/mirrors after initial clone |
+| repo.mirror              | `--mirror`                | Checkout is a repo mirror |
+| repo.partialclone        | `--partial-clone`         | Create [partial git clones] |
+| repo.partialcloneexclude | `--partial-clone-exclude` | Comma-delimited list of project names (not paths) to exclude while using [partial git clones] |
+| repo.reference           | `--reference`             | Reference repo client checkout |
+| repo.submodules          | `--submodules`            | Sync git submodules |
+| repo.superproject        | `--use-superproject`      | Sync [superproject] |
+| repo.worktree            | `--worktree`              | Use [git worktree] for checkouts |
+| user.email               | `--config-name`           | User's e-mail address; Copied into `.git/config` when checking out a new project |
+| user.name                | `--config-name`           | User's name; Copied into `.git/config` when checking out a new project |

 [partial git clones]: https://git-scm.com/docs/gitrepository-layout#_code_partialclone_code
 [superproject]: https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects
--- a/docs/manifest-format.md
+++ b/docs/manifest-format.md
@ -21,6 +21,7 @@ following DTD:

 ```xml
 <!DOCTYPE manifest [
+
  <!ELEMENT manifest (notice?,
                      remote*,
                      default?,
--- a/git_config.py
+++ b/git_config.py
@ -459,6 +459,11 @@ def init_ssh():
 def _open_ssh(host, port=None):
  global _ssh_master

+  # Bail before grabbing the lock if we already know that we aren't going to
+  # try creating new masters below.
+  if sys.platform in ('win32', 'cygwin'):
+    return False
+
  # Acquire the lock.  This is needed to prevent opening multiple masters for
  # the same host when we're running "repo sync -jN" (for N > 1) _and_ the
  # manifest <remote fetch="ssh://xyz"> specifies a different host from the
@ -476,11 +481,8 @@ def _open_ssh(host, port=None):
    if key in _master_keys:
      return True

-    if (not _ssh_master
-            or 'GIT_SSH' in os.environ
-            or sys.platform in ('win32', 'cygwin')):
-      # failed earlier, or cygwin ssh can't do this
-      #
+    if not _ssh_master or 'GIT_SSH' in os.environ:
+      # Failed earlier, so don't retry.
      return False

    # We will make two calls to ssh; this is the common part of both calls.
--- a/git_superproject.py
+++ b/git_superproject.py
@ -121,7 +121,7 @@ class Superproject(object):
      print('git fetch missing drectory: %s' % self._work_git,
            file=sys.stderr)
      return False
-    cmd = ['fetch', url, '--force', '--no-tags', '--filter', 'blob:none']
+    cmd = ['fetch', url, '--depth', '1', '--force', '--no-tags', '--filter', 'blob:none']
    if self._branch:
      cmd += [self._branch + ':' + self._branch]
    p = GitCommand(None,
--- a/main.py
+++ b/main.py
@ -257,6 +257,7 @@ class _Repo(object):
    git_trace2_event_log.CommandEvent(name='repo', subcommands=[name])

    try:
+      cmd.CommonValidateOptions(copts, cargs)
      cmd.ValidateOptions(copts, cargs)
      result = cmd.Execute(copts, cargs)
    except (DownloadError, ManifestInvalidRevisionError,
--- a/manifest_xml.py
+++ b/manifest_xml.py
@ -589,6 +589,12 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
      return self.manifestProject.config.GetString('repo.clonefilter')
    return None

+  @property
+  def PartialCloneExclude(self):
+    exclude = self.manifest.manifestProject.config.GetString(
+        'repo.partialcloneexclude') or ''
+    return set(x.strip() for x in exclude.split(','))
+
  @property
  def IsMirror(self):
    return self.manifestProject.config.GetBoolean('repo.mirror')
--- a/progress.py
+++ b/progress.py
@ -42,18 +42,26 @@ def duration_str(total):


 class Progress(object):
-  def __init__(self, title, total=0, units='', print_newline=False):
+  def __init__(self, title, total=0, units='', print_newline=False, delay=True,
+               quiet=False):
    self._title = title
    self._total = total
    self._done = 0
    self._start = time()
-    self._show = False
+    self._show = not delay
    self._units = units
    self._print_newline = print_newline
    # Only show the active jobs section if we run more than one in parallel.
    self._show_jobs = False
    self._active = 0

+    # When quiet, never show any output.  It's a bit hacky, but reusing the
+    # existing logic that delays initial output keeps the rest of the class
+    # clean.  Basically we set the start time to years in the future.
+    if quiet:
+      self._show = False
+      self._start += 2**32
+
  def start(self, name):
    self._active += 1
    if not self._show_jobs:
--- a/project.py
+++ b/project.py
@ -1050,7 +1050,8 @@ class Project(object):
                       retry_fetches=0,
                       prune=False,
                       submodules=False,
-                       clone_filter=None):
+                       clone_filter=None,
+                       partial_clone_exclude=set()):
    """Perform only the network IO portion of the sync process.
       Local working directory/branch state is not affected.
    """
@ -1087,6 +1088,10 @@ class Project(object):
    if clone_bundle and os.path.exists(self.objdir):
      clone_bundle = False

+    if self.name in partial_clone_exclude:
+      clone_bundle = True
+      clone_filter = None
+
    if is_new is None:
      is_new = not self.Exists
    if is_new:
@ -2170,19 +2175,13 @@ class Project(object):
      elif (gitcmd.stdout and
            'error:' in gitcmd.stdout and
            'HTTP 429' in gitcmd.stdout):
-        if not quiet:
-          print('429 received, sleeping: %s sec' % retry_cur_sleep,
-                file=sys.stderr)
-        time.sleep(retry_cur_sleep)
-        retry_cur_sleep = min(retry_exp_factor * retry_cur_sleep,
-                              MAXIMUM_RETRY_SLEEP_SEC)
-        retry_cur_sleep *= (1 - random.uniform(-RETRY_JITTER_PERCENT,
-                                               RETRY_JITTER_PERCENT))
-        continue
+        # Fallthru to sleep+retry logic at the bottom.
+        pass

-      # If this is not last attempt, try 'git remote prune'.
-      elif (try_n < retry_fetches - 1 and
-            gitcmd.stdout and
+      # Try to prune remote branches once in case there are conflicts.
+      # For example, if the remote had refs/heads/upstream, but deleted that and
+      # now has refs/heads/upstream/foo.
+      elif (gitcmd.stdout and
            'error:' in gitcmd.stdout and
            'git remote prune' in gitcmd.stdout and
            not prune_tried):
@ -2192,6 +2191,8 @@ class Project(object):
        ret = prunecmd.Wait()
        if ret:
          break
+        output_redir.write('retrying fetch after pruning remote branches')
+        # Continue right away so we don't sleep as we shouldn't need to.
        continue
      elif current_branch_only and is_sha1 and ret == 128:
        # Exit code 128 means "couldn't find the ref you asked for"; if we're
@ -2201,9 +2202,17 @@ class Project(object):
      elif ret < 0:
        # Git died with a signal, exit immediately
        break
+
+      # Figure out how long to sleep before the next attempt, if there is one.
      if not verbose:
-        print('\n%s:\n%s' % (self.name, gitcmd.stdout), file=sys.stderr)
-      time.sleep(random.randint(30, 45))
+        output_redir.write('\n%s:\n%s' % (self.name, gitcmd.stdout))
+      if try_n < retry_fetches - 1:
+        output_redir.write('sleeping %s seconds before retrying' % retry_cur_sleep)
+        time.sleep(retry_cur_sleep)
+        retry_cur_sleep = min(retry_exp_factor * retry_cur_sleep,
+                              MAXIMUM_RETRY_SLEEP_SEC)
+        retry_cur_sleep *= (1 - random.uniform(-RETRY_JITTER_PERCENT,
+                                               RETRY_JITTER_PERCENT))

    if initial:
      if alt_dir:
--- a/64
+++ b/64
@ -147,7 +147,7 @@ if not REPO_REV:
  REPO_REV = 'stable'

 # increment this whenever we make important changes to this script
-VERSION = (2, 12)
+VERSION = (2, 14)

 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (2, 3)
@ -275,6 +275,13 @@ def GetParser(gitc_init=False):
    usage = 'repo init [options] [-u] url'

  parser = optparse.OptionParser(usage=usage)
+  InitParser(parser, gitc_init=gitc_init)
+  return parser
+
+
+def InitParser(parser, gitc_init=False):
+  """Setup the CLI parser."""
+  # NB: Keep in sync with command.py:_CommonOptions().

  # Logging.
  group = parser.add_option_group('Logging options')
@ -291,8 +298,22 @@ def GetParser(gitc_init=False):
                   help='manifest repository location', metavar='URL')
  group.add_option('-b', '--manifest-branch', metavar='REVISION',
                   help='manifest branch or revision (use HEAD for default)')
-  group.add_option('-m', '--manifest-name',
+  group.add_option('-m', '--manifest-name', default='default.xml',
                   help='initial manifest file', metavar='NAME.xml')
+  group.add_option('-g', '--groups', default='default',
+                   help='restrict manifest projects to ones with specified '
+                        'group(s) [default|all|G1,G2,G3|G4,-G5,-G6]',
+                   metavar='GROUP')
+  group.add_option('-p', '--platform', default='auto',
+                   help='restrict manifest projects to ones with a specified '
+                        'platform group [auto|all|none|linux|darwin|...]',
+                   metavar='PLATFORM')
+  group.add_option('--submodules', action='store_true',
+                   help='sync any submodules associated with the manifest repo')
+
+  # Options that only affect manifest project, and not any of the projects
+  # specified in the manifest itself.
+  group = parser.add_option_group('Manifest (only) checkout options')
  cbr_opts = ['--current-branch']
  # The gitc-init subcommand allocates -c itself, but a lot of init users
  # want -c, so try to satisfy both as best we can.
@ -301,9 +322,23 @@ def GetParser(gitc_init=False):
  group.add_option(*cbr_opts,
                   dest='current_branch_only', action='store_true',
                   help='fetch only current manifest branch from server')
+  group.add_option('--no-tags',
+                   dest='tags', default=True, action='store_false',
+                   help="don't fetch tags in the manifest")
+
+  # These are fundamentally different ways of structuring the checkout.
+  group = parser.add_option_group('Checkout modes')
  group.add_option('--mirror', action='store_true',
                   help='create a replica of the remote repositories '
                        'rather than a client working directory')
+  group.add_option('--archive', action='store_true',
+                   help='checkout an archive instead of a git repository for '
+                        'each project. See git archive.')
+  group.add_option('--worktree', action='store_true',
+                   help='use git-worktree to manage projects')
+
+  # These are fundamentally different ways of structuring the checkout.
+  group = parser.add_option_group('Project checkout optimizations')
  group.add_option('--reference',
                   help='location of mirror directory', metavar='DIR')
  group.add_option('--dissociate', action='store_true',
@ -314,38 +349,27 @@ def GetParser(gitc_init=False):
  group.add_option('--partial-clone', action='store_true',
                   help='perform partial clone (https://git-scm.com/'
                        'docs/gitrepository-layout#_code_partialclone_code)')
+  group.add_option('--no-partial-clone', action='store_false',
+                   help='disable use of partial clone (https://git-scm.com/'
+                        'docs/gitrepository-layout#_code_partialclone_code)')
+  group.add_option('--partial-clone-exclude', action='store',
+                   help='exclude the specified projects (a comma-delimited '
+                        'project names) from partial clone (https://git-scm.com'
+                        '/docs/gitrepository-layout#_code_partialclone_code)')
  group.add_option('--clone-filter', action='store', default='blob:none',
                   help='filter for use with --partial-clone '
                        '[default: %default]')
-  group.add_option('--worktree', action='store_true',
-                   help='use git-worktree to manage projects')
-  group.add_option('--archive', action='store_true',
-                   help='checkout an archive instead of a git repository for '
-                        'each project. See git archive.')
-  group.add_option('--submodules', action='store_true',
-                   help='sync any submodules associated with the manifest repo')
  group.add_option('--use-superproject', action='store_true', default=None,
                   help='use the manifest superproject to sync projects')
  group.add_option('--no-use-superproject', action='store_false',
                   dest='use_superproject',
                   help='disable use of manifest superprojects')
-  group.add_option('-g', '--groups', default='default',
-                   help='restrict manifest projects to ones with specified '
-                        'group(s) [default|all|G1,G2,G3|G4,-G5,-G6]',
-                   metavar='GROUP')
-  group.add_option('-p', '--platform', default='auto',
-                   help='restrict manifest projects to ones with a specified '
-                        'platform group [auto|all|none|linux|darwin|...]',
-                   metavar='PLATFORM')
  group.add_option('--clone-bundle', action='store_true',
                   help='enable use of /clone.bundle on HTTP/HTTPS '
                        '(default if not --partial-clone)')
  group.add_option('--no-clone-bundle',
                   dest='clone_bundle', action='store_false',
                   help='disable use of /clone.bundle on HTTP/HTTPS (default if --partial-clone)')
-  group.add_option('--no-tags',
-                   dest='tags', default=True, action='store_false',
-                   help="don't fetch tags in the manifest")

  # Tool.
  group = parser.add_option_group('repo Version options')
--- a/6
+++ b/6
@ -34,8 +34,8 @@ def find_pytest():
  if ret:
    return ret

-  print(f'{__file__}: unable to find pytest.', file=sys.stderr)
-  print(f'{__file__}: Try installing: sudo apt-get install python-pytest',
+  print('%s: unable to find pytest.' % (__file__,), file=sys.stderr)
+  print('%s: Try installing: sudo apt-get install python-pytest' % (__file__,),
        file=sys.stderr)


@ -50,7 +50,7 @@ def main(argv):
  os.environ['PYTHONPATH'] = pythonpath

  pytest = find_pytest()
-  return subprocess.run([pytest] + argv, check=True)
+  return subprocess.run([pytest] + argv, check=False).returncode


 if __name__ == '__main__':
--- a/setup.py
+++ b/setup.py
@ -32,7 +32,7 @@ with open(os.path.join(TOPDIR, 'README.md')) as fp:
 # https://packaging.python.org/tutorials/packaging-projects/
 setuptools.setup(
    name='repo',
-    version='1.13.8',
+    version='2',
    maintainer='Various',
    maintainer_email='repo-discuss@googlegroups.com',
    description='Repo helps manage many Git repositories',
@ -56,6 +56,6 @@ setuptools.setup(
        'Programming Language :: Python :: 3 :: Only',
        'Topic :: Software Development :: Version Control :: Git',
    ],
-    python_requires='>=3.6',
+    python_requires='>=3.5',
    packages=['subcmds'],
 )
--- a/subcmds/abandon.py
+++ b/subcmds/abandon.py
@ -15,10 +15,9 @@
 from collections import defaultdict
 import functools
 import itertools
-import multiprocessing
 import sys

-from command import Command, DEFAULT_LOCAL_JOBS, WORKER_BATCH_SIZE
+from command import Command, DEFAULT_LOCAL_JOBS
 from git_command import git
 from progress import Progress

@ -37,10 +36,6 @@ It is equivalent to "git branch -D <branchname>".
  PARALLEL_JOBS = DEFAULT_LOCAL_JOBS

  def _Options(self, p):
-    super()._Options(p)
-    p.add_option('-q', '--quiet',
-                 action='store_true', default=False,
-                 help='be quiet')
    p.add_option('--all',
                 dest='all', action='store_true',
                 help='delete all branches in all projects')
@ -56,9 +51,9 @@ It is equivalent to "git branch -D <branchname>".
    else:
      args.insert(0, "'All local branches'")

-  def _ExecuteOne(self, opt, nb, project):
+  def _ExecuteOne(self, all_branches, nb, project):
    """Abandon one project."""
-    if opt.all:
+    if all_branches:
      branches = project.GetBranches()
    else:
      branches = [nb]
@ -76,7 +71,7 @@ It is equivalent to "git branch -D <branchname>".
    success = defaultdict(list)
    all_projects = self.GetProjects(args[1:])

-    def _ProcessResults(states):
+    def _ProcessResults(_pool, pm, states):
      for (results, project) in states:
        for branch, status in results.items():
          if status:
@ -85,17 +80,12 @@ It is equivalent to "git branch -D <branchname>".
            err[branch].append(project)
        pm.update()

-    pm = Progress('Abandon %s' % nb, len(all_projects))
-    # NB: Multiprocessing is heavy, so don't spin it up for one job.
-    if len(all_projects) == 1 or opt.jobs == 1:
-      _ProcessResults(self._ExecuteOne(opt, nb, x) for x in all_projects)
-    else:
-      with multiprocessing.Pool(opt.jobs) as pool:
-        states = pool.imap_unordered(
-            functools.partial(self._ExecuteOne, opt, nb), all_projects,
-            chunksize=WORKER_BATCH_SIZE)
-        _ProcessResults(states)
-    pm.end()
+    self.ExecuteInParallel(
+        opt.jobs,
+        functools.partial(self._ExecuteOne, opt.all, nb),
+        all_projects,
+        callback=_ProcessResults,
+        output=Progress('Abandon %s' % (nb,), len(all_projects), quiet=opt.quiet))

    width = max(itertools.chain(
        [25], (len(x) for x in itertools.chain(success, err))))
--- a/subcmds/branches.py
+++ b/subcmds/branches.py
@ -13,10 +13,10 @@
 # limitations under the License.

 import itertools
-import multiprocessing
 import sys
+
 from color import Coloring
-from command import Command, DEFAULT_LOCAL_JOBS, WORKER_BATCH_SIZE
+from command import Command, DEFAULT_LOCAL_JOBS


 class BranchColoring(Coloring):
@ -102,15 +102,19 @@ is shown, then the branch appears in all projects.
    out = BranchColoring(self.manifest.manifestProject.config)
    all_branches = {}
    project_cnt = len(projects)
-    with multiprocessing.Pool(processes=opt.jobs) as pool:
-      project_branches = pool.imap_unordered(
-          expand_project_to_branches, projects, chunksize=WORKER_BATCH_SIZE)

-      for name, b in itertools.chain.from_iterable(project_branches):
+    def _ProcessResults(_pool, _output, results):
+      for name, b in itertools.chain.from_iterable(results):
        if name not in all_branches:
          all_branches[name] = BranchInfo(name)
        all_branches[name].add(b)

+    self.ExecuteInParallel(
+        opt.jobs,
+        expand_project_to_branches,
+        projects,
+        callback=_ProcessResults)
+
    names = sorted(all_branches)

    if not names:
--- a/subcmds/checkout.py
+++ b/subcmds/checkout.py
@ -13,10 +13,9 @@
 # limitations under the License.

 import functools
-import multiprocessing
 import sys

-from command import Command, DEFAULT_LOCAL_JOBS, WORKER_BATCH_SIZE
+from command import Command, DEFAULT_LOCAL_JOBS
 from progress import Progress


@ -50,7 +49,7 @@ The command is equivalent to:
    success = []
    all_projects = self.GetProjects(args[1:])

-    def _ProcessResults(results):
+    def _ProcessResults(_pool, pm, results):
      for status, project in results:
        if status is not None:
          if status:
@ -59,17 +58,12 @@ The command is equivalent to:
            err.append(project)
        pm.update()

-    pm = Progress('Checkout %s' % nb, len(all_projects))
-    # NB: Multiprocessing is heavy, so don't spin it up for one job.
-    if len(all_projects) == 1 or opt.jobs == 1:
-      _ProcessResults(self._ExecuteOne(nb, x) for x in all_projects)
-    else:
-      with multiprocessing.Pool(opt.jobs) as pool:
-        results = pool.imap_unordered(
-            functools.partial(self._ExecuteOne, nb), all_projects,
-            chunksize=WORKER_BATCH_SIZE)
-        _ProcessResults(results)
-    pm.end()
+    self.ExecuteInParallel(
+        opt.jobs,
+        functools.partial(self._ExecuteOne, nb),
+        all_projects,
+        callback=_ProcessResults,
+        output=Progress('Checkout %s' % (nb,), len(all_projects), quiet=opt.quiet))

    if err:
      for p in err:
--- a/subcmds/cherry_pick.py
+++ b/subcmds/cherry_pick.py
@ -32,9 +32,6 @@ The change id will be updated, and a reference to the old
 change id will be added.
 """

-  def _Options(self, p):
-    pass
-
  def ValidateOptions(self, opt, args):
    if len(args) != 1:
      self.Usage()
--- a/subcmds/diff.py
+++ b/subcmds/diff.py
@ -14,9 +14,8 @@

 import functools
 import io
-import multiprocessing

-from command import DEFAULT_LOCAL_JOBS, PagedCommand, WORKER_BATCH_SIZE
+from command import DEFAULT_LOCAL_JOBS, PagedCommand


 class Diff(PagedCommand):
@ -32,12 +31,11 @@ to the Unix 'patch' command.
  PARALLEL_JOBS = DEFAULT_LOCAL_JOBS

  def _Options(self, p):
-    super()._Options(p)
    p.add_option('-u', '--absolute',
                 dest='absolute', action='store_true',
                 help='Paths are relative to the repository root')

-  def _DiffHelper(self, absolute, project):
+  def _ExecuteOne(self, absolute, project):
    """Obtains the diff for a specific project.

    Args:
@ -52,22 +50,20 @@ to the Unix 'patch' command.
    return (ret, buf.getvalue())

  def Execute(self, opt, args):
-    ret = 0
    all_projects = self.GetProjects(args)

-    # NB: Multiprocessing is heavy, so don't spin it up for one job.
-    if len(all_projects) == 1 or opt.jobs == 1:
-      for project in all_projects:
-        if not project.PrintWorkTreeDiff(opt.absolute):
+    def _ProcessResults(_pool, _output, results):
+      ret = 0
+      for (state, output) in results:
+        if output:
+          print(output, end='')
+        if not state:
          ret = 1
-    else:
-      with multiprocessing.Pool(opt.jobs) as pool:
-        states = pool.imap(functools.partial(self._DiffHelper, opt.absolute),
-                           all_projects, WORKER_BATCH_SIZE)
-        for (state, output) in states:
-          if output:
-            print(output, end='')
-          if not state:
-            ret = 1
+      return ret

-    return ret
+    return self.ExecuteInParallel(
+        opt.jobs,
+        functools.partial(self._ExecuteOne, opt.absolute),
+        all_projects,
+        callback=_ProcessResults,
+        ordered=True)
--- a/subcmds/forall.py
+++ b/subcmds/forall.py
@ -24,6 +24,7 @@ import subprocess

 from color import Coloring
 from command import DEFAULT_LOCAL_JOBS, Command, MirrorSafeCommand, WORKER_BATCH_SIZE
+from error import ManifestInvalidRevisionError

 _CAN_COLOR = [
    'branch',
@ -44,7 +45,7 @@ class Forall(Command, MirrorSafeCommand):
  helpSummary = "Run a shell command in each project"
  helpUsage = """
 %prog [<project>...] -c <command> [<arg>...]
-%prog -r str1 [str2] ... -c <command> [<arg>...]"
+%prog -r str1 [str2] ... -c <command> [<arg>...]
 """
  helpDescription = """
 Executes the same shell command in each project.
@ -128,8 +129,6 @@ without iterating through the remaining projects.
      del parser.rargs[0]

  def _Options(self, p):
-    super()._Options(p)
-
    p.add_option('-r', '--regex',
                 dest='regex', action='store_true',
                 help="Execute the command only on projects matching regex or wildcard expression")
@ -152,13 +151,10 @@ without iterating through the remaining projects.
                 help='Silently skip & do not exit non-zero due missing '
                      'checkouts')

-    g = p.add_option_group('Output')
+    g = p.get_option_group('--quiet')
    g.add_option('-p',
                 dest='project_header', action='store_true',
                 help='Show project headers before output')
-    g.add_option('-v', '--verbose',
-                 dest='verbose', action='store_true',
-                 help='Show command error messages')
    p.add_option('--interactive',
                 action='store_true',
                 help='force interactive usage')
@ -252,7 +248,7 @@ without iterating through the remaining projects.
      rc = rc or errno.EINTR
    except Exception as e:
      # Catch any other exceptions raised
-      print('Got an error, terminating the pool: %s: %s' %
+      print('forall: unhandled error, terminating the pool: %s: %s' %
            (type(e).__name__, e),
            file=sys.stderr)
      rc = rc or getattr(e, 'errno', 1)
@ -295,7 +291,13 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
  setenv('REPO_PROJECT', project.name)
  setenv('REPO_PATH', project.relpath)
  setenv('REPO_REMOTE', project.remote.name)
-  setenv('REPO_LREV', '' if mirror else project.GetRevisionId())
+  try:
+    # If we aren't in a fully synced state and we don't have the ref the manifest
+    # wants, then this will fail.  Ignore it for the purposes of this code.
+    lrev = '' if mirror else project.GetRevisionId()
+  except ManifestInvalidRevisionError:
+    lrev = ''
+  setenv('REPO_LREV', lrev)
  setenv('REPO_RREV', project.revisionExpr)
  setenv('REPO_UPSTREAM', project.upstream)
  setenv('REPO_DEST_BRANCH', project.dest_branch)
--- a/subcmds/gitc_init.py
+++ b/subcmds/gitc_init.py
@ -48,13 +48,6 @@ use for this GITC client.

  def _Options(self, p):
    super()._Options(p, gitc_init=True)
-    g = p.add_option_group('GITC options')
-    g.add_option('-f', '--manifest-file',
-                 dest='manifest_file',
-                 help='Optional manifest file to use for this GITC client.')
-    g.add_option('-c', '--gitc-client',
-                 dest='gitc_client',
-                 help='The name of the gitc_client instance to create or modify.')

  def Execute(self, opt, args):
    gitc_client = gitc_utils.parse_clientdir(os.getcwd())
--- a/subcmds/grep.py
+++ b/subcmds/grep.py
@ -12,10 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+import functools
 import sys

 from color import Coloring
-from command import PagedCommand
+from command import DEFAULT_LOCAL_JOBS, PagedCommand
 from error import GitError
 from git_command import GitCommand

@ -61,6 +62,7 @@ contain a line that matches both expressions:
  repo grep --all-match -e NODE -e Unexpected

 """
+  PARALLEL_JOBS = DEFAULT_LOCAL_JOBS

  @staticmethod
  def _carry_option(_option, opt_str, value, parser):
@ -79,6 +81,10 @@ contain a line that matches both expressions:
    if value is not None:
      pt.append(value)

+  def _CommonOptions(self, p):
+    """Override common options slightly."""
+    super()._CommonOptions(p, opt_v=False)
+
  def _Options(self, p):
    g = p.add_option_group('Sources')
    g.add_option('--cached',
@ -152,6 +158,72 @@ contain a line that matches both expressions:
                 action='callback', callback=self._carry_option,
                 help='Show only file names not containing matching lines')

+  def _ExecuteOne(self, cmd_argv, project):
+    """Process one project."""
+    try:
+      p = GitCommand(project,
+                     cmd_argv,
+                     bare=False,
+                     capture_stdout=True,
+                     capture_stderr=True)
+    except GitError as e:
+      return (project, -1, None, str(e))
+
+    return (project, p.Wait(), p.stdout, p.stderr)
+
+  @staticmethod
+  def _ProcessResults(full_name, have_rev, _pool, out, results):
+    git_failed = False
+    bad_rev = False
+    have_match = False
+
+    for project, rc, stdout, stderr in results:
+      if rc < 0:
+        git_failed = True
+        out.project('--- project %s ---' % project.relpath)
+        out.nl()
+        out.fail('%s', stderr)
+        out.nl()
+        continue
+
+      if rc:
+        # no results
+        if stderr:
+          if have_rev and 'fatal: ambiguous argument' in stderr:
+            bad_rev = True
+          else:
+            out.project('--- project %s ---' % project.relpath)
+            out.nl()
+            out.fail('%s', stderr.strip())
+            out.nl()
+        continue
+      have_match = True
+
+      # We cut the last element, to avoid a blank line.
+      r = stdout.split('\n')
+      r = r[0:-1]
+
+      if have_rev and full_name:
+        for line in r:
+          rev, line = line.split(':', 1)
+          out.write("%s", rev)
+          out.write(':')
+          out.project(project.relpath)
+          out.write('/')
+          out.write("%s", line)
+          out.nl()
+      elif full_name:
+        for line in r:
+          out.project(project.relpath)
+          out.write('/')
+          out.write("%s", line)
+          out.nl()
+      else:
+        for line in r:
+          print(line)
+
+    return (git_failed, bad_rev, have_match)
+
  def Execute(self, opt, args):
    out = GrepColoring(self.manifest.manifestProject.config)

@ -183,62 +255,13 @@ contain a line that matches both expressions:
      cmd_argv.extend(opt.revision)
    cmd_argv.append('--')

-    git_failed = False
-    bad_rev = False
-    have_match = False
-
-    for project in projects:
-      try:
-        p = GitCommand(project,
-                       cmd_argv,
-                       bare=False,
-                       capture_stdout=True,
-                       capture_stderr=True)
-      except GitError as e:
-        git_failed = True
-        out.project('--- project %s ---' % project.relpath)
-        out.nl()
-        out.fail('%s', str(e))
-        out.nl()
-        continue
-
-      if p.Wait() != 0:
-        # no results
-        #
-        if p.stderr:
-          if have_rev and 'fatal: ambiguous argument' in p.stderr:
-            bad_rev = True
-          else:
-            out.project('--- project %s ---' % project.relpath)
-            out.nl()
-            out.fail('%s', p.stderr.strip())
-            out.nl()
-        continue
-      have_match = True
-
-      # We cut the last element, to avoid a blank line.
-      #
-      r = p.stdout.split('\n')
-      r = r[0:-1]
-
-      if have_rev and full_name:
-        for line in r:
-          rev, line = line.split(':', 1)
-          out.write("%s", rev)
-          out.write(':')
-          out.project(project.relpath)
-          out.write('/')
-          out.write("%s", line)
-          out.nl()
-      elif full_name:
-        for line in r:
-          out.project(project.relpath)
-          out.write('/')
-          out.write("%s", line)
-          out.nl()
-      else:
-        for line in r:
-          print(line)
+    git_failed, bad_rev, have_match = self.ExecuteInParallel(
+        opt.jobs,
+        functools.partial(self._ExecuteOne, cmd_argv),
+        projects,
+        callback=functools.partial(self._ProcessResults, full_name, have_rev),
+        output=out,
+        ordered=True)

    if git_failed:
      sys.exit(1)
--- a/subcmds/help.py
+++ b/subcmds/help.py
@ -14,7 +14,7 @@

 import re
 import sys
-from formatter import AbstractFormatter, DumbWriter
+import textwrap

 from subcmds import all_commands
 from color import Coloring
@ -84,8 +84,7 @@ Displays detailed usage information about a command.
      def __init__(self, gc):
        Coloring.__init__(self, gc, 'help')
        self.heading = self.printer('heading', attr='bold')
-
-        self.wrap = AbstractFormatter(DumbWriter())
+        self._first = True

      def _PrintSection(self, heading, bodyAttr):
        try:
@ -95,7 +94,9 @@ Displays detailed usage information about a command.
        if body == '' or body is None:
          return

-        self.nl()
+        if not self._first:
+          self.nl()
+        self._first = False

        self.heading('%s%s', header_prefix, heading)
        self.nl()
@ -105,7 +106,8 @@ Displays detailed usage information about a command.
        body = body.strip()
        body = body.replace('%prog', me)

-        asciidoc_hdr = re.compile(r'^\n?#+ (.+)$')
+        # Extract the title, but skip any trailing {#anchors}.
+        asciidoc_hdr = re.compile(r'^\n?#+ ([^{]+)(\{#.+\})?$')
        for para in body.split("\n\n"):
          if para.startswith(' '):
            self.write('%s', para)
@ -120,9 +122,12 @@ Displays detailed usage information about a command.
            self.nl()
            continue

-          self.wrap.add_flowing_data(para)
-          self.wrap.end_paragraph(1)
-        self.wrap.end_paragraph(0)
+          lines = textwrap.wrap(para.replace('  ', ' '), width=80,
+                                break_long_words=False, break_on_hyphens=False)
+          for line in lines:
+            self.write('%s', line)
+            self.nl()
+          self.nl()

    out = _Out(self.client.globalConfig)
    out._PrintSection('Summary', 'helpSummary')
--- a/subcmds/init.py
+++ b/subcmds/init.py
@ -79,105 +79,11 @@ manifest, a subsequent `repo sync` (or `repo sync -d`) is necessary
 to update the working directory files.
 """

+  def _CommonOptions(self, p):
+    """Disable due to re-use of Wrapper()."""
+
  def _Options(self, p, gitc_init=False):
-    # Logging
-    g = p.add_option_group('Logging options')
-    g.add_option('-v', '--verbose',
-                 dest='output_mode', action='store_true',
-                 help='show all output')
-    g.add_option('-q', '--quiet',
-                 dest='output_mode', action='store_false',
-                 help='only show errors')
-
-    # Manifest
-    g = p.add_option_group('Manifest options')
-    g.add_option('-u', '--manifest-url',
-                 dest='manifest_url',
-                 help='manifest repository location', metavar='URL')
-    g.add_option('-b', '--manifest-branch', metavar='REVISION',
-                 help='manifest branch or revision (use HEAD for default)')
-    cbr_opts = ['--current-branch']
-    # The gitc-init subcommand allocates -c itself, but a lot of init users
-    # want -c, so try to satisfy both as best we can.
-    if not gitc_init:
-      cbr_opts += ['-c']
-    g.add_option(*cbr_opts,
-                 dest='current_branch_only', action='store_true',
-                 help='fetch only current manifest branch from server')
-    g.add_option('-m', '--manifest-name',
-                 dest='manifest_name', default='default.xml',
-                 help='initial manifest file', metavar='NAME.xml')
-    g.add_option('--mirror',
-                 dest='mirror', action='store_true',
-                 help='create a replica of the remote repositories '
-                      'rather than a client working directory')
-    g.add_option('--reference',
-                 dest='reference',
-                 help='location of mirror directory', metavar='DIR')
-    g.add_option('--dissociate',
-                 dest='dissociate', action='store_true',
-                 help='dissociate from reference mirrors after clone')
-    g.add_option('--depth', type='int', default=None,
-                 dest='depth',
-                 help='create a shallow clone with given depth; see git clone')
-    g.add_option('--partial-clone', action='store_true',
-                 dest='partial_clone',
-                 help='perform partial clone (https://git-scm.com/'
-                 'docs/gitrepository-layout#_code_partialclone_code)')
-    g.add_option('--clone-filter', action='store', default='blob:none',
-                 dest='clone_filter',
-                 help='filter for use with --partial-clone [default: %default]')
-    g.add_option('--worktree', action='store_true',
-                 help='use git-worktree to manage projects')
-    g.add_option('--archive',
-                 dest='archive', action='store_true',
-                 help='checkout an archive instead of a git repository for '
-                      'each project. See git archive.')
-    g.add_option('--submodules',
-                 dest='submodules', action='store_true',
-                 help='sync any submodules associated with the manifest repo')
-    g.add_option('--use-superproject', action='store_true',
-                 help='use the manifest superproject to sync projects')
-    g.add_option('--no-use-superproject', action='store_false',
-                 dest='use_superproject',
-                 help='disable use of manifest superprojects')
-    g.add_option('-g', '--groups',
-                 dest='groups', default='default',
-                 help='restrict manifest projects to ones with specified '
-                      'group(s) [default|all|G1,G2,G3|G4,-G5,-G6]',
-                 metavar='GROUP')
-    g.add_option('-p', '--platform',
-                 dest='platform', default='auto',
-                 help='restrict manifest projects to ones with a specified '
-                      'platform group [auto|all|none|linux|darwin|...]',
-                 metavar='PLATFORM')
-    g.add_option('--clone-bundle', action='store_true',
-                 help='force use of /clone.bundle on HTTP/HTTPS (default if not --partial-clone)')
-    g.add_option('--no-clone-bundle',
-                 dest='clone_bundle', action='store_false',
-                 help='disable use of /clone.bundle on HTTP/HTTPS (default if --partial-clone)')
-    g.add_option('--no-tags',
-                 dest='tags', default=True, action='store_false',
-                 help="don't fetch tags in the manifest")
-
-    # Tool
-    g = p.add_option_group('repo Version options')
-    g.add_option('--repo-url',
-                 dest='repo_url',
-                 help='repo repository location', metavar='URL')
-    g.add_option('--repo-rev', metavar='REV',
-                 help='repo branch or revision')
-    g.add_option('--repo-branch', dest='repo_rev',
-                 help=optparse.SUPPRESS_HELP)
-    g.add_option('--no-repo-verify',
-                 dest='repo_verify', default=True, action='store_false',
-                 help='do not verify repo source code')
-
-    # Other
-    g = p.add_option_group('Other options')
-    g.add_option('--config-name',
-                 dest='config_name', action="store_true", default=False,
-                 help='Always prompt for name/e-mail')
+    Wrapper().InitParser(p, gitc_init=gitc_init)

  def _RegisteredEnvironmentOptions(self):
    return {'REPO_MANIFEST_URL': 'manifest_url',
@ -311,7 +217,7 @@ to update the working directory files.
              'in another location.', file=sys.stderr)
        sys.exit(1)

-    if opt.partial_clone:
+    if opt.partial_clone is not None:
      if opt.mirror:
        print('fatal: --mirror and --partial-clone are mutually exclusive',
              file=sys.stderr)
@ -319,9 +225,14 @@ to update the working directory files.
      m.config.SetBoolean('repo.partialclone', opt.partial_clone)
      if opt.clone_filter:
        m.config.SetString('repo.clonefilter', opt.clone_filter)
+    elif m.config.GetBoolean('repo.partialclone'):
+      opt.clone_filter = m.config.GetString('repo.clonefilter')
    else:
      opt.clone_filter = None

+    if opt.partial_clone_exclude is not None:
+      m.config.SetString('repo.partialcloneexclude', opt.partial_clone_exclude)
+
    if opt.clone_bundle is None:
      opt.clone_bundle = False if opt.partial_clone else True
    else:
@ -337,7 +248,8 @@ to update the working directory files.
                              clone_bundle=opt.clone_bundle,
                              current_branch_only=opt.current_branch_only,
                              tags=opt.tags, submodules=opt.submodules,
-                              clone_filter=opt.clone_filter):
+                              clone_filter=opt.clone_filter,
+                              partial_clone_exclude=self.manifest.PartialCloneExclude):
      r = m.GetRemote(m.remote.name)
      print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)

@ -527,9 +439,6 @@ to update the working directory files.
            % ('.'.join(str(x) for x in MIN_GIT_VERSION_SOFT),),
            file=sys.stderr)

-    opt.quiet = opt.output_mode is False
-    opt.verbose = opt.output_mode is True
-
    rp = self.manifest.repoProject

    # Handle new --repo-url requests.
--- a/subcmds/list.py
+++ b/subcmds/list.py
@ -20,11 +20,16 @@ class List(Command, MirrorSafeCommand):
  helpSummary = "List projects and their associated directories"
  helpUsage = """
 %prog [-f] [<project>...]
-%prog [-f] -r str1 [str2]..."
+%prog [-f] -r str1 [str2]...
 """
  helpDescription = """
 List all projects; pass '.' to list the project for the cwd.

+By default, only projects that currently exist in the checkout are shown.  If
+you want to list all projects (using the specified filter settings), use the
+--all option.  If you want to show all projects regardless of the manifest
+groups, then also pass --groups all.
+
 This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
 """

@ -35,6 +40,9 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
    p.add_option('-g', '--groups',
                 dest='groups',
                 help="Filter the project list based on the groups the project is in")
+    p.add_option('-a', '--all',
+                 action='store_true',
+                 help='Show projects regardless of checkout state')
    p.add_option('-f', '--fullpath',
                 dest='fullpath', action='store_true',
                 help="Display the full work tree path instead of the relative path")
@ -61,7 +69,7 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
      args: Positional args.  Can be a list of projects to list, or empty.
    """
    if not opt.regex:
-      projects = self.GetProjects(args, groups=opt.groups)
+      projects = self.GetProjects(args, groups=opt.groups, missing_ok=opt.all)
    else:
      projects = self.FindProjects(args)

@ -79,5 +87,6 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
      else:
        lines.append("%s : %s" % (_getpath(project), project.name))

-    lines.sort()
-    print('\n'.join(lines))
+    if lines:
+      lines.sort()
+      print('\n'.join(lines))
--- a/subcmds/prune.py
+++ b/subcmds/prune.py
@ -12,8 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+import itertools
+
 from color import Coloring
-from command import PagedCommand
+from command import DEFAULT_LOCAL_JOBS, PagedCommand


 class Prune(PagedCommand):
@ -22,11 +24,26 @@ class Prune(PagedCommand):
  helpUsage = """
 %prog [<project>...]
 """
+  PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
+
+  def _ExecuteOne(self, project):
+    """Process one project."""
+    return project.PruneHeads()

  def Execute(self, opt, args):
-    all_branches = []
-    for project in self.GetProjects(args):
-      all_branches.extend(project.PruneHeads())
+    projects = self.GetProjects(args)
+
+    # NB: Should be able to refactor this module to display summary as results
+    # come back from children.
+    def _ProcessResults(_pool, _output, results):
+      return list(itertools.chain.from_iterable(results))
+
+    all_branches = self.ExecuteInParallel(
+        opt.jobs,
+        self._ExecuteOne,
+        projects,
+        callback=_ProcessResults,
+        ordered=True)

    if not all_branches:
      return
--- a/subcmds/rebase.py
+++ b/subcmds/rebase.py
@ -39,7 +39,8 @@ branch but need to incorporate new upstream changes "underneath" them.
 """

  def _Options(self, p):
-    p.add_option('-i', '--interactive',
+    g = p.get_option_group('--quiet')
+    g.add_option('-i', '--interactive',
                 dest="interactive", action="store_true",
                 help="interactive rebase (single project only)")

@ -52,9 +53,6 @@ branch but need to incorporate new upstream changes "underneath" them.
    p.add_option('--no-ff',
                 dest='ff', default=True, action='store_false',
                 help='Pass --no-ff to git rebase')
-    p.add_option('-q', '--quiet',
-                 dest='quiet', action='store_true',
-                 help='Pass --quiet to git rebase')
    p.add_option('--autosquash',
                 dest='autosquash', action='store_true',
                 help='Pass --autosquash to git rebase')
--- a/subcmds/stage.py
+++ b/subcmds/stage.py
@ -38,7 +38,8 @@ The '%prog' command stages files to prepare the next commit.
 """

  def _Options(self, p):
-    p.add_option('-i', '--interactive',
+    g = p.get_option_group('--quiet')
+    g.add_option('-i', '--interactive',
                 dest='interactive', action='store_true',
                 help='use interactive staging')

--- a/subcmds/start.py
+++ b/subcmds/start.py
@ -13,11 +13,10 @@
 # limitations under the License.

 import functools
-import multiprocessing
 import os
 import sys

-from command import Command, DEFAULT_LOCAL_JOBS, WORKER_BATCH_SIZE
+from command import Command, DEFAULT_LOCAL_JOBS
 from git_config import IsImmutable
 from git_command import git
 import gitc_utils
@ -38,13 +37,13 @@ revision specified in the manifest.
  PARALLEL_JOBS = DEFAULT_LOCAL_JOBS

  def _Options(self, p):
-    super()._Options(p)
    p.add_option('--all',
                 dest='all', action='store_true',
                 help='begin branch in all projects')
    p.add_option('-r', '--rev', '--revision', dest='revision',
                 help='point branch at this revision instead of upstream')
-    p.add_option('--head', dest='revision', action='store_const', const='HEAD',
+    p.add_option('--head', '--HEAD',
+                 dest='revision', action='store_const', const='HEAD',
                 help='abbreviation for --rev HEAD')

  def ValidateOptions(self, opt, args):
@ -55,7 +54,7 @@ revision specified in the manifest.
    if not git.check_ref_format('heads/%s' % nb):
      self.OptionParser.error("'%s' is not a valid name" % nb)

-  def _ExecuteOne(self, opt, nb, project):
+  def _ExecuteOne(self, revision, nb, project):
    """Start one project."""
    # If the current revision is immutable, such as a SHA1, a tag or
    # a change, then we can't push back to it. Substitute with
@ -69,7 +68,7 @@ revision specified in the manifest.

    try:
      ret = project.StartBranch(
-          nb, branch_merge=branch_merge, revision=opt.revision)
+          nb, branch_merge=branch_merge, revision=revision)
    except Exception as e:
      print('error: unable to checkout %s: %s' % (project.name, e), file=sys.stderr)
      ret = False
@ -106,7 +105,7 @@ revision specified in the manifest.
      if not os.path.exists(os.getcwd()):
        os.chdir(self.manifest.topdir)

-      pm = Progress('Syncing %s' % nb, len(all_projects))
+      pm = Progress('Syncing %s' % nb, len(all_projects), quiet=opt.quiet)
      for project in all_projects:
        gitc_project = self.gitc_manifest.paths[project.relpath]
        # Sync projects that have not been opened.
@ -123,23 +122,18 @@ revision specified in the manifest.
        pm.update()
      pm.end()

-    def _ProcessResults(results):
+    def _ProcessResults(_pool, pm, results):
      for (result, project) in results:
        if not result:
          err.append(project)
        pm.update()

-    pm = Progress('Starting %s' % nb, len(all_projects))
-    # NB: Multiprocessing is heavy, so don't spin it up for one job.
-    if len(all_projects) == 1 or opt.jobs == 1:
-      _ProcessResults(self._ExecuteOne(opt, nb, x) for x in all_projects)
-    else:
-      with multiprocessing.Pool(opt.jobs) as pool:
-        results = pool.imap_unordered(
-            functools.partial(self._ExecuteOne, opt, nb), all_projects,
-            chunksize=WORKER_BATCH_SIZE)
-        _ProcessResults(results)
-    pm.end()
+    self.ExecuteInParallel(
+        opt.jobs,
+        functools.partial(self._ExecuteOne, opt.revision, nb),
+        all_projects,
+        callback=_ProcessResults,
+        output=Progress('Starting %s' % (nb,), len(all_projects), quiet=opt.quiet))

    if err:
      for p in err:
--- a/subcmds/status.py
+++ b/subcmds/status.py
@ -15,10 +15,9 @@
 import functools
 import glob
 import io
-import multiprocessing
 import os

-from command import DEFAULT_LOCAL_JOBS, PagedCommand, WORKER_BATCH_SIZE
+from command import DEFAULT_LOCAL_JOBS, PagedCommand

 from color import Coloring
 import platform_utils
@ -80,12 +79,9 @@ the following meanings:
  PARALLEL_JOBS = DEFAULT_LOCAL_JOBS

  def _Options(self, p):
-    super()._Options(p)
    p.add_option('-o', '--orphans',
                 dest='orphans', action='store_true',
                 help="include objects in working directory outside of repo projects")
-    p.add_option('-q', '--quiet', action='store_true',
-                 help="only print the name of modified projects")

  def _StatusHelper(self, quiet, project):
    """Obtains the status for a specific project.
@ -122,22 +118,23 @@ the following meanings:

  def Execute(self, opt, args):
    all_projects = self.GetProjects(args)
-    counter = 0

-    if opt.jobs == 1:
-      for project in all_projects:
-        state = project.PrintWorkTreeStatus(quiet=opt.quiet)
+    def _ProcessResults(_pool, _output, results):
+      ret = 0
+      for (state, output) in results:
+        if output:
+          print(output, end='')
        if state == 'CLEAN':
-          counter += 1
-    else:
-      with multiprocessing.Pool(opt.jobs) as pool:
-        states = pool.imap(functools.partial(self._StatusHelper, opt.quiet),
-                           all_projects, chunksize=WORKER_BATCH_SIZE)
-        for (state, output) in states:
-          if output:
-            print(output, end='')
-          if state == 'CLEAN':
-            counter += 1
+          ret += 1
+      return ret
+
+    counter = self.ExecuteInParallel(
+        opt.jobs,
+        functools.partial(self._StatusHelper, opt.quiet),
+        all_projects,
+        callback=_ProcessResults,
+        ordered=True)
+
    if not opt.quiet and len(all_projects) == counter:
      print('nothing to commit (working directory clean)')

--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@ -20,9 +20,7 @@ import multiprocessing
 import netrc
 from optparse import SUPPRESS_HELP
 import os
-import re
 import socket
-import subprocess
 import sys
 import tempfile
 import time
@ -45,13 +43,8 @@ except ImportError:
  def _rlimit_nofile():
    return (256, 256)

-try:
-  import multiprocessing
-except ImportError:
-  multiprocessing = None
-
 import event_log
-from git_command import GIT, git_require
+from git_command import git_require
 from git_config import GetUrlCookieFile
 from git_refs import R_HEADS, HEAD
 import git_superproject
@ -69,10 +62,6 @@ from manifest_xml import GitcManifest
 _ONE_DAY_S = 24 * 60 * 60


-class _FetchError(Exception):
-  """Internal error thrown in _FetchHelper() when we don't want stack trace."""
-
-
 class Sync(Command, MirrorSafeCommand):
  jobs = 1
  common = True
@ -178,12 +167,18 @@ later is required to fix a server side protocol bug.
 """
  PARALLEL_JOBS = 1

-  def _Options(self, p, show_smart=True):
+  def _CommonOptions(self, p):
    try:
      self.PARALLEL_JOBS = self.manifest.default.sync_j
    except ManifestParseError:
      pass
-    super()._Options(p)
+    super()._CommonOptions(p)
+
+  def _Options(self, p, show_smart=True):
+    p.add_option('--jobs-network', default=None, type=int, metavar='JOBS',
+                 help='number of network jobs to run in parallel (defaults to --jobs)')
+    p.add_option('--jobs-checkout', default=None, type=int, metavar='JOBS',
+                 help='number of local checkout jobs to run in parallel (defaults to --jobs)')

    p.add_option('-f', '--force-broken',
                 dest='force_broken', action='store_true',
@ -217,12 +212,6 @@ later is required to fix a server side protocol bug.
    p.add_option('-c', '--current-branch',
                 dest='current_branch_only', action='store_true',
                 help='fetch only current branch from server')
-    p.add_option('-v', '--verbose',
-                 dest='output_mode', action='store_true',
-                 help='show all sync output')
-    p.add_option('-q', '--quiet',
-                 dest='output_mode', action='store_false',
-                 help='only show errors')
    p.add_option('-m', '--manifest-name',
                 dest='manifest_name',
                 help='temporary manifest to use for this sync', metavar='NAME.xml')
@ -277,6 +266,16 @@ later is required to fix a server side protocol bug.
      branch = branch[len(R_HEADS):]
    return branch

+  def _UseSuperproject(self, opt):
+    """Returns True if use-superproject option is enabled"""
+    return (opt.use_superproject or
+            self.manifest.manifestProject.config.GetBoolean(
+                'repo.superproject'))
+
+  def _GetCurrentBranchOnly(self, opt):
+    """Returns True if current-branch or use-superproject options are enabled."""
+    return opt.current_branch_only or self._UseSuperproject(opt)
+
  def _UpdateProjectsRevisionId(self, opt, args):
    """Update revisionId of every project with the SHA from superproject.

@ -305,148 +304,123 @@ later is required to fix a server side protocol bug.
    self._ReloadManifest(manifest_path)
    return manifest_path

-  def _FetchProjectList(self, opt, projects, sem, *args, **kwargs):
-    """Main function of the fetch threads.
+  def _FetchProjectList(self, opt, projects):
+    """Main function of the fetch worker.
+
+    The projects we're given share the same underlying git object store, so we
+    have to fetch them in serial.

    Delegates most of the work to _FetchHelper.

    Args:
      opt: Program options returned from optparse.  See _Options().
      projects: Projects to fetch.
-      sem: We'll release() this semaphore when we exit so that another thread
-          can be started up.
-      *args, **kwargs: Remaining arguments to pass to _FetchHelper. See the
-          _FetchHelper docstring for details.
    """
-    try:
-        for project in projects:
-          success = self._FetchHelper(opt, project, *args, **kwargs)
-          if not success and opt.fail_fast:
-            break
-    finally:
-        sem.release()
+    return [self._FetchOne(opt, x) for x in projects]

-  def _FetchHelper(self, opt, project, lock, fetched, pm, err_event,
-                   clone_filter):
+  def _FetchOne(self, opt, project):
    """Fetch git objects for a single project.

    Args:
      opt: Program options returned from optparse.  See _Options().
      project: Project object for the project to fetch.
-      lock: Lock for accessing objects that are shared amongst multiple
-          _FetchHelper() threads.
-      fetched: set object that we will add project.gitdir to when we're done
-          (with our lock held).
-      pm: Instance of a Project object.  We will call pm.update() (with our
-          lock held).
-      err_event: We'll set this event in the case of an error (after printing
-          out info about the error).
-      clone_filter: Filter for use in a partial clone.

    Returns:
      Whether the fetch was successful.
    """
-    # We'll set to true once we've locked the lock.
-    did_lock = False
-
-    # Encapsulate everything in a try/except/finally so that:
-    # - We always set err_event in the case of an exception.
-    # - We always make sure we unlock the lock if we locked it.
    start = time.time()
    success = False
    buf = io.StringIO()
-    with lock:
-      pm.start(project.name)
    try:
-      try:
-        success = project.Sync_NetworkHalf(
-            quiet=opt.quiet,
-            verbose=opt.verbose,
-            output_redir=buf,
-            current_branch_only=opt.current_branch_only,
-            force_sync=opt.force_sync,
-            clone_bundle=opt.clone_bundle,
-            tags=opt.tags, archive=self.manifest.IsArchive,
-            optimized_fetch=opt.optimized_fetch,
-            retry_fetches=opt.retry_fetches,
-            prune=opt.prune,
-            clone_filter=clone_filter)
-        self._fetch_times.Set(project, time.time() - start)
+      success = project.Sync_NetworkHalf(
+          quiet=opt.quiet,
+          verbose=opt.verbose,
+          output_redir=buf,
+          current_branch_only=self._GetCurrentBranchOnly(opt),
+          force_sync=opt.force_sync,
+          clone_bundle=opt.clone_bundle,
+          tags=opt.tags, archive=self.manifest.IsArchive,
+          optimized_fetch=opt.optimized_fetch,
+          retry_fetches=opt.retry_fetches,
+          prune=opt.prune,
+          clone_filter=self.manifest.CloneFilter,
+          partial_clone_exclude=self.manifest.PartialCloneExclude)

-        # Lock around all the rest of the code, since printing, updating a set
-        # and Progress.update() are not thread safe.
-        lock.acquire()
-        did_lock = True
+      output = buf.getvalue()
+      if opt.verbose and output:
+        print('\n' + output.rstrip())

-        output = buf.getvalue()
-        if opt.verbose and output:
-          pm.update(inc=0, msg=output.rstrip())
+      if not success:
+        print('error: Cannot fetch %s from %s'
+              % (project.name, project.remote.url),
+              file=sys.stderr)
+    except GitError as e:
+      print('error.GitError: Cannot fetch %s' % str(e), file=sys.stderr)
+    except Exception as e:
+      print('error: Cannot fetch %s (%s: %s)'
+            % (project.name, type(e).__name__, str(e)), file=sys.stderr)
+      raise

-        if not success:
-          err_event.set()
-          print('error: Cannot fetch %s from %s'
-                % (project.name, project.remote.url),
-                file=sys.stderr)
-          if opt.fail_fast:
-            raise _FetchError()
-
-        fetched.add(project.gitdir)
-      except _FetchError:
-        pass
-      except Exception as e:
-        print('error: Cannot fetch %s (%s: %s)'
-              % (project.name, type(e).__name__, str(e)), file=sys.stderr)
-        err_event.set()
-        raise
-    finally:
-      if not did_lock:
-        lock.acquire()
-      pm.finish(project.name)
-      lock.release()
-      finish = time.time()
-      self.event_log.AddSync(project, event_log.TASK_SYNC_NETWORK,
-                             start, finish, success)
-
-    return success
+    finish = time.time()
+    return (success, project, start, finish)

  def _Fetch(self, projects, opt, err_event):
+    ret = True
+
+    jobs = opt.jobs_network if opt.jobs_network else self.jobs
    fetched = set()
-    lock = _threading.Lock()
-    pm = Progress('Fetching', len(projects))
+    pm = Progress('Fetching', len(projects), delay=False, quiet=opt.quiet)

    objdir_project_map = dict()
    for project in projects:
      objdir_project_map.setdefault(project.objdir, []).append(project)
+    projects_list = list(objdir_project_map.values())

-    threads = set()
-    sem = _threading.Semaphore(self.jobs)
-    for project_list in objdir_project_map.values():
-      # Check for any errors before running any more tasks.
-      # ...we'll let existing threads finish, though.
-      if err_event.is_set() and opt.fail_fast:
-        break
+    def _ProcessResults(results_sets):
+      ret = True
+      for results in results_sets:
+        for (success, project, start, finish) in results:
+          self._fetch_times.Set(project, finish - start)
+          self.event_log.AddSync(project, event_log.TASK_SYNC_NETWORK,
+                                 start, finish, success)
+          # Check for any errors before running any more tasks.
+          # ...we'll let existing jobs finish, though.
+          if not success:
+            ret = False
+          else:
+            fetched.add(project.gitdir)
+          pm.update(msg=project.name)
+        if not ret and opt.fail_fast:
+          break
+      return ret

-      sem.acquire()
-      kwargs = dict(opt=opt,
-                    projects=project_list,
-                    sem=sem,
-                    lock=lock,
-                    fetched=fetched,
-                    pm=pm,
-                    err_event=err_event,
-                    clone_filter=self.manifest.CloneFilter)
-      if self.jobs > 1:
-        t = _threading.Thread(target=self._FetchProjectList,
-                              kwargs=kwargs)
-        # Ensure that Ctrl-C will not freeze the repo process.
-        t.daemon = True
-        threads.add(t)
-        t.start()
+    # NB: Multiprocessing is heavy, so don't spin it up for one job.
+    if len(projects_list) == 1 or jobs == 1:
+      if not _ProcessResults(self._FetchProjectList(opt, x) for x in projects_list):
+        ret = False
+    else:
+      # Favor throughput over responsiveness when quiet.  It seems that imap()
+      # will yield results in batches relative to chunksize, so even as the
+      # children finish a sync, we won't see the result until one child finishes
+      # ~chunksize jobs.  When using a large --jobs with large chunksize, this
+      # can be jarring as there will be a large initial delay where repo looks
+      # like it isn't doing anything and sits at 0%, but then suddenly completes
+      # a lot of jobs all at once.  Since this code is more network bound, we
+      # can accept a bit more CPU overhead with a smaller chunksize so that the
+      # user sees more immediate & continuous feedback.
+      if opt.quiet:
+        chunksize = WORKER_BATCH_SIZE
      else:
-        self._FetchProjectList(**kwargs)
-
-    for t in threads:
-      t.join()
+        pm.update(inc=0, msg='warming up')
+        chunksize = 4
+      with multiprocessing.Pool(jobs) as pool:
+        results = pool.imap_unordered(
+            functools.partial(self._FetchProjectList, opt),
+            projects_list,
+            chunksize=chunksize)
+        if not _ProcessResults(results):
+          ret = False
+          pool.close()

    pm.end()
    self._fetch_times.Save()
@ -454,13 +428,14 @@ later is required to fix a server side protocol bug.
    if not self.manifest.IsArchive:
      self._GCProjects(projects, opt, err_event)

-    return fetched
+    return (ret, fetched)

-  def _CheckoutOne(self, opt, project):
+  def _CheckoutOne(self, detach_head, force_sync, project):
    """Checkout work tree for one project

    Args:
-      opt: Program options returned from optparse.  See _Options().
+      detach_head: Whether to leave a detached HEAD.
+      force_sync: Force checking out of the repo.
      project: Project object for the project to checkout.

    Returns:
@ -468,11 +443,14 @@ later is required to fix a server side protocol bug.
    """
    start = time.time()
    syncbuf = SyncBuffer(self.manifest.manifestProject.config,
-                         detach_head=opt.detach_head)
+                         detach_head=detach_head)
    success = False
    try:
-      project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
+      project.Sync_LocalHalf(syncbuf, force_sync=force_sync)
      success = syncbuf.Finish()
+    except GitError as e:
+      print('error.GitError: Cannot checkout %s: %s' %
+            (project.name, str(e)), file=sys.stderr)
    except Exception as e:
      print('error: Cannot checkout %s: %s: %s' %
            (project.name, type(e).__name__, str(e)),
@ -492,73 +470,66 @@ later is required to fix a server side protocol bug.
      opt: Program options returned from optparse.  See _Options().
      err_results: A list of strings, paths to git repos where checkout failed.
    """
-    ret = True
-
    # Only checkout projects with worktrees.
    all_projects = [x for x in all_projects if x.worktree]

-    pm = Progress('Checking out', len(all_projects))
-
-    def _ProcessResults(results):
+    def _ProcessResults(pool, pm, results):
+      ret = True
      for (success, project, start, finish) in results:
        self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL,
                               start, finish, success)
        # Check for any errors before running any more tasks.
-        # ...we'll let existing threads finish, though.
+        # ...we'll let existing jobs finish, though.
        if not success:
+          ret = False
          err_results.append(project.relpath)
          if opt.fail_fast:
-            return False
+            if pool:
+              pool.close()
+            return ret
        pm.update(msg=project.name)
-      return True
+      return ret

-    # NB: Multiprocessing is heavy, so don't spin it up for one job.
-    if len(all_projects) == 1 or opt.jobs == 1:
-      if not _ProcessResults(self._CheckoutOne(opt, x) for x in all_projects):
-        ret = False
-    else:
-      with multiprocessing.Pool(opt.jobs) as pool:
-        results = pool.imap_unordered(
-            functools.partial(self._CheckoutOne, opt),
-            all_projects,
-            chunksize=WORKER_BATCH_SIZE)
-        if not _ProcessResults(results):
-          ret = False
-          pool.close()
-
-    pm.end()
-
-    return ret and not err_results
+    return self.ExecuteInParallel(
+        opt.jobs_checkout if opt.jobs_checkout else self.jobs,
+        functools.partial(self._CheckoutOne, opt.detach_head, opt.force_sync),
+        all_projects,
+        callback=_ProcessResults,
+        output=Progress('Checking out', len(all_projects), quiet=opt.quiet)) and not err_results

  def _GCProjects(self, projects, opt, err_event):
+    pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet)
+    pm.update(inc=0, msg='prescan')
+
    gc_gitdirs = {}
    for project in projects:
      # Make sure pruning never kicks in with shared projects.
      if (not project.use_git_worktrees and
              len(project.manifest.GetProjectsWithName(project.name)) > 1):
        if not opt.quiet:
-          print('%s: Shared project %s found, disabling pruning.' %
+          print('\r%s: Shared project %s found, disabling pruning.' %
                (project.relpath, project.name))
        if git_require((2, 7, 0)):
          project.EnableRepositoryExtension('preciousObjects')
        else:
          # This isn't perfect, but it's the best we can do with old git.
-          print('%s: WARNING: shared projects are unreliable when using old '
+          print('\r%s: WARNING: shared projects are unreliable when using old '
                'versions of git; please upgrade to git-2.7.0+.'
                % (project.relpath,),
                file=sys.stderr)
          project.config.SetString('gc.pruneExpire', 'never')
      gc_gitdirs[project.gitdir] = project.bare_git

-    if multiprocessing:
-      cpu_count = multiprocessing.cpu_count()
-    else:
-      cpu_count = 1
+    pm.update(inc=len(projects) - len(gc_gitdirs), msg='warming up')
+
+    cpu_count = os.cpu_count()
    jobs = min(self.jobs, cpu_count)

    if jobs < 2:
      for bare_git in gc_gitdirs.values():
+        pm.update(msg=bare_git._project.name)
        bare_git.gc('--auto')
+      pm.end()
      return

    config = {'pack.threads': cpu_count // jobs if cpu_count > jobs else 1}
@ -567,6 +538,7 @@ later is required to fix a server side protocol bug.
    sem = _threading.Semaphore(jobs)

    def GC(bare_git):
+      pm.start(bare_git._project.name)
      try:
        try:
          bare_git.gc('--auto', config=config)
@ -576,6 +548,7 @@ later is required to fix a server side protocol bug.
          err_event.set()
          raise
      finally:
+        pm.finish(bare_git._project.name)
        sem.release()

    for bare_git in gc_gitdirs.values():
@ -589,6 +562,7 @@ later is required to fix a server side protocol bug.

    for t in threads:
      t.join()
+    pm.end()

  def _ReloadManifest(self, manifest_name=None):
    if manifest_name:
@ -735,13 +709,14 @@ later is required to fix a server side protocol bug.
    if not opt.local_only:
      start = time.time()
      success = mp.Sync_NetworkHalf(quiet=opt.quiet, verbose=opt.verbose,
-                                    current_branch_only=opt.current_branch_only,
+                                    current_branch_only=self._GetCurrentBranchOnly(opt),
                                    force_sync=opt.force_sync,
                                    tags=opt.tags,
                                    optimized_fetch=opt.optimized_fetch,
                                    retry_fetches=opt.retry_fetches,
                                    submodules=self.manifest.HasSubmodules,
-                                    clone_filter=self.manifest.CloneFilter)
+                                    clone_filter=self.manifest.CloneFilter,
+                                    partial_clone_exclude=self.manifest.PartialCloneExclude)
      finish = time.time()
      self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK,
                             start, finish, success)
@ -784,9 +759,6 @@ later is required to fix a server side protocol bug.
      soft_limit, _ = _rlimit_nofile()
      self.jobs = min(self.jobs, (soft_limit - 5) // 3)

-    opt.quiet = opt.output_mode is False
-    opt.verbose = opt.output_mode is True
-
    if opt.manifest_name:
      self.manifest.Override(opt.manifest_name)

@ -830,9 +802,7 @@ later is required to fix a server side protocol bug.
    else:
      self._UpdateManifestProject(opt, mp, manifest_name)

-    if (opt.use_superproject or
-        self.manifest.manifestProject.config.GetBoolean(
-            'repo.superproject')):
+    if self._UseSuperproject(opt):
      manifest_name = self._UpdateProjectsRevisionId(opt, args)

    if self.gitc_manifest:
@ -886,7 +856,9 @@ later is required to fix a server side protocol bug.
      to_fetch.extend(all_projects)
      to_fetch.sort(key=self._fetch_times.Get, reverse=True)

-      fetched = self._Fetch(to_fetch, opt, err_event)
+      success, fetched = self._Fetch(to_fetch, opt, err_event)
+      if not success:
+        err_event.set()

      _PostRepoFetch(rp, opt.repo_verify)
      if opt.network_only:
@ -915,7 +887,10 @@ later is required to fix a server side protocol bug.
        if previously_missing_set == missing_set:
          break
        previously_missing_set = missing_set
-        fetched.update(self._Fetch(missing, opt, err_event))
+        success, new_fetched = self._Fetch(to_fetch, opt, err_event)
+        if not success:
+          err_event.set()
+        fetched.update(new_fetched)

      # If we saw an error, exit with code 1 so that other scripts can check.
      if err_event.is_set():
@ -981,12 +956,25 @@ def _PostRepoUpgrade(manifest, quiet=False):
 def _PostRepoFetch(rp, repo_verify=True, verbose=False):
  if rp.HasChanges:
    print('info: A new version of repo is available', file=sys.stderr)
-    print(file=sys.stderr)
-    if not repo_verify or _VerifyTag(rp):
-      syncbuf = SyncBuffer(rp.config)
-      rp.Sync_LocalHalf(syncbuf)
-      if not syncbuf.Finish():
-        sys.exit(1)
+    wrapper = Wrapper()
+    try:
+      rev = rp.bare_git.describe(rp.GetRevisionId())
+    except GitError:
+      rev = None
+    _, new_rev = wrapper.check_repo_rev(rp.gitdir, rev, repo_verify=repo_verify)
+    # See if we're held back due to missing signed tag.
+    current_revid = rp.bare_git.rev_parse('HEAD')
+    new_revid = rp.bare_git.rev_parse('--verify', new_rev)
+    if current_revid != new_revid:
+      # We want to switch to the new rev, but also not trash any uncommitted
+      # changes.  This helps with local testing/hacking.
+      # If a local change has been made, we will throw that away.
+      # We also have to make sure this will switch to an older commit if that's
+      # the latest tag in order to support release rollback.
+      try:
+        rp.work_git.reset('--keep', new_rev)
+      except GitError as e:
+        sys.exit(str(e))
      print('info: Restarting repo with latest version', file=sys.stderr)
      raise RepoChangedException(['--repo-upgraded'])
    else:
@ -997,45 +985,6 @@ def _PostRepoFetch(rp, repo_verify=True, verbose=False):
            file=sys.stderr)


-def _VerifyTag(project):
-  gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
-  if not os.path.exists(gpg_dir):
-    print('warning: GnuPG was not available during last "repo init"\n'
-          'warning: Cannot automatically authenticate repo."""',
-          file=sys.stderr)
-    return True
-
-  try:
-    cur = project.bare_git.describe(project.GetRevisionId())
-  except GitError:
-    cur = None
-
-  if not cur \
-     or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
-    rev = project.revisionExpr
-    if rev.startswith(R_HEADS):
-      rev = rev[len(R_HEADS):]
-
-    print(file=sys.stderr)
-    print("warning: project '%s' branch '%s' is not signed"
-          % (project.name, rev), file=sys.stderr)
-    return False
-
-  env = os.environ.copy()
-  env['GIT_DIR'] = project.gitdir
-  env['GNUPGHOME'] = gpg_dir
-
-  cmd = [GIT, 'tag', '-v', cur]
-  result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
-                          env=env, check=False)
-  if result.returncode:
-    print(file=sys.stderr)
-    print(result.stdout, file=sys.stderr)
-    print(file=sys.stderr)
-    return False
-  return True
-
-
 class _FetchTimes(object):
  _ALPHA = 0.5

--- a/tests/test_project.py
+++ b/tests/test_project.py
@ -46,7 +46,7 @@ def TempGitTree():
      templatedir = tempfile.mkdtemp(prefix='.test-template')
      with open(os.path.join(templatedir, 'HEAD'), 'w') as fp:
        fp.write('ref: refs/heads/main\n')
-      cmd += ['--template=', templatedir]
+      cmd += ['--template', templatedir]
    subprocess.check_call(cmd, cwd=tempdir)
    yield tempdir
  finally:
--- a/tox.ini
+++ b/tox.ini
@ -15,13 +15,15 @@
 # https://tox.readthedocs.io/

 [tox]
-envlist = py36, py37, py38
+envlist = py35, py36, py37, py38, py39

 [gh-actions]
 python =
+    3.5: py35
    3.6: py36
    3.7: py37
    3.8: py38
+    3.9: py39

 [testenv]
 deps = pytest