Update manifest_name with the returned manifest name for superproject.

This is similar to smart sync, allowing sync to benefit from the patched
manifest.

Bug: [google internal] b/190688390
Change-Id: I158a80afceca606dcd81ec76b2caede369f7ed03
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/312142
Reviewed-by: Raman Tenneti <rtenneti@google.com>
Tested-by: Xin Li <delphij@google.com>

.github/workflows/test-ci.yml

@@ -14,7 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.6, 3.7, 3.8, 3.9]
     runs-on: ${{ matrix.os }}
 
     steps:

command.py

@@ -12,15 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import multiprocessing
 import os
 import optparse
-import platform
 import re
 import sys
 
 from event_log import EventLog
 from error import NoSuchProjectError
 from error import InvalidProjectGroupsError
+import progress
 
 
 # Number of projects to submit to a single worker process at a time.
@@ -41,15 +42,32 @@ class Command(object):
   """Base class for any command line action in repo.
   """
 
-  common = False
+  # Singleton for all commands to track overall repo command execution and
+  # provide event summary to callers.  Only used by sync subcommand currently.
+  #
+  # NB: This is being replaced by git trace2 events.  See git_trace2_event_log.
   event_log = EventLog()
-  manifest = None
-  _optparse = None
+
+  # Whether this command is a "common" one, i.e. whether the user would commonly
+  # use it or it's a more uncommon command.  This is used by the help command to
+  # show short-vs-full summaries.
+  COMMON = False
 
   # Whether this command supports running in parallel.  If greater than 0,
   # it is the number of parallel jobs to default to.
   PARALLEL_JOBS = None
 
+  def __init__(self, repodir=None, client=None, manifest=None, gitc_manifest=None,
+               git_event_log=None):
+    self.repodir = repodir
+    self.client = client
+    self.manifest = manifest
+    self.gitc_manifest = gitc_manifest
+    self.git_event_log = git_event_log
+
+    # Cache for the OptionParser property.
+    self._optparse = None
+
   def WantPager(self, _opt):
     return False
 
@@ -84,18 +102,34 @@ class Command(object):
         usage = 'repo %s' % self.NAME
       epilog = 'Run `repo help %s` to view the detailed manual.' % self.NAME
       self._optparse = optparse.OptionParser(usage=usage, epilog=epilog)
+      self._CommonOptions(self._optparse)
       self._Options(self._optparse)
     return self._optparse
 
-  def _Options(self, p):
-    """Initialize the option parser.
+  def _CommonOptions(self, p, opt_v=True):
+    """Initialize the option parser with common options.
+
+    These will show up for *all* subcommands, so use sparingly.
+    NB: Keep in sync with repo:InitParser().
     """
+    g = p.add_option_group('Logging options')
+    opts = ['-v'] if opt_v else []
+    g.add_option(*opts, '--verbose',
+                 dest='output_mode', action='store_true',
+                 help='show all output')
+    g.add_option('-q', '--quiet',
+                 dest='output_mode', action='store_false',
+                 help='only show errors')
+
     if self.PARALLEL_JOBS is not None:
       p.add_option(
           '-j', '--jobs',
           type=int, default=self.PARALLEL_JOBS,
           help='number of jobs to run in parallel (default: %s)' % self.PARALLEL_JOBS)
 
+  def _Options(self, p):
+    """Initialize the option parser with subcommand-specific options."""
+
   def _RegisteredEnvironmentOptions(self):
     """Get options that can be set from environment variables.
 
@@ -120,6 +154,11 @@ class Command(object):
     self.OptionParser.print_usage()
     sys.exit(1)
 
+  def CommonValidateOptions(self, opt, args):
+    """Validate common options."""
+    opt.quiet = opt.output_mode is False
+    opt.verbose = opt.output_mode is True
+
   def ValidateOptions(self, opt, args):
     """Validate the user options & arguments before executing.
 
@@ -135,6 +174,44 @@ class Command(object):
     """
     raise NotImplementedError
 
+  @staticmethod
+  def ExecuteInParallel(jobs, func, inputs, callback, output=None, ordered=False):
+    """Helper for managing parallel execution boiler plate.
+
+    For subcommands that can easily split their work up.
+
+    Args:
+      jobs: How many parallel processes to use.
+      func: The function to apply to each of the |inputs|.  Usually a
+          functools.partial for wrapping additional arguments.  It will be run
+          in a separate process, so it must be pickalable, so nested functions
+          won't work.  Methods on the subcommand Command class should work.
+      inputs: The list of items to process.  Must be a list.
+      callback: The function to pass the results to for processing.  It will be
+          executed in the main thread and process the results of |func| as they
+          become available.  Thus it may be a local nested function.  Its return
+          value is passed back directly.  It takes three arguments:
+          - The processing pool (or None with one job).
+          - The |output| argument.
+          - An iterator for the results.
+      output: An output manager.  May be progress.Progess or color.Coloring.
+      ordered: Whether the jobs should be processed in order.
+
+    Returns:
+      The |callback| function's results are returned.
+    """
+    try:
+      # NB: Multiprocessing is heavy, so don't spin it up for one job.
+      if len(inputs) == 1 or jobs == 1:
+        return callback(None, output, (func(x) for x in inputs))
+      else:
+        with multiprocessing.Pool(jobs) as pool:
+          submit = pool.imap if ordered else pool.imap_unordered
+          return callback(pool, output, submit(func, inputs, chunksize=WORKER_BATCH_SIZE))
+    finally:
+      if isinstance(output, progress.Progress):
+        output.end()
+
   def _ResetPathToProjectMap(self, projects):
     self._by_path = dict((p.worktree, p) for p in projects)
 
@@ -178,9 +255,7 @@ class Command(object):
     mp = manifest.manifestProject
 
     if not groups:
-      groups = mp.config.GetString('manifest.groups')
-    if not groups:
-      groups = 'default,platform-' + platform.system().lower()
+      groups = manifest.GetGroupsStr()
     groups = [x for x in re.split(r'[,\s]+', groups) if x]
 
     if not args:

completion.bash

@@ -0,0 +1,121 @@
+# Copyright 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Programmable bash completion.  https://github.com/scop/bash-completion
+
+# Complete the list of repo subcommands.
+__complete_repo_list_commands() {
+  local repo=${COMP_WORDS[0]}
+  (
+  # Handle completions if running outside of a checkout.
+  if ! "${repo}" help --all 2>/dev/null; then
+    repo help 2>/dev/null
+  fi
+  ) | sed -n '/^  /{s/  \([^ ]\+\) .\+/\1/;p}'
+}
+
+# Complete list of all branches available in all projects in the repo client
+# checkout.
+__complete_repo_list_branches() {
+  local repo=${COMP_WORDS[0]}
+  "${repo}" branches 2>/dev/null | \
+    sed -n '/|/{s/[ *][Pp ] *\([^ ]\+\) .*/\1/;p}'
+}
+
+# Complete list of all projects available in the repo client checkout.
+__complete_repo_list_projects() {
+  local repo=${COMP_WORDS[0]}
+  "${repo}" list -n 2>/dev/null
+}
+
+# Complete the repo <command> argument.
+__complete_repo_command() {
+  if [[ ${COMP_CWORD} -ne 1 ]]; then
+    return 1
+  fi
+
+  local command=${COMP_WORDS[1]}
+  COMPREPLY=($(compgen -W "$(__complete_repo_list_commands)" -- "${command}"))
+  return 0
+}
+
+# Complete repo subcommands that take <branch> <projects>.
+__complete_repo_command_branch_projects() {
+  local current=$1
+  if [[ ${COMP_CWORD} -eq 2 ]]; then
+    COMPREPLY=($(compgen -W "$(__complete_repo_list_branches)" -- "${current}"))
+  else
+    COMPREPLY=($(compgen -W "$(__complete_repo_list_projects)" -- "${current}"))
+  fi
+}
+
+# Complete repo subcommands that take only <projects>.
+__complete_repo_command_projects() {
+  local current=$1
+  COMPREPLY=($(compgen -W "$(__complete_repo_list_projects)" -- "${current}"))
+}
+
+# Complete the repo subcommand arguments.
+__complete_repo_arg() {
+  if [[ ${COMP_CWORD} -le 1 ]]; then
+    return 1
+  fi
+
+  local command=${COMP_WORDS[1]}
+  local current=${COMP_WORDS[COMP_CWORD]}
+  case ${command} in
+  abandon|checkout)
+    __complete_repo_command_branch_projects "${current}"
+    return 0
+    ;;
+
+  branch|branches|diff|info|list|overview|prune|rebase|smartsync|stage|status|\
+  sync|upload)
+    __complete_repo_command_projects "${current}"
+    return 0
+    ;;
+
+  help)
+    if [[ ${COMP_CWORD} -eq 2 ]]; then
+      COMPREPLY=(
+        $(compgen -W "$(__complete_repo_list_commands)" -- "${current}")
+      )
+    fi
+    return 0
+    ;;
+
+  start)
+    if [[ ${COMP_CWORD} -gt 2 ]]; then
+      COMPREPLY=(
+        $(compgen -W "$(__complete_repo_list_projects)" -- "${current}")
+      )
+    fi
+    return 0
+    ;;
+
+  *)
+    return 1
+    ;;
+  esac
+}
+
+# Complete the repo arguments.
+__complete_repo() {
+  COMPREPLY=()
+  __complete_repo_command && return 0
+  __complete_repo_arg && return 0
+  return 0
+}
+
+complete -F __complete_repo repo

docs/internal-fs-layout.md

@@ -110,6 +110,8 @@ Instead, you should use standard Git workflows like [git worktree] or
 [gitsubmodules] with [superprojects].
 ***
 
+*   `copy-link-files.json`: Tracking file used by `repo sync` to determine when
+    copyfile or linkfile are added or removed and need corresponding updates.
 *   `project.list`: Tracking file used by `repo sync` to determine when projects
     are added or removed and need corresponding updates in the checkout.
 *   `projects/`: Bare checkouts of every project synced by the manifest.  The
@@ -147,22 +149,23 @@ repo client checkout.
 Most settings use the `[repo]` section to avoid conflicts with git.
 User controlled settings are initialized when running `repo init`.
 
-| Setting           | `repo init` Option        | Use/Meaning |
-|-------------------|---------------------------|-------------|
-| manifest.groups   | `--groups` & `--platform` | The manifest groups to sync |
-| repo.archive      | `--archive`               | Use `git archive` for checkouts |
-| repo.clonebundle  | `--clone-bundle`          | Whether the initial sync used clone.bundle explicitly |
-| repo.clonefilter  | `--clone-filter`          | Filter setting when using [partial git clones] |
-| repo.depth        | `--depth`                 | Create shallow checkouts when cloning |
-| repo.dissociate   | `--dissociate`            | Dissociate from any reference/mirrors after initial clone |
-| repo.mirror       | `--mirror`                | Checkout is a repo mirror |
-| repo.partialclone | `--partial-clone`         | Create [partial git clones] |
-| repo.reference    | `--reference`             | Reference repo client checkout |
-| repo.submodules   | `--submodules`            | Sync git submodules |
-| repo.superproject | `--use-superproject`      | Sync [superproject] |
-| repo.worktree     | `--worktree`              | Use [git worktree] for checkouts |
-| user.email        | `--config-name`           | User's e-mail address; Copied into `.git/config` when checking out a new project |
-| user.name         | `--config-name`           | User's name; Copied into `.git/config` when checking out a new project |
+| Setting                  | `repo init` Option        | Use/Meaning |
+|-------------------       |---------------------------|-------------|
+| manifest.groups          | `--groups` & `--platform` | The manifest groups to sync |
+| repo.archive             | `--archive`               | Use `git archive` for checkouts |
+| repo.clonebundle         | `--clone-bundle`          | Whether the initial sync used clone.bundle explicitly |
+| repo.clonefilter         | `--clone-filter`          | Filter setting when using [partial git clones] |
+| repo.depth               | `--depth`                 | Create shallow checkouts when cloning |
+| repo.dissociate          | `--dissociate`            | Dissociate from any reference/mirrors after initial clone |
+| repo.mirror              | `--mirror`                | Checkout is a repo mirror |
+| repo.partialclone        | `--partial-clone`         | Create [partial git clones] |
+| repo.partialcloneexclude | `--partial-clone-exclude` | Comma-delimited list of project names (not paths) to exclude while using [partial git clones] |
+| repo.reference           | `--reference`             | Reference repo client checkout |
+| repo.submodules          | `--submodules`            | Sync git submodules |
+| repo.superproject        | `--use-superproject`      | Sync [superproject] |
+| repo.worktree            | `--worktree`              | Use [git worktree] for checkouts |
+| user.email               | `--config-name`           | User's e-mail address; Copied into `.git/config` when checking out a new project |
+| user.name                | `--config-name`           | User's name; Copied into `.git/config` when checking out a new project |
 
 [partial git clones]: https://git-scm.com/docs/gitrepository-layout#_code_partialclone_code
 [superproject]: https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects

docs/manifest-format.md

@@ -21,6 +21,7 @@ following DTD:
 
 ```xml
 <!DOCTYPE manifest [
+
   <!ELEMENT manifest (notice?,
                       remote*,
                       default?,
@@ -30,6 +31,7 @@ following DTD:
                       extend-project*,
                       repo-hooks?,
                       superproject?,
+                      contactinfo?,
                       include*)>
 
   <!ELEMENT notice (#PCDATA)>
@@ -94,15 +96,19 @@ following DTD:
 
   <!ELEMENT remove-project EMPTY>
   <!ATTLIST remove-project name  CDATA #REQUIRED>
+  <!ATTLIST remove-project optional  CDATA #IMPLIED>
 
   <!ELEMENT repo-hooks EMPTY>
   <!ATTLIST repo-hooks in-project CDATA #REQUIRED>
   <!ATTLIST repo-hooks enabled-list CDATA #REQUIRED>
 
-  <!ELEMENT superproject (EMPTY)>
+  <!ELEMENT superproject EMPTY>
   <!ATTLIST superproject name    CDATA #REQUIRED>
   <!ATTLIST superproject remote  IDREF #IMPLIED>
 
+  <!ELEMENT contactinfo EMPTY>
+  <!ATTLIST contactinfo bugurl  CDATA #REQUIRED>
+
   <!ELEMENT include EMPTY>
   <!ATTLIST include name   CDATA #REQUIRED>
   <!ATTLIST include groups CDATA #IMPLIED>
@@ -252,12 +258,25 @@ name will be prefixed by the parent's.
 The project name must match the name Gerrit knows, if Gerrit is
 being used for code reviews.
 
+"name" must not be empty, and may not be an absolute path or use "." or ".."
+path components.  It is always interpreted relative to the remote's fetch
+settings, so if a different base path is needed, declare a different remote
+with the new settings needed.
+These restrictions are not enforced for [Local Manifests].
+
 Attribute `path`: An optional path relative to the top directory
 of the repo client where the Git working directory for this project
-should be placed.  If not supplied the project name is used.
+should be placed.  If not supplied the project "name" is used.
 If the project has a parent element, its path will be prefixed
 by the parent's.
 
+"path" may not be an absolute path or use "." or ".." path components.
+These restrictions are not enforced for [Local Manifests].
+
+If you want to place files into the root of the checkout (e.g. a README or
+Makefile or another build script), use the [copyfile] or [linkfile] elements
+instead.
+
 Attribute `remote`: Name of a previously defined remote element.
 If not supplied the remote given by the default element is used.
 
@@ -375,6 +394,9 @@ This element is mostly useful in a local manifest file, where
 the user can remove a project, and possibly replace it with their
 own definition.
 
+Attribute `optional`: Set to true to ignore remove-project elements with no
+matching `project` element.
+
 ### Element repo-hooks
 
 NB: See the [practical documentation](./repo-hooks.md) for using repo hooks.
@@ -391,7 +413,7 @@ Attribute `enabled-list`: List of hooks to use, whitespace or comma separated.
 ### Element superproject
 
 ***
- *Note*: This is currently a WIP.
+*Note*: This is currently a WIP.
 ***
 
 NB: See the [git superprojects documentation](
@@ -410,6 +432,19 @@ same meaning as project's name attribute. See the
 Attribute `remote`: Name of a previously defined remote element.
 If not supplied the remote given by the default element is used.
 
+### Element contactinfo
+
+***
+*Note*: This is currently a WIP.
+***
+
+This element is used to let manifest authors self-register contact info.
+It has "bugurl" as a required atrribute. This element can be repeated,
+and any later entries will clobber earlier ones. This would allow manifest
+authors who extend manifests to specify their own contact info.
+
+Attribute `bugurl`: The URL to file a bug against the manifest owner.
+
 ### Element include
 
 This element provides the capability of including another manifest
@@ -419,12 +454,15 @@ target manifest to include - it must be a usable manifest on its own.
 Attribute `name`: the manifest to include, specified relative to
 the manifest repository's root.
 
+"name" may not be an absolute path or use "." or ".." path components.
+These restrictions are not enforced for [Local Manifests].
+
 Attribute `groups`: List of additional groups to which all projects
 in the included manifest belong. This appends and recurses, meaning
 all projects in sub-manifests carry all parent include groups.
 Same syntax as the corresponding element of `project`.
 
-## Local Manifests
+## Local Manifests {#local-manifests}
 
 Additional remotes and projects may be added through local manifest
 files stored in `$TOP_DIR/.repo/local_manifests/*.xml`.
@@ -451,4 +489,12 @@ these extra projects.
 Manifest files stored in `$TOP_DIR/.repo/local_manifests/*.xml` will
 be loaded in alphabetical order.
 
+Projects from local manifest files are added into
+local::<local manifest filename> group.
+
 The legacy `$TOP_DIR/.repo/local_manifest.xml` path is no longer supported.
+
+
+[copyfile]: #Element-copyfile
+[linkfile]: #Element-linkfile
+[Local Manifests]: #local-manifests

docs/release-process.md

@@ -83,7 +83,8 @@ control how repo finds updates:
 *   `--repo-rev`: This tells repo which branch to use for the full project.
     It defaults to the `stable` branch (`REPO_REV` in the launcher script).
 
-Whenever `repo sync` is run, repo will check to see if an update is available.
+Whenever `repo sync` is run, repo will, once every 24 hours, see if an update
+is available.
 It fetches the latest repo-rev from the repo-url.
 Then it verifies that the latest commit in the branch has a valid signed tag
 using `git tag -v` (which uses gpg).
@@ -95,6 +96,11 @@ If that tag is valid, then repo will warn and use that commit instead.
 
 If that tag cannot be verified, it gives up and forces the user to resolve.
 
+### Force an update
+
+The `repo selfupdate` command can be used to force an immediate update.
+It is not subject to the 24 hour limitation.
+
 ## Branch management
 
 All development happens on the `main` branch and should generally be stable.

error.py

@@ -13,10 +13,6 @@
 # limitations under the License.
 
 
-# URL to file bug reports for repo tool issues.
-BUG_REPORT_URL = 'https://bugs.chromium.org/p/gerrit/issues/entry?template=Repo+tool+issue'
-
-
 class ManifestParseError(Exception):
   """Failed to parse the manifest file.
   """

git_command.py

@@ -12,12 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import functools
 import os
-import re
 import sys
 import subprocess
-import tempfile
-from signal import SIGTERM
 
 from error import GitError
 from git_refs import HEAD
@@ -42,101 +40,15 @@ GIT_DIR = 'GIT_DIR'
 LAST_GITDIR = None
 LAST_CWD = None
 
-_ssh_proxy_path = None
-_ssh_sock_path = None
-_ssh_clients = []
-_ssh_version = None
-
-
-def _run_ssh_version():
-  """run ssh -V to display the version number"""
-  return subprocess.check_output(['ssh', '-V'], stderr=subprocess.STDOUT).decode()
-
-
-def _parse_ssh_version(ver_str=None):
-  """parse a ssh version string into a tuple"""
-  if ver_str is None:
-    ver_str = _run_ssh_version()
-  m = re.match(r'^OpenSSH_([0-9.]+)(p[0-9]+)?\s', ver_str)
-  if m:
-    return tuple(int(x) for x in m.group(1).split('.'))
-  else:
-    return ()
-
-
-def ssh_version():
-  """return ssh version as a tuple"""
-  global _ssh_version
-  if _ssh_version is None:
-    try:
-      _ssh_version = _parse_ssh_version()
-    except subprocess.CalledProcessError:
-      print('fatal: unable to detect ssh version', file=sys.stderr)
-      sys.exit(1)
-  return _ssh_version
-
-
-def ssh_sock(create=True):
-  global _ssh_sock_path
-  if _ssh_sock_path is None:
-    if not create:
-      return None
-    tmp_dir = '/tmp'
-    if not os.path.exists(tmp_dir):
-      tmp_dir = tempfile.gettempdir()
-    if ssh_version() < (6, 7):
-      tokens = '%r@%h:%p'
-    else:
-      tokens = '%C'  # hash of %l%h%p%r
-    _ssh_sock_path = os.path.join(
-        tempfile.mkdtemp('', 'ssh-', tmp_dir),
-        'master-' + tokens)
-  return _ssh_sock_path
-
-
-def _ssh_proxy():
-  global _ssh_proxy_path
-  if _ssh_proxy_path is None:
-    _ssh_proxy_path = os.path.join(
-        os.path.dirname(__file__),
-        'git_ssh')
-  return _ssh_proxy_path
-
-
-def _add_ssh_client(p):
-  _ssh_clients.append(p)
-
-
-def _remove_ssh_client(p):
-  try:
-    _ssh_clients.remove(p)
-  except ValueError:
-    pass
-
-
-def terminate_ssh_clients():
-  global _ssh_clients
-  for p in _ssh_clients:
-    try:
-      os.kill(p.pid, SIGTERM)
-      p.wait()
-    except OSError:
-      pass
-  _ssh_clients = []
-
-
-_git_version = None
-
 
 class _GitCall(object):
+  @functools.lru_cache(maxsize=None)
   def version_tuple(self):
-    global _git_version
-    if _git_version is None:
-      _git_version = Wrapper().ParseGitVersion()
-      if _git_version is None:
-        print('fatal: unable to detect git version', file=sys.stderr)
-        sys.exit(1)
-    return _git_version
+    ret = Wrapper().ParseGitVersion()
+    if ret is None:
+      print('fatal: unable to detect git version', file=sys.stderr)
+      sys.exit(1)
+    return ret
 
   def __getattr__(self, name):
     name = name.replace('_', '-')
@@ -163,7 +75,8 @@ def RepoSourceVersion():
     proj = os.path.dirname(os.path.abspath(__file__))
     env[GIT_DIR] = os.path.join(proj, '.git')
     result = subprocess.run([GIT, 'describe', HEAD], stdout=subprocess.PIPE,
-                            encoding='utf-8', env=env, check=False)
+                            stderr=subprocess.DEVNULL, encoding='utf-8',
+                            env=env, check=False)
     if result.returncode == 0:
       ver = result.stdout.strip()
       if ver.startswith('v'):
@@ -254,7 +167,7 @@ class GitCommand(object):
                capture_stderr=False,
                merge_output=False,
                disable_editor=False,
-               ssh_proxy=False,
+               ssh_proxy=None,
                cwd=None,
                gitdir=None):
     env = self._GetBasicEnv()
@@ -262,8 +175,8 @@ class GitCommand(object):
     if disable_editor:
       env['GIT_EDITOR'] = ':'
     if ssh_proxy:
-      env['REPO_SSH_SOCK'] = ssh_sock()
-      env['GIT_SSH'] = _ssh_proxy()
+      env['REPO_SSH_SOCK'] = ssh_proxy.sock()
+      env['GIT_SSH'] = ssh_proxy.proxy
       env['GIT_SSH_VARIANT'] = 'ssh'
     if 'http_proxy' in env and 'darwin' == sys.platform:
       s = "'http.proxy=%s'" % (env['http_proxy'],)
@@ -346,7 +259,7 @@ class GitCommand(object):
       raise GitError('%s: %s' % (command[1], e))
 
     if ssh_proxy:
-      _add_ssh_client(p)
+      ssh_proxy.add_client(p)
 
     self.process = p
     if input:
@@ -358,7 +271,8 @@ class GitCommand(object):
     try:
       self.stdout, self.stderr = p.communicate()
     finally:
-      _remove_ssh_client(p)
+      if ssh_proxy:
+        ssh_proxy.remove_client(p)
     self.rc = p.wait()
 
   @staticmethod

git_config.py

@@ -18,25 +18,16 @@ from http.client import HTTPException
 import json
 import os
 import re
-import signal
 import ssl
 import subprocess
 import sys
-try:
-  import threading as _threading
-except ImportError:
-  import dummy_threading as _threading
-import time
 import urllib.error
 import urllib.request
 
 from error import GitError, UploadError
 import platform_utils
 from repo_trace import Trace
-
 from git_command import GitCommand
-from git_command import ssh_sock
-from git_command import terminate_ssh_clients
 from git_refs import R_CHANGES, R_HEADS, R_TAGS
 
 ID_RE = re.compile(r'^[0-9a-f]{40}$')
@@ -74,6 +65,15 @@ class GitConfig(object):
 
   _USER_CONFIG = '~/.gitconfig'
 
+  _ForSystem = None
+  _SYSTEM_CONFIG = '/etc/gitconfig'
+
+  @classmethod
+  def ForSystem(cls):
+    if cls._ForSystem is None:
+      cls._ForSystem = cls(configfile=cls._SYSTEM_CONFIG)
+    return cls._ForSystem
+
   @classmethod
   def ForUser(cls):
     if cls._ForUser is None:
@@ -145,6 +145,21 @@ class GitConfig(object):
     except ValueError:
       return None
 
+  def DumpConfigDict(self):
+    """Returns the current configuration dict.
+
+    Configuration data is information only (e.g. logging) and
+    should not be considered a stable data-source.
+
+    Returns:
+      dict of {<key>, <value>} for git configuration cache.
+      <value> are strings converted by GetString.
+    """
+    config_dict = {}
+    for key in self._cache:
+      config_dict[key] = self.GetString(key)
+    return config_dict
+
   def GetBoolean(self, name):
     """Returns a boolean from the configuration file.
        None : The value was not defined, or is not a boolean.
@@ -350,7 +365,10 @@ class GitConfig(object):
     return c
 
   def _do(self, *args):
-    command = ['config', '--file', self.file, '--includes']
+    if self.file == self._SYSTEM_CONFIG:
+      command = ['config', '--system', '--includes']
+    else:
+      command = ['config', '--file', self.file, '--includes']
     command.extend(args)
 
     p = GitCommand(None,
@@ -425,127 +443,6 @@ class RefSpec(object):
     return s
 
 
-_master_processes = []
-_master_keys = set()
-_ssh_master = True
-_master_keys_lock = None
-
-
-def init_ssh():
-  """Should be called once at the start of repo to init ssh master handling.
-
-  At the moment, all we do is to create our lock.
-  """
-  global _master_keys_lock
-  assert _master_keys_lock is None, "Should only call init_ssh once"
-  _master_keys_lock = _threading.Lock()
-
-
-def _open_ssh(host, port=None):
-  global _ssh_master
-
-  # Acquire the lock.  This is needed to prevent opening multiple masters for
-  # the same host when we're running "repo sync -jN" (for N > 1) _and_ the
-  # manifest <remote fetch="ssh://xyz"> specifies a different host from the
-  # one that was passed to repo init.
-  _master_keys_lock.acquire()
-  try:
-
-    # Check to see whether we already think that the master is running; if we
-    # think it's already running, return right away.
-    if port is not None:
-      key = '%s:%s' % (host, port)
-    else:
-      key = host
-
-    if key in _master_keys:
-      return True
-
-    if (not _ssh_master
-            or 'GIT_SSH' in os.environ
-            or sys.platform in ('win32', 'cygwin')):
-      # failed earlier, or cygwin ssh can't do this
-      #
-      return False
-
-    # We will make two calls to ssh; this is the common part of both calls.
-    command_base = ['ssh',
-                    '-o', 'ControlPath %s' % ssh_sock(),
-                    host]
-    if port is not None:
-      command_base[1:1] = ['-p', str(port)]
-
-    # Since the key wasn't in _master_keys, we think that master isn't running.
-    # ...but before actually starting a master, we'll double-check.  This can
-    # be important because we can't tell that that 'git@myhost.com' is the same
-    # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file.
-    check_command = command_base + ['-O', 'check']
-    try:
-      Trace(': %s', ' '.join(check_command))
-      check_process = subprocess.Popen(check_command,
-                                       stdout=subprocess.PIPE,
-                                       stderr=subprocess.PIPE)
-      check_process.communicate()  # read output, but ignore it...
-      isnt_running = check_process.wait()
-
-      if not isnt_running:
-        # Our double-check found that the master _was_ infact running.  Add to
-        # the list of keys.
-        _master_keys.add(key)
-        return True
-    except Exception:
-      # Ignore excpetions.  We we will fall back to the normal command and print
-      # to the log there.
-      pass
-
-    command = command_base[:1] + ['-M', '-N'] + command_base[1:]
-    try:
-      Trace(': %s', ' '.join(command))
-      p = subprocess.Popen(command)
-    except Exception as e:
-      _ssh_master = False
-      print('\nwarn: cannot enable ssh control master for %s:%s\n%s'
-            % (host, port, str(e)), file=sys.stderr)
-      return False
-
-    time.sleep(1)
-    ssh_died = (p.poll() is not None)
-    if ssh_died:
-      return False
-
-    _master_processes.append(p)
-    _master_keys.add(key)
-    return True
-  finally:
-    _master_keys_lock.release()
-
-
-def close_ssh():
-  global _master_keys_lock
-
-  terminate_ssh_clients()
-
-  for p in _master_processes:
-    try:
-      os.kill(p.pid, signal.SIGTERM)
-      p.wait()
-    except OSError:
-      pass
-  del _master_processes[:]
-  _master_keys.clear()
-
-  d = ssh_sock(create=False)
-  if d:
-    try:
-      platform_utils.rmdir(os.path.dirname(d))
-    except OSError:
-      pass
-
-  # We're done with the lock, so we can delete it.
-  _master_keys_lock = None
-
-
-URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
 URI_ALL = re.compile(r'^([a-z][a-z+-]*)://([^@/]*@?[^/]*)/')
 
 
@@ -597,27 +494,6 @@ def GetUrlCookieFile(url, quiet):
   yield cookiefile, None
 
 
-def _preconnect(url):
-  m = URI_ALL.match(url)
-  if m:
-    scheme = m.group(1)
-    host = m.group(2)
-    if ':' in host:
-      host, port = host.split(':')
-    else:
-      port = None
-    if scheme in ('ssh', 'git+ssh', 'ssh+git'):
-      return _open_ssh(host, port)
-    return False
-
-  m = URI_SCP.match(url)
-  if m:
-    host = m.group(1)
-    return _open_ssh(host)
-
-  return False
-
-
 class Remote(object):
   """Configuration options related to a remote.
   """
@@ -654,9 +530,23 @@ class Remote(object):
 
     return self.url.replace(longest, longestUrl, 1)
 
-  def PreConnectFetch(self):
+  def PreConnectFetch(self, ssh_proxy):
+    """Run any setup for this remote before we connect to it.
+
+    In practice, if the remote is using SSH, we'll attempt to create a new
+    SSH master session to it for reuse across projects.
+
+    Args:
+      ssh_proxy: The SSH settings for managing master sessions.
+
+    Returns:
+      Whether the preconnect phase for this remote was successful.
+    """
+    if not ssh_proxy:
+      return True
+
     connectionUrl = self._InsteadOf()
-    return _preconnect(connectionUrl)
+    return ssh_proxy.preconnect(connectionUrl)
 
   def ReviewUrl(self, userEmail, validate_certs):
     if self._review_url is None:

git_superproject.py

@@ -19,21 +19,52 @@ https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects
 
 Examples:
   superproject = Superproject()
-  project_commit_ids = superproject.UpdateProjectsRevisionId(projects)
+  UpdateProjectsResult = superproject.UpdateProjectsRevisionId(projects)
 """
 
 import hashlib
+import functools
 import os
 import sys
+import time
+from typing import NamedTuple
 
-from error import BUG_REPORT_URL
-from git_command import GitCommand
+from git_command import git_require, GitCommand
+from git_config import RepoConfig
 from git_refs import R_HEADS
+from manifest_xml import LOCAL_MANIFEST_GROUP_PREFIX
 
 _SUPERPROJECT_GIT_NAME = 'superproject.git'
 _SUPERPROJECT_MANIFEST_NAME = 'superproject_override.xml'
 
 
+class SyncResult(NamedTuple):
+  """Return the status of sync and whether caller should exit."""
+
+  # Whether the superproject sync was successful.
+  success: bool
+  # Whether the caller should exit.
+  fatal: bool
+
+
+class CommitIdsResult(NamedTuple):
+  """Return the commit ids and whether caller should exit."""
+
+  # A dictionary with the projects/commit ids on success, otherwise None.
+  commit_ids: dict
+  # Whether the caller should exit.
+  fatal: bool
+
+
+class UpdateProjectsResult(NamedTuple):
+  """Return the overriding manifest file and whether caller should exit."""
+
+  # Path name of the overriding manfiest file if successful, otherwise None.
+  manifest_path: str
+  # Whether the caller should exit.
+  fatal: bool
+
+
 class Superproject(object):
   """Get commit ids from superproject.
 
@@ -41,19 +72,21 @@ class Superproject(object):
   lookup of commit ids for all projects. It contains _project_commit_ids which
   is a dictionary with project/commit id entries.
   """
-  def __init__(self, manifest, repodir, superproject_dir='exp-superproject',
-               quiet=False):
+  def __init__(self, manifest, repodir, git_event_log,
+               superproject_dir='exp-superproject', quiet=False):
     """Initializes superproject.
 
     Args:
       manifest: A Manifest object that is to be written to a file.
       repodir: Path to the .repo/ dir for holding all internal checkout state.
           It must be in the top directory of the repo client checkout.
+      git_event_log: A git trace2 event log to log events.
       superproject_dir: Relative path under |repodir| to checkout superproject.
       quiet:  If True then only print the progress messages.
     """
     self._project_commit_ids = None
     self._manifest = manifest
+    self._git_event_log = git_event_log
     self._quiet = quiet
     self._branch = self._GetBranch()
     self._repodir = os.path.abspath(repodir)
@@ -84,6 +117,11 @@ class Superproject(object):
       branch = branch[len(R_HEADS):]
     return branch
 
+  def _LogError(self, message):
+    """Logs message to stderr and _git_event_log."""
+    print(message, file=sys.stderr)
+    self._git_event_log.ErrorEvent(message, '')
+
   def _Init(self):
     """Sets up a local Git repository to get a copy of a superproject.
 
@@ -103,8 +141,8 @@ class Superproject(object):
                    capture_stderr=True)
     retval = p.Wait()
     if retval:
-      print('repo: error: git init call failed with return code: %r, stderr: %r' %
-            (retval, p.stderr), file=sys.stderr)
+      self._LogError(f'repo: error: git init call failed, command: git {cmd}, '
+                     f'return code: {retval}, stderr: {p.stderr}')
       return False
     return True
 
@@ -118,10 +156,12 @@ class Superproject(object):
       True if fetch is successful, or False.
     """
     if not os.path.exists(self._work_git):
-      print('git fetch missing drectory: %s' % self._work_git,
-            file=sys.stderr)
+      self._LogError(f'git fetch missing directory: {self._work_git}')
       return False
-    cmd = ['fetch', url, '--force', '--no-tags', '--filter', 'blob:none']
+    if not git_require((2, 28, 0)):
+      print('superproject requires a git version 2.28 or later', file=sys.stderr)
+      return False
+    cmd = ['fetch', url, '--depth', '1', '--force', '--no-tags', '--filter', 'blob:none']
     if self._branch:
       cmd += [self._branch + ':' + self._branch]
     p = GitCommand(None,
@@ -131,8 +171,8 @@ class Superproject(object):
                    capture_stderr=True)
     retval = p.Wait()
     if retval:
-      print('repo: error: git fetch call failed with return code: %r, stderr: %r' %
-            (retval, p.stderr), file=sys.stderr)
+      self._LogError(f'repo: error: git fetch call failed, command: git {cmd}, '
+                     f'return code: {retval}, stderr: {p.stderr}')
       return False
     return True
 
@@ -145,8 +185,7 @@ class Superproject(object):
       data: data returned from 'git ls-tree ...' instead of None.
     """
     if not os.path.exists(self._work_git):
-      print('git ls-tree missing drectory: %s' % self._work_git,
-            file=sys.stderr)
+      self._LogError(f'git ls-tree missing directory: {self._work_git}')
       return None
     data = None
     branch = 'HEAD' if not self._branch else self._branch
@@ -161,52 +200,54 @@ class Superproject(object):
     if retval == 0:
       data = p.stdout
     else:
-      print('repo: error: git ls-tree call failed with return code: %r, stderr: %r' % (
-          retval, p.stderr), file=sys.stderr)
+      self._LogError(f'repo: error: git ls-tree call failed, command: git {cmd}, '
+                     f'return code: {retval}, stderr: {p.stderr}')
     return data
 
   def Sync(self):
     """Gets a local copy of a superproject for the manifest.
 
     Returns:
-      True if sync of superproject is successful, or False.
+      SyncResult
     """
-    print('WARNING: --use-superproject is experimental and not '
-          'for general use', file=sys.stderr)
+    print('NOTICE: --use-superproject is in beta; report any issues to the '
+          'address described in `repo version`', file=sys.stderr)
 
     if not self._manifest.superproject:
-      print('error: superproject tag is not defined in manifest',
-            file=sys.stderr)
-      return False
+      self._LogError(f'repo error: superproject tag is not defined in manifest: '
+                     f'{self._manifest.manifestFile}')
+      return SyncResult(False, False)
 
+    should_exit = True
     url = self._manifest.superproject['remote'].url
     if not url:
-      print('error: superproject URL is not defined in manifest',
-            file=sys.stderr)
-      return False
+      self._LogError(f'repo error: superproject URL is not defined in manifest: '
+                     f'{self._manifest.manifestFile}')
+      return SyncResult(False, should_exit)
 
     if not self._Init():
-      return False
+      return SyncResult(False, should_exit)
     if not self._Fetch(url):
-      return False
+      return SyncResult(False, should_exit)
     if not self._quiet:
       print('%s: Initial setup for superproject completed.' % self._work_git)
-    return True
+    return SyncResult(True, False)
 
   def _GetAllProjectsCommitIds(self):
     """Get commit ids for all projects from superproject and save them in _project_commit_ids.
 
     Returns:
-      A dictionary with the projects/commit ids on success, otherwise None.
+      CommitIdsResult
     """
-    if not self.Sync():
-      return None
+    sync_result = self.Sync()
+    if not sync_result.success:
+      return CommitIdsResult(None, sync_result.fatal)
 
     data = self._LsTree()
     if not data:
-      print('error: git ls-tree failed to return data for superproject',
+      print('warning: git ls-tree failed to return data for superproject',
             file=sys.stderr)
-      return None
+      return CommitIdsResult(None, True)
 
     # Parse lines like the following to select lines starting with '160000' and
     # build a dictionary with project path (last element) and its commit id (3rd element).
@@ -222,7 +263,7 @@ class Superproject(object):
         commit_ids[ls_data[3]] = ls_data[2]
 
     self._project_commit_ids = commit_ids
-    return commit_ids
+    return CommitIdsResult(commit_ids, False)
 
   def _WriteManfiestFile(self):
     """Writes manifest to a file.
@@ -231,22 +272,38 @@ class Superproject(object):
       manifest_path: Path name of the file into which manifest is written instead of None.
     """
     if not os.path.exists(self._superproject_path):
-      print('error: missing superproject directory %s' %
-            self._superproject_path,
-            file=sys.stderr)
+      self._LogError(f'error: missing superproject directory: {self._superproject_path}')
       return None
-    manifest_str = self._manifest.ToXml().toxml()
+    manifest_str = self._manifest.ToXml(groups=self._manifest.GetGroupsStr()).toxml()
     manifest_path = self._manifest_path
     try:
       with open(manifest_path, 'w', encoding='utf-8') as fp:
         fp.write(manifest_str)
     except IOError as e:
-      print('error: cannot write manifest to %s:\n%s'
-            % (manifest_path, e),
-            file=sys.stderr)
+      self._LogError(f'error: cannot write manifest to : {manifest_path} {e}')
       return None
     return manifest_path
 
+  def _SkipUpdatingProjectRevisionId(self, project):
+    """Checks if a project's revision id needs to be updated or not.
+
+    Revision id for projects from local manifest will not be updated.
+
+    Args:
+      project: project whose revision id is being updated.
+
+    Returns:
+      True if a project's revision id should not be updated, or False,
+    """
+    path = project.relpath
+    if not path:
+      return True
+    # Skip the project with revisionId.
+    if project.revisionId:
+      return True
+    # Skip the project if it comes from the local manifest.
+    return any(s.startswith(LOCAL_MANIFEST_GROUP_PREFIX) for s in project.groups)
+
   def UpdateProjectsRevisionId(self, projects):
     """Update revisionId of every project in projects with the commit id.
 
@@ -254,27 +311,111 @@ class Superproject(object):
       projects: List of projects whose revisionId needs to be updated.
 
     Returns:
-      manifest_path: Path name of the overriding manfiest file instead of None.
+      UpdateProjectsResult
     """
-    commit_ids = self._GetAllProjectsCommitIds()
+    commit_ids_result = self._GetAllProjectsCommitIds()
+    commit_ids = commit_ids_result.commit_ids
     if not commit_ids:
-      print('error: Cannot get project commit ids from manifest', file=sys.stderr)
-      return None
+      print('warning: Cannot get project commit ids from manifest', file=sys.stderr)
+      return UpdateProjectsResult(None, commit_ids_result.fatal)
 
     projects_missing_commit_ids = []
     for project in projects:
-      path = project.relpath
-      if not path:
+      if self._SkipUpdatingProjectRevisionId(project):
         continue
+      path = project.relpath
       commit_id = commit_ids.get(path)
-      if commit_id:
-        project.SetRevisionId(commit_id)
-      else:
+      if not commit_id:
         projects_missing_commit_ids.append(path)
+
+    # If superproject doesn't have a commit id for a project, then report an
+    # error event and continue as if do not use superproject is specified.
     if projects_missing_commit_ids:
-      print('error: please file a bug using %s to report missing commit_ids for: %s' %
-            (BUG_REPORT_URL, projects_missing_commit_ids), file=sys.stderr)
-      return None
+      self._LogError(f'error: please file a bug using {self._manifest.contactinfo.bugurl} '
+                     f'to report missing commit_ids for: {projects_missing_commit_ids}')
+      return UpdateProjectsResult(None, False)
+
+    for project in projects:
+      if not self._SkipUpdatingProjectRevisionId(project):
+        project.SetRevisionId(commit_ids.get(project.relpath))
 
     manifest_path = self._WriteManfiestFile()
-    return manifest_path
+    return UpdateProjectsResult(manifest_path, False)
+
+
+@functools.lru_cache(maxsize=None)
+def _UseSuperprojectFromConfiguration():
+  """Returns the user choice of whether to use superproject."""
+  user_cfg = RepoConfig.ForUser()
+  system_cfg = RepoConfig.ForSystem()
+  time_now = int(time.time())
+
+  user_value = user_cfg.GetBoolean('repo.superprojectChoice')
+  if user_value is not None:
+    user_expiration = user_cfg.GetInt('repo.superprojectChoiceExpire')
+    if user_expiration is not None and (user_expiration <= 0 or user_expiration >= time_now):
+      # TODO(b/190688390) - Remove prompt when we are comfortable with the new
+      # default value.
+      print(('You are currently enrolled in Git submodules experiment '
+             '(go/android-submodules-quickstart).  Use --no-use-superproject '
+             'to override.\n'), file=sys.stderr)
+    return user_value
+
+  # We don't have an unexpired choice, ask for one.
+  system_value = system_cfg.GetBoolean('repo.superprojectChoice')
+  if system_value:
+    # The system configuration is proposing that we should enable the
+    # use of superproject. Present this to user for confirmation if we
+    # are on a TTY, or, when we are not on a TTY, accept the system
+    # default for this time only.
+    #
+    # TODO(b/190688390) - Remove prompt when we are comfortable with the new
+    # default value.
+    prompt = ('Repo can now use Git submodules (go/android-submodules-quickstart) '
+              'instead of manifests to represent the state of the Android '
+              'superproject, which results in faster syncs and better atomicity.\n\n')
+    if sys.stdout.isatty():
+      prompt += 'Would you like to opt in for two weeks (y/N)? '
+      response = input(prompt).lower()
+      time_choiceexpire = time_now + (86400 * 14)
+      if response in ('y', 'yes'):
+        userchoice = True
+      elif response in ('a', 'always'):
+        userchoice = True
+        time_choiceexpire = 0
+      elif response == 'never':
+        userchoice = False
+        time_choiceexpire = 0
+      elif response in ('n', 'no'):
+        userchoice = False
+      else:
+        # Unrecognized user response, assume the intention was no, but
+        # only for 2 hours instead of 2 weeks to balance between not
+        # being overly pushy while still retain the opportunity to
+        # enroll.
+        userchoice = False
+        time_choiceexpire = time_now + 7200
+
+      user_cfg.SetString('repo.superprojectChoiceExpire', str(time_choiceexpire))
+      user_cfg.SetBoolean('repo.superprojectChoice', userchoice)
+
+      return userchoice
+    else:
+      print('Accepting once since we are not on a TTY', file=sys.stderr)
+      return True
+
+  # For all other cases, we would not use superproject by default.
+  return False
+
+
+def UseSuperproject(opt, manifest):
+  """Returns a boolean if use-superproject option is enabled."""
+
+  if opt.use_superproject is not None:
+    return opt.use_superproject
+  else:
+    client_value = manifest.manifestProject.config.GetBoolean('repo.superproject')
+    if client_value is not None:
+      return client_value
+    else:
+      return _UseSuperprojectFromConfiguration()

git_trace2_event_log.py

@@ -132,6 +132,40 @@ class EventLog(object):
     exit_event['code'] = result
     self._log.append(exit_event)
 
+  def CommandEvent(self, name, subcommands):
+    """Append a 'command' event to the current log.
+
+    Args:
+      name: Name of the primary command (ex: repo, git)
+      subcommands: List of the sub-commands (ex: version, init, sync)
+    """
+    command_event = self._CreateEventDict('command')
+    command_event['name'] = name
+    command_event['subcommands'] = subcommands
+    self._log.append(command_event)
+
+  def DefParamRepoEvents(self, config):
+    """Append a 'def_param' event for each repo.* config key to the current log.
+
+    Args:
+      config: Repo configuration dictionary
+    """
+    # Only output the repo.* config parameters.
+    repo_config = {k: v for k, v in config.items() if k.startswith('repo.')}
+
+    for param, value in repo_config.items():
+      def_param_event = self._CreateEventDict('def_param')
+      def_param_event['param'] = param
+      def_param_event['value'] = value
+      self._log.append(def_param_event)
+
+  def ErrorEvent(self, msg, fmt):
+    """Append a 'error' event to the current log."""
+    error_event = self._CreateEventDict('error')
+    error_event['msg'] = msg
+    error_event['fmt'] = fmt
+    self._log.append(error_event)
+
   def _GetEventTargetPath(self):
     """Get the 'trace2.eventtarget' path from git configuration.
 

gitc_utils.py

@@ -77,22 +77,6 @@ def _set_project_revisions(projects):
       project.revisionExpr = revisionExpr
 
 
-def _manifest_groups(manifest):
-  """Returns the manifest group string that should be synced
-
-  This is the same logic used by Command.GetProjects(), which is used during
-  repo sync
-
-  Args:
-    manifest: The XmlManifest object
-  """
-  mp = manifest.manifestProject
-  groups = mp.config.GetString('manifest.groups')
-  if not groups:
-    groups = 'default,platform-' + platform.system().lower()
-  return groups
-
-
 def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
   """Generate a manifest for shafsd to use for this GITC client.
 
@@ -107,7 +91,7 @@ def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
   if paths is None:
     paths = list(manifest.paths.keys())
 
-  groups = [x for x in re.split(r'[,\s]+', _manifest_groups(manifest)) if x]
+  groups = [x for x in re.split(r'[,\s]+', manifest.GetGroupsStr()) if x]
 
   # Convert the paths to projects, and filter them to the matched groups.
   projects = [manifest.paths[p] for p in paths]
@@ -166,7 +150,7 @@ def save_manifest(manifest, client_dir=None):
   else:
     manifest_file = os.path.join(client_dir, '.manifest')
   with open(manifest_file, 'w') as f:
-    manifest.Save(f, groups=_manifest_groups(manifest))
+    manifest.Save(f, groups=manifest.GetGroupsStr())
   # TODO(sbasi/jorg): Come up with a solution to remove the sleep below.
   # Give the GITC filesystem time to register the manifest changes.
   time.sleep(3)

main.py

@@ -39,7 +39,7 @@ from color import SetDefaultColoring
 import event_log
 from repo_trace import SetTrace
 from git_command import user_agent
-from git_config import init_ssh, close_ssh, RepoConfig
+from git_config import RepoConfig
 from git_trace2_event_log import EventLog
 from command import InteractiveCommand
 from command import MirrorSafeCommand
@@ -71,7 +71,7 @@ from subcmds import all_commands
 #
 # python-3.6 is in Ubuntu Bionic.
 MIN_PYTHON_VERSION_SOFT = (3, 6)
-MIN_PYTHON_VERSION_HARD = (3, 5)
+MIN_PYTHON_VERSION_HARD = (3, 6)
 
 if sys.version_info.major < 3:
   print('repo: error: Python 2 is no longer supported; '
@@ -195,23 +195,26 @@ class _Repo(object):
 
     SetDefaultColoring(gopts.color)
 
+    git_trace2_event_log = EventLog()
+    repo_client = RepoClient(self.repodir)
+    gitc_manifest = None
+    gitc_client_name = gitc_utils.parse_clientdir(os.getcwd())
+    if gitc_client_name:
+      gitc_manifest = GitcClient(self.repodir, gitc_client_name)
+      repo_client.isGitcClient = True
+
     try:
-      cmd = self.commands[name]()
+      cmd = self.commands[name](
+          repodir=self.repodir,
+          client=repo_client,
+          manifest=repo_client.manifest,
+          gitc_manifest=gitc_manifest,
+          git_event_log=git_trace2_event_log)
     except KeyError:
       print("repo: '%s' is not a repo command.  See 'repo help'." % name,
             file=sys.stderr)
       return 1
 
-    git_trace2_event_log = EventLog()
-    cmd.repodir = self.repodir
-    cmd.client = RepoClient(cmd.repodir)
-    cmd.manifest = cmd.client.manifest
-    cmd.gitc_manifest = None
-    gitc_client_name = gitc_utils.parse_clientdir(os.getcwd())
-    if gitc_client_name:
-      cmd.gitc_manifest = GitcClient(cmd.repodir, gitc_client_name)
-      cmd.client.isGitcClient = True
-
     Editor.globalConfig = cmd.client.globalConfig
 
     if not isinstance(cmd, MirrorSafeCommand) and cmd.manifest.IsMirror:
@@ -254,8 +257,10 @@ class _Repo(object):
     cmd_event = cmd.event_log.Add(name, event_log.TASK_COMMAND, start)
     cmd.event_log.SetParent(cmd_event)
     git_trace2_event_log.StartEvent()
+    git_trace2_event_log.CommandEvent(name='repo', subcommands=[name])
 
     try:
+      cmd.CommonValidateOptions(copts, cargs)
       cmd.ValidateOptions(copts, cargs)
       result = cmd.Execute(copts, cargs)
     except (DownloadError, ManifestInvalidRevisionError,
@@ -297,6 +302,8 @@ class _Repo(object):
 
       cmd.event_log.FinishEvent(cmd_event, finish,
                                 result is None or result == 0)
+      git_trace2_event_log.DefParamRepoEvents(
+          cmd.manifest.manifestProject.config.DumpConfigDict())
       git_trace2_event_log.ExitEvent(result)
 
       if gopts.event_log:
@@ -587,20 +594,16 @@ def _Main(argv):
 
   repo = _Repo(opt.repodir)
   try:
-    try:
-      init_ssh()
-      init_http()
-      name, gopts, argv = repo._ParseArgs(argv)
-      run = lambda: repo._Run(name, gopts, argv) or 0
-      if gopts.trace_python:
-        import trace
-        tracer = trace.Trace(count=False, trace=True, timing=True,
-                             ignoredirs=set(sys.path[1:]))
-        result = tracer.runfunc(run)
-      else:
-        result = run()
-    finally:
-      close_ssh()
+    init_http()
+    name, gopts, argv = repo._ParseArgs(argv)
+    run = lambda: repo._Run(name, gopts, argv) or 0
+    if gopts.trace_python:
+      import trace
+      tracer = trace.Trace(count=False, trace=True, timing=True,
+                           ignoredirs=set(sys.path[1:]))
+      result = tracer.runfunc(run)
+    else:
+      result = run()
   except KeyboardInterrupt:
     print('aborted by user', file=sys.stderr)
     result = 1

manifest_xml.py

@@ -12,8 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
 import itertools
 import os
+import platform
 import re
 import sys
 import xml.dom.minidom
@@ -26,11 +28,18 @@ import platform_utils
 from project import RemoteSpec, Project, MetaProject
 from error import (ManifestParseError, ManifestInvalidPathError,
                    ManifestInvalidRevisionError)
+from wrapper import Wrapper
 
 MANIFEST_FILE_NAME = 'manifest.xml'
 LOCAL_MANIFEST_NAME = 'local_manifest.xml'
 LOCAL_MANIFESTS_DIR_NAME = 'local_manifests'
 
+# Add all projects from local manifest into a group.
+LOCAL_MANIFEST_GROUP_PREFIX = 'local:'
+
+# ContactInfo has the self-registered bug url, supplied by the manifest authors.
+ContactInfo = collections.namedtuple('ContactInfo', 'bugurl')
+
 # urljoin gets confused if the scheme is not known.
 urllib.parse.uses_relative.extend([
     'ssh',
@@ -113,9 +122,13 @@ class _Default(object):
   sync_tags = True
 
   def __eq__(self, other):
+    if not isinstance(other, _Default):
+      return False
     return self.__dict__ == other.__dict__
 
   def __ne__(self, other):
+    if not isinstance(other, _Default):
+      return True
     return self.__dict__ != other.__dict__
 
 
@@ -138,12 +151,18 @@ class _XmlRemote(object):
     self.resolvedFetchUrl = self._resolveFetchUrl()
 
   def __eq__(self, other):
+    if not isinstance(other, _XmlRemote):
+      return False
     return self.__dict__ == other.__dict__
 
   def __ne__(self, other):
+    if not isinstance(other, _XmlRemote):
+      return True
     return self.__dict__ != other.__dict__
 
   def _resolveFetchUrl(self):
+    if self.fetchUrl is None:
+      return ''
     url = self.fetchUrl.rstrip('/')
     manifestUrl = self.manifestUrl.rstrip('/')
     # urljoin will gets confused over quite a few things.  The ones we care
@@ -478,6 +497,12 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         e.setAttribute('remote', remoteName)
       root.appendChild(e)
 
+    if self._contactinfo.bugurl != Wrapper().BUG_URL:
+      root.appendChild(doc.createTextNode(''))
+      e = doc.createElement('contactinfo')
+      e.setAttribute('bugurl', self._contactinfo.bugurl)
+      root.appendChild(e)
+
     return doc
 
   def ToDict(self, **kwargs):
@@ -489,6 +514,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         'manifest-server',
         'repo-hooks',
         'superproject',
+        'contactinfo',
     }
     # Elements that may be repeated.
     MULTI_ELEMENTS = {
@@ -564,6 +590,11 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     self._Load()
     return self._superproject
 
+  @property
+  def contactinfo(self):
+    self._Load()
+    return self._contactinfo
+
   @property
   def notice(self):
     self._Load()
@@ -588,6 +619,16 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
       return self.manifestProject.config.GetString('repo.clonefilter')
     return None
 
+  @property
+  def PartialCloneExclude(self):
+    exclude = self.manifest.manifestProject.config.GetString(
+        'repo.partialcloneexclude') or ''
+    return set(x.strip() for x in exclude.split(','))
+
+  @property
+  def HasLocalManifests(self):
+    return self._load_local_manifests and self.local_manifests
+
   @property
   def IsMirror(self):
     return self.manifestProject.config.GetBoolean('repo.mirror')
@@ -604,6 +645,17 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
   def HasSubmodules(self):
     return self.manifestProject.config.GetBoolean('repo.submodules')
 
+  def GetDefaultGroupsStr(self):
+    """Returns the default group string for the platform."""
+    return 'default,platform-' + platform.system().lower()
+
+  def GetGroupsStr(self):
+    """Returns the manifest group string that should be synced."""
+    groups = self.manifestProject.config.GetString('manifest.groups')
+    if not groups:
+      groups = self.GetDefaultGroupsStr()
+    return groups
+
   def _Unload(self):
     self._loaded = False
     self._projects = {}
@@ -612,6 +664,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     self._default = None
     self._repo_hooks_project = None
     self._superproject = {}
+    self._contactinfo = ContactInfo(Wrapper().BUG_URL)
     self._notice = None
     self.branch = None
     self._manifest_server = None
@@ -639,7 +692,9 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
               # Since local manifests are entirely managed by the user, allow
               # them to point anywhere the user wants.
               nodes.append(self._ParseManifestXml(
-                  local, self.repodir, restrict_includes=False))
+                  local, self.repodir,
+                  parent_groups=f'{LOCAL_MANIFEST_GROUP_PREFIX}:{local_file[:-4]}',
+                  restrict_includes=False))
         except OSError:
           pass
 
@@ -736,9 +791,10 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     for node in itertools.chain(*node_list):
       if node.nodeName == 'default':
         new_default = self._ParseDefault(node)
+        emptyDefault = not node.hasAttributes() and not node.hasChildNodes()
         if self._default is None:
           self._default = new_default
-        elif new_default != self._default:
+        elif not emptyDefault and new_default != self._default:
           raise ManifestParseError('duplicate default in %s' %
                                    (self.manifestFile))
 
@@ -854,22 +910,27 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
           raise ManifestParseError("no remote for superproject %s within %s" %
                                    (name, self.manifestFile))
         self._superproject['remote'] = remote.ToRemoteSpec(name)
+      if node.nodeName == 'contactinfo':
+        bugurl = self._reqatt(node, 'bugurl')
+        # This element can be repeated, later entries will clobber earlier ones.
+        self._contactinfo = ContactInfo(bugurl)
+
       if node.nodeName == 'remove-project':
         name = self._reqatt(node, 'name')
 
-        if name not in self._projects:
+        if name in self._projects:
+          for p in self._projects[name]:
+            del self._paths[p.relpath]
+          del self._projects[name]
+
+          # If the manifest removes the hooks project, treat it as if it deleted
+          # the repo-hooks element too.
+          if self._repo_hooks_project and (self._repo_hooks_project.name == name):
+            self._repo_hooks_project = None
+        elif not XmlBool(node, 'optional', False):
           raise ManifestParseError('remove-project element specifies non-existent '
                                    'project: %s' % name)
 
-        for p in self._projects[name]:
-          del self._paths[p.relpath]
-        del self._projects[name]
-
-        # If the manifest removes the hooks project, treat it as if it deleted
-        # the repo-hooks element too.
-        if self._repo_hooks_project and (self._repo_hooks_project.name == name):
-          self._repo_hooks_project = None
-
   def _AddMetaProjectMirror(self, m):
     name = None
     m_url = m.GetRemote(m.remote.name).url
@@ -1027,7 +1088,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     if not path:
       path = name
     else:
-      msg = self._CheckLocalPath(path, dir_ok=True)
+      # NB: The "." project is handled specially in Project.Sync_LocalHalf.
+      msg = self._CheckLocalPath(path, dir_ok=True, cwd_dot_ok=True)
       if msg:
         raise ManifestInvalidPathError(
             '<project> invalid "path": %s: %s' % (path, msg))
@@ -1180,6 +1242,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     if '~' in path:
       return '~ not allowed (due to 8.3 filenames on Windows filesystems)'
 
+    path_codepoints = set(path)
+
     # Some filesystems (like Apple's HFS+) try to normalize Unicode codepoints
     # which means there are alternative names for ".git".  Reject paths with
     # these in it as there shouldn't be any reasonable need for them here.
@@ -1203,10 +1267,17 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         u'\u206F',  # NOMINAL DIGIT SHAPES
         u'\uFEFF',  # ZERO WIDTH NO-BREAK SPACE
     }
-    if BAD_CODEPOINTS & set(path):
+    if BAD_CODEPOINTS & path_codepoints:
       # This message is more expansive than reality, but should be fine.
       return 'Unicode combining characters not allowed'
 
+    # Reject newlines as there shouldn't be any legitmate use for them, they'll
+    # be confusing to users, and they can easily break tools that expect to be
+    # able to iterate over newline delimited lists.  This even applies to our
+    # own code like .repo/project.list.
+    if {'\r', '\n'} & path_codepoints:
+      return 'Newlines not allowed'
+
     # Assume paths might be used on case-insensitive filesystems.
     path = path.lower()
 
@@ -1215,7 +1286,9 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     # our constructed logic here.  Especially since manifest authors only use
     # / in their paths.
     resep = re.compile(r'[/%s]' % re.escape(os.path.sep))
-    parts = resep.split(path)
+    # Strip off trailing slashes as those only produce '' elements, and we use
+    # parts to look for individual bad components.
+    parts = resep.split(path.rstrip('/'))
 
     # Some people use src="." to create stable links to projects.  Lets allow
     # that but reject all other uses of "." to keep things simple.

progress.py

@@ -42,18 +42,26 @@ def duration_str(total):
 
 
 class Progress(object):
-  def __init__(self, title, total=0, units='', print_newline=False):
+  def __init__(self, title, total=0, units='', print_newline=False, delay=True,
+               quiet=False):
     self._title = title
     self._total = total
     self._done = 0
     self._start = time()
-    self._show = False
+    self._show = not delay
     self._units = units
     self._print_newline = print_newline
     # Only show the active jobs section if we run more than one in parallel.
     self._show_jobs = False
     self._active = 0
 
+    # When quiet, never show any output.  It's a bit hacky, but reusing the
+    # existing logic that delays initial output keeps the rest of the class
+    # clean.  Basically we set the start time to years in the future.
+    if quiet:
+      self._show = False
+      self._start += 2**32
+
   def start(self, name):
     self._active += 1
     if not self._show_jobs:

project.py

@@ -863,7 +863,7 @@ class Project(object):
       out.nl()
       out.project('project %s/' % self.relpath)
       out.nl()
-      out.write(p.stdout)
+      out.write('%s', p.stdout)
     return p.Wait() == 0
 
 # Publish / Upload ##
@@ -1041,16 +1041,18 @@ class Project(object):
                        verbose=False,
                        output_redir=None,
                        is_new=None,
-                       current_branch_only=False,
+                       current_branch_only=None,
                        force_sync=False,
                        clone_bundle=True,
-                       tags=True,
+                       tags=None,
                        archive=False,
                        optimized_fetch=False,
                        retry_fetches=0,
                        prune=False,
                        submodules=False,
-                       clone_filter=None):
+                       ssh_proxy=None,
+                       clone_filter=None,
+                       partial_clone_exclude=set()):
     """Perform only the network IO portion of the sync process.
        Local working directory/branch state is not affected.
     """
@@ -1087,6 +1089,10 @@ class Project(object):
     if clone_bundle and os.path.exists(self.objdir):
       clone_bundle = False
 
+    if self.name in partial_clone_exclude:
+      clone_bundle = True
+      clone_filter = None
+
     if is_new is None:
       is_new = not self.Exists
     if is_new:
@@ -1111,7 +1117,7 @@ class Project(object):
             and self._ApplyCloneBundle(initial=is_new, quiet=quiet, verbose=verbose)):
       is_new = False
 
-    if not current_branch_only:
+    if current_branch_only is None:
       if self.sync_c:
         current_branch_only = True
       elif not self.manifest._loaded:
@@ -1120,8 +1126,8 @@ class Project(object):
       elif self.manifest.default.sync_c:
         current_branch_only = True
 
-    if not self.sync_tags:
-      tags = False
+    if tags is None:
+      tags = self.sync_tags
 
     if self.clone_depth:
       depth = self.clone_depth
@@ -1138,6 +1144,7 @@ class Project(object):
               alt_dir=alt_dir, current_branch_only=current_branch_only,
               tags=tags, prune=prune, depth=depth,
               submodules=submodules, force_sync=force_sync,
+              ssh_proxy=ssh_proxy,
               clone_filter=clone_filter, retry_fetches=retry_fetches):
         return False
 
@@ -1209,6 +1216,9 @@ class Project(object):
                                          (self.revisionExpr, self.name))
 
   def SetRevisionId(self, revisionId):
+    if self.revisionExpr:
+      self.upstream = self.revisionExpr
+
     self.revisionId = revisionId
 
   def Sync_LocalHalf(self, syncbuf, force_sync=False, submodules=False):
@@ -1227,6 +1237,18 @@ class Project(object):
     self.CleanPublishedCache(all_refs)
     revid = self.GetRevisionId(all_refs)
 
+    # Special case the root of the repo client checkout.  Make sure it doesn't
+    # contain files being checked out to dirs we don't allow.
+    if self.relpath == '.':
+      PROTECTED_PATHS = {'.repo'}
+      paths = set(self.work_git.ls_tree('-z', '--name-only', '--', revid).split('\0'))
+      bad_paths = paths & PROTECTED_PATHS
+      if bad_paths:
+        syncbuf.fail(self,
+                     'Refusing to checkout project that writes to protected '
+                     'paths: %s' % (', '.join(bad_paths),))
+        return
+
     def _doff():
       self._FastForward(revid)
       self._CopyAndLinkFiles()
@@ -1698,6 +1720,11 @@ class Project(object):
         if cb is None or name != cb:
           kill.append(name)
 
+    # Minor optimization: If there's nothing to prune, then don't try to read
+    # any project state.
+    if not kill and not cb:
+      return []
+
     rev = self.GetRevisionId(left)
     if cb is not None \
        and not self._revlist(HEAD + '...' + rev) \
@@ -1940,6 +1967,10 @@ class Project(object):
       # throws an error.
       self.bare_git.rev_list('-1', '--missing=allow-any',
                              '%s^0' % self.revisionExpr, '--')
+      if self.upstream:
+        rev = self.GetRemote(self.remote.name).ToLocal(self.upstream)
+        self.bare_git.rev_list('-1', '--missing=allow-any',
+                               '%s^0' % rev, '--')
       return True
     except GitError:
       # There is no such persistent revision. We have to fetch it.
@@ -1969,6 +2000,7 @@ class Project(object):
                    prune=False,
                    depth=None,
                    submodules=False,
+                   ssh_proxy=None,
                    force_sync=False,
                    clone_filter=None,
                    retry_fetches=2,
@@ -2016,16 +2048,14 @@ class Project(object):
     if not name:
       name = self.remote.name
 
-    ssh_proxy = False
     remote = self.GetRemote(name)
-    if remote.PreConnectFetch():
-      ssh_proxy = True
+    if not remote.PreConnectFetch(ssh_proxy):
+      ssh_proxy = None
 
     if initial:
       if alt_dir and 'objects' == os.path.basename(alt_dir):
         ref_dir = os.path.dirname(alt_dir)
         packed_refs = os.path.join(self.gitdir, 'packed-refs')
-        remote = self.GetRemote(name)
 
         all_refs = self.bare_ref.all
         ids = set(all_refs.values())
@@ -2112,6 +2142,8 @@ class Project(object):
       # Shallow checkout of a specific commit, fetch from that commit and not
       # the heads only as the commit might be deeper in the history.
       spec.append(branch)
+      if self.upstream:
+        spec.append(self.upstream)
     else:
       if is_sha1:
         branch = self.upstream
@@ -2153,19 +2185,13 @@ class Project(object):
       elif (gitcmd.stdout and
             'error:' in gitcmd.stdout and
             'HTTP 429' in gitcmd.stdout):
-        if not quiet:
-          print('429 received, sleeping: %s sec' % retry_cur_sleep,
-                file=sys.stderr)
-        time.sleep(retry_cur_sleep)
-        retry_cur_sleep = min(retry_exp_factor * retry_cur_sleep,
-                              MAXIMUM_RETRY_SLEEP_SEC)
-        retry_cur_sleep *= (1 - random.uniform(-RETRY_JITTER_PERCENT,
-                                               RETRY_JITTER_PERCENT))
-        continue
+        # Fallthru to sleep+retry logic at the bottom.
+        pass
 
-      # If this is not last attempt, try 'git remote prune'.
-      elif (try_n < retry_fetches - 1 and
-            gitcmd.stdout and
+      # Try to prune remote branches once in case there are conflicts.
+      # For example, if the remote had refs/heads/upstream, but deleted that and
+      # now has refs/heads/upstream/foo.
+      elif (gitcmd.stdout and
             'error:' in gitcmd.stdout and
             'git remote prune' in gitcmd.stdout and
             not prune_tried):
@@ -2175,6 +2201,8 @@ class Project(object):
         ret = prunecmd.Wait()
         if ret:
           break
+        print('retrying fetch after pruning remote branches', file=output_redir)
+        # Continue right away so we don't sleep as we shouldn't need to.
         continue
       elif current_branch_only and is_sha1 and ret == 128:
         # Exit code 128 means "couldn't find the ref you asked for"; if we're
@@ -2184,9 +2212,18 @@ class Project(object):
       elif ret < 0:
         # Git died with a signal, exit immediately
         break
-      if not verbose:
-        print('\n%s:\n%s' % (self.name, gitcmd.stdout), file=sys.stderr)
-      time.sleep(random.randint(30, 45))
+
+      # Figure out how long to sleep before the next attempt, if there is one.
+      if not verbose and gitcmd.stdout:
+        print('\n%s:\n%s' % (self.name, gitcmd.stdout), end='', file=output_redir)
+      if try_n < retry_fetches - 1:
+        print('%s: sleeping %s seconds before retrying' % (self.name, retry_cur_sleep),
+              file=output_redir)
+        time.sleep(retry_cur_sleep)
+        retry_cur_sleep = min(retry_exp_factor * retry_cur_sleep,
+                              MAXIMUM_RETRY_SLEEP_SEC)
+        retry_cur_sleep *= (1 - random.uniform(-RETRY_JITTER_PERCENT,
+                                               RETRY_JITTER_PERCENT))
 
     if initial:
       if alt_dir:
@@ -2207,7 +2244,7 @@ class Project(object):
             name=name, quiet=quiet, verbose=verbose, output_redir=output_redir,
             current_branch_only=current_branch_only and depth,
             initial=False, alt_dir=alt_dir,
-            depth=None, clone_filter=clone_filter)
+            depth=None, ssh_proxy=ssh_proxy, clone_filter=clone_filter)
 
     return ok
 
@@ -2412,14 +2449,6 @@ class Project(object):
         self.bare_objdir.init()
 
         if self.use_git_worktrees:
-          # Set up the m/ space to point to the worktree-specific ref space.
-          # We'll update the worktree-specific ref space on each checkout.
-          if self.manifest.branch:
-            self.bare_git.symbolic_ref(
-                '-m', 'redirecting to worktree scope',
-                R_M + self.manifest.branch,
-                R_WORKTREE_M + self.manifest.branch)
-
           # Enable per-worktree config file support if possible.  This is more a
           # nice-to-have feature for users rather than a hard requirement.
           if git_require((2, 20, 0)):
@@ -2556,6 +2585,14 @@ class Project(object):
   def _InitMRef(self):
     if self.manifest.branch:
       if self.use_git_worktrees:
+        # Set up the m/ space to point to the worktree-specific ref space.
+        # We'll update the worktree-specific ref space on each checkout.
+        ref = R_M + self.manifest.branch
+        if not self.bare_ref.symref(ref):
+          self.bare_git.symbolic_ref(
+              '-m', 'redirecting to worktree scope',
+              ref, R_WORKTREE_M + self.manifest.branch)
+
         # We can't update this ref with git worktrees until it exists.
         # We'll wait until the initial checkout to set it.
         if not os.path.exists(self.worktree):

repo

@@ -117,7 +117,7 @@ def check_python_version():
 
     # If the python3 version looks like it's new enough, give it a try.
     if (python3_ver and python3_ver >= MIN_PYTHON_VERSION_HARD
-        and python3_ver != (major, minor)):
+            and python3_ver != (major, minor)):
       reexec('python3')
 
     # We're still here, so diagnose things for the user.
@@ -145,9 +145,11 @@ if not REPO_URL:
 REPO_REV = os.environ.get('REPO_REV')
 if not REPO_REV:
   REPO_REV = 'stable'
+# URL to file bug reports for repo tool issues.
+BUG_URL = 'https://bugs.chromium.org/p/gerrit/issues/entry?template=Repo+tool+issue'
 
 # increment this whenever we make important changes to this script
-VERSION = (2, 12)
+VERSION = (2, 15)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (2, 3)
@@ -275,6 +277,13 @@ def GetParser(gitc_init=False):
     usage = 'repo init [options] [-u] url'
 
   parser = optparse.OptionParser(usage=usage)
+  InitParser(parser, gitc_init=gitc_init)
+  return parser
+
+
+def InitParser(parser, gitc_init=False):
+  """Setup the CLI parser."""
+  # NB: Keep in sync with command.py:_CommonOptions().
 
   # Logging.
   group = parser.add_option_group('Logging options')
@@ -291,8 +300,22 @@ def GetParser(gitc_init=False):
                    help='manifest repository location', metavar='URL')
   group.add_option('-b', '--manifest-branch', metavar='REVISION',
                    help='manifest branch or revision (use HEAD for default)')
-  group.add_option('-m', '--manifest-name',
+  group.add_option('-m', '--manifest-name', default='default.xml',
                    help='initial manifest file', metavar='NAME.xml')
+  group.add_option('-g', '--groups', default='default',
+                   help='restrict manifest projects to ones with specified '
+                        'group(s) [default|all|G1,G2,G3|G4,-G5,-G6]',
+                   metavar='GROUP')
+  group.add_option('-p', '--platform', default='auto',
+                   help='restrict manifest projects to ones with a specified '
+                        'platform group [auto|all|none|linux|darwin|...]',
+                   metavar='PLATFORM')
+  group.add_option('--submodules', action='store_true',
+                   help='sync any submodules associated with the manifest repo')
+
+  # Options that only affect manifest project, and not any of the projects
+  # specified in the manifest itself.
+  group = parser.add_option_group('Manifest (only) checkout options')
   cbr_opts = ['--current-branch']
   # The gitc-init subcommand allocates -c itself, but a lot of init users
   # want -c, so try to satisfy both as best we can.
@@ -301,9 +324,29 @@ def GetParser(gitc_init=False):
   group.add_option(*cbr_opts,
                    dest='current_branch_only', action='store_true',
                    help='fetch only current manifest branch from server')
+  group.add_option('--no-current-branch',
+                   dest='current_branch_only', action='store_false',
+                   help='fetch all manifest branches from server')
+  group.add_option('--tags',
+                   action='store_true',
+                   help='fetch tags in the manifest')
+  group.add_option('--no-tags',
+                   dest='tags', action='store_false',
+                   help="don't fetch tags in the manifest")
+
+  # These are fundamentally different ways of structuring the checkout.
+  group = parser.add_option_group('Checkout modes')
   group.add_option('--mirror', action='store_true',
                    help='create a replica of the remote repositories '
                         'rather than a client working directory')
+  group.add_option('--archive', action='store_true',
+                   help='checkout an archive instead of a git repository for '
+                        'each project. See git archive.')
+  group.add_option('--worktree', action='store_true',
+                   help='use git-worktree to manage projects')
+
+  # These are fundamentally different ways of structuring the checkout.
+  group = parser.add_option_group('Project checkout optimizations')
   group.add_option('--reference',
                    help='location of mirror directory', metavar='DIR')
   group.add_option('--dissociate', action='store_true',
@@ -314,38 +357,27 @@ def GetParser(gitc_init=False):
   group.add_option('--partial-clone', action='store_true',
                    help='perform partial clone (https://git-scm.com/'
                         'docs/gitrepository-layout#_code_partialclone_code)')
+  group.add_option('--no-partial-clone', action='store_false',
+                   help='disable use of partial clone (https://git-scm.com/'
+                        'docs/gitrepository-layout#_code_partialclone_code)')
+  group.add_option('--partial-clone-exclude', action='store',
+                   help='exclude the specified projects (a comma-delimited '
+                        'project names) from partial clone (https://git-scm.com'
+                        '/docs/gitrepository-layout#_code_partialclone_code)')
   group.add_option('--clone-filter', action='store', default='blob:none',
                    help='filter for use with --partial-clone '
                         '[default: %default]')
-  group.add_option('--worktree', action='store_true',
-                   help=optparse.SUPPRESS_HELP)
-  group.add_option('--archive', action='store_true',
-                   help='checkout an archive instead of a git repository for '
-                        'each project. See git archive.')
-  group.add_option('--submodules', action='store_true',
-                   help='sync any submodules associated with the manifest repo')
   group.add_option('--use-superproject', action='store_true', default=None,
                    help='use the manifest superproject to sync projects')
   group.add_option('--no-use-superproject', action='store_false',
                    dest='use_superproject',
                    help='disable use of manifest superprojects')
-  group.add_option('-g', '--groups', default='default',
-                   help='restrict manifest projects to ones with specified '
-                        'group(s) [default|all|G1,G2,G3|G4,-G5,-G6]',
-                   metavar='GROUP')
-  group.add_option('-p', '--platform', default='auto',
-                   help='restrict manifest projects to ones with a specified '
-                        'platform group [auto|all|none|linux|darwin|...]',
-                   metavar='PLATFORM')
   group.add_option('--clone-bundle', action='store_true',
                    help='enable use of /clone.bundle on HTTP/HTTPS '
                         '(default if not --partial-clone)')
   group.add_option('--no-clone-bundle',
                    dest='clone_bundle', action='store_false',
                    help='disable use of /clone.bundle on HTTP/HTTPS (default if --partial-clone)')
-  group.add_option('--no-tags',
-                   dest='tags', default=True, action='store_false',
-                   help="don't fetch tags in the manifest")
 
   # Tool.
   group = parser.add_option_group('repo Version options')
@@ -827,11 +859,10 @@ def _DownloadBundle(url, cwd, quiet, verbose):
     try:
       r = urllib.request.urlopen(url)
     except urllib.error.HTTPError as e:
-      if e.code in [401, 403, 404, 501]:
-        return False
-      print('fatal: Cannot get %s' % url, file=sys.stderr)
-      print('fatal: HTTP error %s' % e.code, file=sys.stderr)
-      raise CloneFailure()
+      if e.code not in [400, 401, 403, 404, 501]:
+        print('warning: Cannot get %s' % url, file=sys.stderr)
+        print('warning: HTTP error %s' % e.code, file=sys.stderr)
+      return False
     except urllib.error.URLError as e:
       print('fatal: Cannot get %s' % url, file=sys.stderr)
       print('fatal: error %s' % e.reason, file=sys.stderr)
@@ -1147,6 +1178,7 @@ The most commonly used repo commands are:
 
 For access to the full online help, install repo ("repo init").
 """)
+  print('Bug reports:', BUG_URL)
   sys.exit(0)
 
 
@@ -1180,6 +1212,7 @@ def _Version():
     print('OS %s %s (%s)' % (uname.system, uname.release, uname.version))
     print('CPU %s (%s)' %
           (uname.machine, uname.processor if uname.processor else 'unknown'))
+  print('Bug reports:', BUG_URL)
   sys.exit(0)
 
 

requirements.json

@@ -38,9 +38,9 @@
   # Supported Python versions.
   #
   # python-3.6 is in Ubuntu Bionic.
-  # python-3.5 is in Debian Stretch.
+  # python-3.7 is in Debian Buster.
   "python": {
-    "hard": [3, 5],
+    "hard": [3, 6],
     "soft": [3, 6]
   },
 

run_tests

@@ -24,6 +24,10 @@ import sys
 
 def find_pytest():
   """Try to locate a good version of pytest."""
+  # If we're in a virtualenv, assume that it's provided the right pytest.
+  if 'VIRTUAL_ENV' in os.environ:
+    return 'pytest'
+
   # Use the Python 3 version if available.
   ret = shutil.which('pytest-3')
   if ret:
@@ -34,8 +38,8 @@ def find_pytest():
   if ret:
     return ret
 
-  print(f'{__file__}: unable to find pytest.', file=sys.stderr)
-  print(f'{__file__}: Try installing: sudo apt-get install python-pytest',
+  print('%s: unable to find pytest.' % (__file__,), file=sys.stderr)
+  print('%s: Try installing: sudo apt-get install python-pytest' % (__file__,),
         file=sys.stderr)
 
 
@@ -50,7 +54,7 @@ def main(argv):
   os.environ['PYTHONPATH'] = pythonpath
 
   pytest = find_pytest()
-  return subprocess.run([pytest] + argv, check=True)
+  return subprocess.run([pytest] + argv, check=False).returncode
 
 
 if __name__ == '__main__':

setup.py

@@ -32,7 +32,7 @@ with open(os.path.join(TOPDIR, 'README.md')) as fp:
 # https://packaging.python.org/tutorials/packaging-projects/
 setuptools.setup(
     name='repo',
-    version='1.13.8',
+    version='2',
     maintainer='Various',
     maintainer_email='repo-discuss@googlegroups.com',
     description='Repo helps manage many Git repositories',

ssh.py

@@ -0,0 +1,277 @@
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Common SSH management logic."""
+
+import functools
+import multiprocessing
+import os
+import re
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+
+import platform_utils
+from repo_trace import Trace
+
+
+PROXY_PATH = os.path.join(os.path.dirname(__file__), 'git_ssh')
+
+
+def _run_ssh_version():
+  """run ssh -V to display the version number"""
+  return subprocess.check_output(['ssh', '-V'], stderr=subprocess.STDOUT).decode()
+
+
+def _parse_ssh_version(ver_str=None):
+  """parse a ssh version string into a tuple"""
+  if ver_str is None:
+    ver_str = _run_ssh_version()
+  m = re.match(r'^OpenSSH_([0-9.]+)(p[0-9]+)?\s', ver_str)
+  if m:
+    return tuple(int(x) for x in m.group(1).split('.'))
+  else:
+    return ()
+
+
+@functools.lru_cache(maxsize=None)
+def version():
+  """return ssh version as a tuple"""
+  try:
+    return _parse_ssh_version()
+  except subprocess.CalledProcessError:
+    print('fatal: unable to detect ssh version', file=sys.stderr)
+    sys.exit(1)
+
+
+URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
+URI_ALL = re.compile(r'^([a-z][a-z+-]*)://([^@/]*@?[^/]*)/')
+
+
+class ProxyManager:
+  """Manage various ssh clients & masters that we spawn.
+
+  This will take care of sharing state between multiprocessing children, and
+  make sure that if we crash, we don't leak any of the ssh sessions.
+
+  The code should work with a single-process scenario too, and not add too much
+  overhead due to the manager.
+  """
+
+  # Path to the ssh program to run which will pass our master settings along.
+  # Set here more as a convenience API.
+  proxy = PROXY_PATH
+
+  def __init__(self, manager):
+    # Protect access to the list of active masters.
+    self._lock = multiprocessing.Lock()
+    # List of active masters (pid).  These will be spawned on demand, and we are
+    # responsible for shutting them all down at the end.
+    self._masters = manager.list()
+    # Set of active masters indexed by "host:port" information.
+    # The value isn't used, but multiprocessing doesn't provide a set class.
+    self._master_keys = manager.dict()
+    # Whether ssh masters are known to be broken, so we give up entirely.
+    self._master_broken = manager.Value('b', False)
+    # List of active ssh sesssions.  Clients will be added & removed as
+    # connections finish, so this list is just for safety & cleanup if we crash.
+    self._clients = manager.list()
+    # Path to directory for holding master sockets.
+    self._sock_path = None
+
+  def __enter__(self):
+    """Enter a new context."""
+    return self
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    """Exit a context & clean up all resources."""
+    self.close()
+
+  def add_client(self, proc):
+    """Track a new ssh session."""
+    self._clients.append(proc.pid)
+
+  def remove_client(self, proc):
+    """Remove a completed ssh session."""
+    try:
+      self._clients.remove(proc.pid)
+    except ValueError:
+      pass
+
+  def add_master(self, proc):
+    """Track a new master connection."""
+    self._masters.append(proc.pid)
+
+  def _terminate(self, procs):
+    """Kill all |procs|."""
+    for pid in procs:
+      try:
+        os.kill(pid, signal.SIGTERM)
+        os.waitpid(pid, 0)
+      except OSError:
+        pass
+
+    # The multiprocessing.list() API doesn't provide many standard list()
+    # methods, so we have to manually clear the list.
+    while True:
+      try:
+        procs.pop(0)
+      except:
+        break
+
+  def close(self):
+    """Close this active ssh session.
+
+    Kill all ssh clients & masters we created, and nuke the socket dir.
+    """
+    self._terminate(self._clients)
+    self._terminate(self._masters)
+
+    d = self.sock(create=False)
+    if d:
+      try:
+        platform_utils.rmdir(os.path.dirname(d))
+      except OSError:
+        pass
+
+  def _open_unlocked(self, host, port=None):
+    """Make sure a ssh master session exists for |host| & |port|.
+
+    If one doesn't exist already, we'll create it.
+
+    We won't grab any locks, so the caller has to do that.  This helps keep the
+    business logic of actually creating the master separate from grabbing locks.
+    """
+    # Check to see whether we already think that the master is running; if we
+    # think it's already running, return right away.
+    if port is not None:
+      key = '%s:%s' % (host, port)
+    else:
+      key = host
+
+    if key in self._master_keys:
+      return True
+
+    if self._master_broken.value or 'GIT_SSH' in os.environ:
+      # Failed earlier, so don't retry.
+      return False
+
+    # We will make two calls to ssh; this is the common part of both calls.
+    command_base = ['ssh', '-o', 'ControlPath %s' % self.sock(), host]
+    if port is not None:
+      command_base[1:1] = ['-p', str(port)]
+
+    # Since the key wasn't in _master_keys, we think that master isn't running.
+    # ...but before actually starting a master, we'll double-check.  This can
+    # be important because we can't tell that that 'git@myhost.com' is the same
+    # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file.
+    check_command = command_base + ['-O', 'check']
+    try:
+      Trace(': %s', ' '.join(check_command))
+      check_process = subprocess.Popen(check_command,
+                                       stdout=subprocess.PIPE,
+                                       stderr=subprocess.PIPE)
+      check_process.communicate()  # read output, but ignore it...
+      isnt_running = check_process.wait()
+
+      if not isnt_running:
+        # Our double-check found that the master _was_ infact running.  Add to
+        # the list of keys.
+        self._master_keys[key] = True
+        return True
+    except Exception:
+      # Ignore excpetions.  We we will fall back to the normal command and print
+      # to the log there.
+      pass
+
+    command = command_base[:1] + ['-M', '-N'] + command_base[1:]
+    try:
+      Trace(': %s', ' '.join(command))
+      p = subprocess.Popen(command)
+    except Exception as e:
+      self._master_broken.value = True
+      print('\nwarn: cannot enable ssh control master for %s:%s\n%s'
+            % (host, port, str(e)), file=sys.stderr)
+      return False
+
+    time.sleep(1)
+    ssh_died = (p.poll() is not None)
+    if ssh_died:
+      return False
+
+    self.add_master(p)
+    self._master_keys[key] = True
+    return True
+
+  def _open(self, host, port=None):
+    """Make sure a ssh master session exists for |host| & |port|.
+
+    If one doesn't exist already, we'll create it.
+
+    This will obtain any necessary locks to avoid inter-process races.
+    """
+    # Bail before grabbing the lock if we already know that we aren't going to
+    # try creating new masters below.
+    if sys.platform in ('win32', 'cygwin'):
+      return False
+
+    # Acquire the lock.  This is needed to prevent opening multiple masters for
+    # the same host when we're running "repo sync -jN" (for N > 1) _and_ the
+    # manifest <remote fetch="ssh://xyz"> specifies a different host from the
+    # one that was passed to repo init.
+    with self._lock:
+      return self._open_unlocked(host, port)
+
+  def preconnect(self, url):
+    """If |uri| will create a ssh connection, setup the ssh master for it."""
+    m = URI_ALL.match(url)
+    if m:
+      scheme = m.group(1)
+      host = m.group(2)
+      if ':' in host:
+        host, port = host.split(':')
+      else:
+        port = None
+      if scheme in ('ssh', 'git+ssh', 'ssh+git'):
+        return self._open(host, port)
+      return False
+
+    m = URI_SCP.match(url)
+    if m:
+      host = m.group(1)
+      return self._open(host)
+
+    return False
+
+  def sock(self, create=True):
+    """Return the path to the ssh socket dir.
+
+    This has all the master sockets so clients can talk to them.
+    """
+    if self._sock_path is None:
+      if not create:
+        return None
+      tmp_dir = '/tmp'
+      if not os.path.exists(tmp_dir):
+        tmp_dir = tempfile.gettempdir()
+      if version() < (6, 7):
+        tokens = '%r@%h:%p'
+      else:
+        tokens = '%C'  # hash of %l%h%p%r
+      self._sock_path = os.path.join(
+          tempfile.mkdtemp('', 'ssh-', tmp_dir),
+          'master-' + tokens)
+    return self._sock_path

subcmds/abandon.py

@@ -15,16 +15,15 @@
 from collections import defaultdict
 import functools
 import itertools
-import multiprocessing
 import sys
 
-from command import Command, DEFAULT_LOCAL_JOBS, WORKER_BATCH_SIZE
+from command import Command, DEFAULT_LOCAL_JOBS
 from git_command import git
 from progress import Progress
 
 
 class Abandon(Command):
-  common = True
+  COMMON = True
   helpSummary = "Permanently abandon a development branch"
   helpUsage = """
 %prog [--all | <branchname>] [<project>...]
@@ -37,10 +36,6 @@ It is equivalent to "git branch -D <branchname>".
   PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
   def _Options(self, p):
-    super()._Options(p)
-    p.add_option('-q', '--quiet',
-                 action='store_true', default=False,
-                 help='be quiet')
     p.add_option('--all',
                  dest='all', action='store_true',
                  help='delete all branches in all projects')
@@ -56,9 +51,9 @@ It is equivalent to "git branch -D <branchname>".
     else:
       args.insert(0, "'All local branches'")
 
-  def _ExecuteOne(self, opt, nb, project):
+  def _ExecuteOne(self, all_branches, nb, project):
     """Abandon one project."""
-    if opt.all:
+    if all_branches:
       branches = project.GetBranches()
     else:
       branches = [nb]
@@ -76,7 +71,7 @@ It is equivalent to "git branch -D <branchname>".
     success = defaultdict(list)
     all_projects = self.GetProjects(args[1:])
 
-    def _ProcessResults(states):
+    def _ProcessResults(_pool, pm, states):
       for (results, project) in states:
         for branch, status in results.items():
           if status:
@@ -85,17 +80,12 @@ It is equivalent to "git branch -D <branchname>".
             err[branch].append(project)
         pm.update()
 
-    pm = Progress('Abandon %s' % nb, len(all_projects))
-    # NB: Multiprocessing is heavy, so don't spin it up for one job.
-    if len(all_projects) == 1 or opt.jobs == 1:
-      _ProcessResults(self._ExecuteOne(opt, nb, x) for x in all_projects)
-    else:
-      with multiprocessing.Pool(opt.jobs) as pool:
-        states = pool.imap_unordered(
-            functools.partial(self._ExecuteOne, opt, nb), all_projects,
-            chunksize=WORKER_BATCH_SIZE)
-        _ProcessResults(states)
-    pm.end()
+    self.ExecuteInParallel(
+        opt.jobs,
+        functools.partial(self._ExecuteOne, opt.all, nb),
+        all_projects,
+        callback=_ProcessResults,
+        output=Progress('Abandon %s' % (nb,), len(all_projects), quiet=opt.quiet))
 
     width = max(itertools.chain(
         [25], (len(x) for x in itertools.chain(success, err))))

subcmds/branches.py

@@ -13,10 +13,10 @@
 # limitations under the License.
 
 import itertools
-import multiprocessing
 import sys
+
 from color import Coloring
-from command import Command, DEFAULT_LOCAL_JOBS, WORKER_BATCH_SIZE
+from command import Command, DEFAULT_LOCAL_JOBS
 
 
 class BranchColoring(Coloring):
@@ -62,7 +62,7 @@ class BranchInfo(object):
 
 
 class Branches(Command):
-  common = True
+  COMMON = True
   helpSummary = "View current topic branches"
   helpUsage = """
 %prog [<project>...]
@@ -102,15 +102,19 @@ is shown, then the branch appears in all projects.
     out = BranchColoring(self.manifest.manifestProject.config)
     all_branches = {}
     project_cnt = len(projects)
-    with multiprocessing.Pool(processes=opt.jobs) as pool:
-      project_branches = pool.imap_unordered(
-          expand_project_to_branches, projects, chunksize=WORKER_BATCH_SIZE)
 
-      for name, b in itertools.chain.from_iterable(project_branches):
+    def _ProcessResults(_pool, _output, results):
+      for name, b in itertools.chain.from_iterable(results):
         if name not in all_branches:
           all_branches[name] = BranchInfo(name)
         all_branches[name].add(b)
 
+    self.ExecuteInParallel(
+        opt.jobs,
+        expand_project_to_branches,
+        projects,
+        callback=_ProcessResults)
+
     names = sorted(all_branches)
 
     if not names:

subcmds/checkout.py

@@ -13,15 +13,14 @@
 # limitations under the License.
 
 import functools
-import multiprocessing
 import sys
 
-from command import Command, DEFAULT_LOCAL_JOBS, WORKER_BATCH_SIZE
+from command import Command, DEFAULT_LOCAL_JOBS
 from progress import Progress
 
 
 class Checkout(Command):
-  common = True
+  COMMON = True
   helpSummary = "Checkout a branch for development"
   helpUsage = """
 %prog <branchname> [<project>...]
@@ -50,7 +49,7 @@ The command is equivalent to:
     success = []
     all_projects = self.GetProjects(args[1:])
 
-    def _ProcessResults(results):
+    def _ProcessResults(_pool, pm, results):
       for status, project in results:
         if status is not None:
           if status:
@@ -59,17 +58,12 @@ The command is equivalent to:
             err.append(project)
         pm.update()
 
-    pm = Progress('Checkout %s' % nb, len(all_projects))
-    # NB: Multiprocessing is heavy, so don't spin it up for one job.
-    if len(all_projects) == 1 or opt.jobs == 1:
-      _ProcessResults(self._ExecuteOne(nb, x) for x in all_projects)
-    else:
-      with multiprocessing.Pool(opt.jobs) as pool:
-        results = pool.imap_unordered(
-            functools.partial(self._ExecuteOne, nb), all_projects,
-            chunksize=WORKER_BATCH_SIZE)
-        _ProcessResults(results)
-    pm.end()
+    self.ExecuteInParallel(
+        opt.jobs,
+        functools.partial(self._ExecuteOne, nb),
+        all_projects,
+        callback=_ProcessResults,
+        output=Progress('Checkout %s' % (nb,), len(all_projects), quiet=opt.quiet))
 
     if err:
       for p in err:

subcmds/cherry_pick.py

@@ -21,7 +21,7 @@ CHANGE_ID_RE = re.compile(r'^\s*Change-Id: I([0-9a-f]{40})\s*$')
 
 
 class CherryPick(Command):
-  common = True
+  COMMON = True
   helpSummary = "Cherry-pick a change."
   helpUsage = """
 %prog <sha1>
@@ -32,9 +32,6 @@ The change id will be updated, and a reference to the old
 change id will be added.
 """
 
-  def _Options(self, p):
-    pass
-
   def ValidateOptions(self, opt, args):
     if len(args) != 1:
       self.Usage()

subcmds/diff.py

@@ -14,13 +14,12 @@
 
 import functools
 import io
-import multiprocessing
 
-from command import DEFAULT_LOCAL_JOBS, PagedCommand, WORKER_BATCH_SIZE
+from command import DEFAULT_LOCAL_JOBS, PagedCommand
 
 
 class Diff(PagedCommand):
-  common = True
+  COMMON = True
   helpSummary = "Show changes between commit and working tree"
   helpUsage = """
 %prog [<project>...]
@@ -32,12 +31,11 @@ to the Unix 'patch' command.
   PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
   def _Options(self, p):
-    super()._Options(p)
     p.add_option('-u', '--absolute',
                  dest='absolute', action='store_true',
-                 help='Paths are relative to the repository root')
+                 help='paths are relative to the repository root')
 
-  def _DiffHelper(self, absolute, project):
+  def _ExecuteOne(self, absolute, project):
     """Obtains the diff for a specific project.
 
     Args:
@@ -52,22 +50,20 @@ to the Unix 'patch' command.
     return (ret, buf.getvalue())
 
   def Execute(self, opt, args):
-    ret = 0
     all_projects = self.GetProjects(args)
 
-    # NB: Multiprocessing is heavy, so don't spin it up for one job.
-    if len(all_projects) == 1 or opt.jobs == 1:
-      for project in all_projects:
-        if not project.PrintWorkTreeDiff(opt.absolute):
+    def _ProcessResults(_pool, _output, results):
+      ret = 0
+      for (state, output) in results:
+        if output:
+          print(output, end='')
+        if not state:
           ret = 1
-    else:
-      with multiprocessing.Pool(opt.jobs) as pool:
-        states = pool.imap(functools.partial(self._DiffHelper, opt.absolute),
-                           all_projects, WORKER_BATCH_SIZE)
-        for (state, output) in states:
-          if output:
-            print(output, end='')
-          if not state:
-            ret = 1
+      return ret
 
-    return ret
+    return self.ExecuteInParallel(
+        opt.jobs,
+        functools.partial(self._ExecuteOne, opt.absolute),
+        all_projects,
+        callback=_ProcessResults,
+        ordered=True)

subcmds/diffmanifests.py

@@ -31,7 +31,7 @@ class Diffmanifests(PagedCommand):
   deeper level.
   """
 
-  common = True
+  COMMON = True
   helpSummary = "Manifest diff utility"
   helpUsage = """%prog manifest1.xml [manifest2.xml] [options]"""
 
@@ -68,10 +68,10 @@ synced and their revisions won't be found.
   def _Options(self, p):
     p.add_option('--raw',
                  dest='raw', action='store_true',
-                 help='Display raw diff.')
+                 help='display raw diff')
     p.add_option('--no-color',
                  dest='color', action='store_false', default=True,
-                 help='does not display the diff in color.')
+                 help='does not display the diff in color')
     p.add_option('--pretty-format',
                  dest='pretty_format', action='store',
                  metavar='<FORMAT>',

subcmds/download.py

@@ -22,7 +22,7 @@ CHANGE_RE = re.compile(r'^([1-9][0-9]*)(?:[/\.-]([1-9][0-9]*))?$')
 
 
 class Download(Command):
-  common = True
+  COMMON = True
   helpSummary = "Download and checkout a change"
   helpUsage = """
 %prog {[project] change[/patchset]}...

subcmds/forall.py

@@ -24,6 +24,7 @@ import subprocess
 
 from color import Coloring
 from command import DEFAULT_LOCAL_JOBS, Command, MirrorSafeCommand, WORKER_BATCH_SIZE
+from error import ManifestInvalidRevisionError
 
 _CAN_COLOR = [
     'branch',
@@ -40,11 +41,11 @@ class ForallColoring(Coloring):
 
 
 class Forall(Command, MirrorSafeCommand):
-  common = False
+  COMMON = False
   helpSummary = "Run a shell command in each project"
   helpUsage = """
 %prog [<project>...] -c <command> [<arg>...]
-%prog -r str1 [str2] ... -c <command> [<arg>...]"
+%prog -r str1 [str2] ... -c <command> [<arg>...]
 """
   helpDescription = """
 Executes the same shell command in each project.
@@ -52,6 +53,11 @@ Executes the same shell command in each project.
 The -r option allows running the command only on projects matching
 regex or wildcard expression.
 
+By default, projects are processed non-interactively in parallel.  If you want
+to run interactive commands, make sure to pass --interactive to force --jobs 1.
+While the processing order of projects is not guaranteed, the order of project
+output is stable.
+
 # Output Formatting
 
 The -p option causes '%prog' to bind pipes to the command's stdin,
@@ -123,37 +129,35 @@ without iterating through the remaining projects.
       del parser.rargs[0]
 
   def _Options(self, p):
-    super()._Options(p)
-
     p.add_option('-r', '--regex',
                  dest='regex', action='store_true',
-                 help="Execute the command only on projects matching regex or wildcard expression")
+                 help='execute the command only on projects matching regex or wildcard expression')
     p.add_option('-i', '--inverse-regex',
                  dest='inverse_regex', action='store_true',
-                 help="Execute the command only on projects not matching regex or "
-                      "wildcard expression")
+                 help='execute the command only on projects not matching regex or '
+                      'wildcard expression')
     p.add_option('-g', '--groups',
                  dest='groups',
-                 help="Execute the command only on projects matching the specified groups")
+                 help='execute the command only on projects matching the specified groups')
     p.add_option('-c', '--command',
-                 help='Command (and arguments) to execute',
+                 help='command (and arguments) to execute',
                  dest='command',
                  action='callback',
                  callback=self._cmd_option)
     p.add_option('-e', '--abort-on-errors',
                  dest='abort_on_errors', action='store_true',
-                 help='Abort if a command exits unsuccessfully')
+                 help='abort if a command exits unsuccessfully')
     p.add_option('--ignore-missing', action='store_true',
-                 help='Silently skip & do not exit non-zero due missing '
+                 help='silently skip & do not exit non-zero due missing '
                       'checkouts')
 
-    g = p.add_option_group('Output')
+    g = p.get_option_group('--quiet')
     g.add_option('-p',
                  dest='project_header', action='store_true',
-                 help='Show project headers before output')
-    g.add_option('-v', '--verbose',
-                 dest='verbose', action='store_true',
-                 help='Show command error messages')
+                 help='show project headers before output')
+    p.add_option('--interactive',
+                 action='store_true',
+                 help='force interactive usage')
 
   def WantPager(self, opt):
     return opt.project_header and opt.jobs == 1
@@ -173,6 +177,11 @@ without iterating through the remaining projects.
       cmd.append(cmd[0])
     cmd.extend(opt.command[1:])
 
+    # Historically, forall operated interactively, and in serial.  If the user
+    # has selected 1 job, then default to interacive mode.
+    if opt.jobs == 1:
+      opt.interactive = True
+
     if opt.project_header \
             and not shell \
             and cmd[0] == 'git':
@@ -239,7 +248,7 @@ without iterating through the remaining projects.
       rc = rc or errno.EINTR
     except Exception as e:
       # Catch any other exceptions raised
-      print('Got an error, terminating the pool: %s: %s' %
+      print('forall: unhandled error, terminating the pool: %s: %s' %
             (type(e).__name__, e),
             file=sys.stderr)
       rc = rc or getattr(e, 'errno', 1)
@@ -282,7 +291,13 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
   setenv('REPO_PROJECT', project.name)
   setenv('REPO_PATH', project.relpath)
   setenv('REPO_REMOTE', project.remote.name)
-  setenv('REPO_LREV', '' if mirror else project.GetRevisionId())
+  try:
+    # If we aren't in a fully synced state and we don't have the ref the manifest
+    # wants, then this will fail.  Ignore it for the purposes of this code.
+    lrev = '' if mirror else project.GetRevisionId()
+  except ManifestInvalidRevisionError:
+    lrev = ''
+  setenv('REPO_LREV', lrev)
   setenv('REPO_RREV', project.revisionExpr)
   setenv('REPO_UPSTREAM', project.upstream)
   setenv('REPO_DEST_BRANCH', project.dest_branch)
@@ -313,10 +328,12 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
   else:
     stderr = subprocess.DEVNULL
 
+  stdin = None if opt.interactive else subprocess.DEVNULL
+
   result = subprocess.run(
       cmd, cwd=cwd, shell=shell, env=env, check=False,
       encoding='utf-8', errors='replace',
-      stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=stderr)
+      stdin=stdin, stdout=subprocess.PIPE, stderr=stderr)
 
   output = result.stdout
   if opt.project_header:

subcmds/gitc_delete.py

@@ -19,7 +19,7 @@ import platform_utils
 
 
 class GitcDelete(Command, GitcClientCommand):
-  common = True
+  COMMON = True
   visible_everywhere = False
   helpSummary = "Delete a GITC Client."
   helpUsage = """
@@ -33,7 +33,7 @@ and all locally downloaded sources.
   def _Options(self, p):
     p.add_option('-f', '--force',
                  dest='force', action='store_true',
-                 help='Force the deletion (no prompt).')
+                 help='force the deletion (no prompt)')
 
   def Execute(self, opt, args):
     if not opt.force:

subcmds/gitc_init.py

@@ -23,7 +23,7 @@ import wrapper
 
 
 class GitcInit(init.Init, GitcAvailableCommand):
-  common = True
+  COMMON = True
   helpSummary = "Initialize a GITC Client."
   helpUsage = """
 %prog [options] [client name]
@@ -48,13 +48,6 @@ use for this GITC client.
 
   def _Options(self, p):
     super()._Options(p, gitc_init=True)
-    g = p.add_option_group('GITC options')
-    g.add_option('-f', '--manifest-file',
-                 dest='manifest_file',
-                 help='Optional manifest file to use for this GITC client.')
-    g.add_option('-c', '--gitc-client',
-                 dest='gitc_client',
-                 help='The name of the gitc_client instance to create or modify.')
 
   def Execute(self, opt, args):
     gitc_client = gitc_utils.parse_clientdir(os.getcwd())

subcmds/grep.py

@@ -12,10 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import functools
 import sys
 
 from color import Coloring
-from command import PagedCommand
+from command import DEFAULT_LOCAL_JOBS, PagedCommand
 from error import GitError
 from git_command import GitCommand
 
@@ -28,7 +29,7 @@ class GrepColoring(Coloring):
 
 
 class Grep(PagedCommand):
-  common = True
+  COMMON = True
   helpSummary = "Print lines matching a pattern"
   helpUsage = """
 %prog {pattern | -e pattern} [<project>...]
@@ -61,6 +62,7 @@ contain a line that matches both expressions:
   repo grep --all-match -e NODE -e Unexpected
 
 """
+  PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
   @staticmethod
   def _carry_option(_option, opt_str, value, parser):
@@ -79,6 +81,10 @@ contain a line that matches both expressions:
     if value is not None:
       pt.append(value)
 
+  def _CommonOptions(self, p):
+    """Override common options slightly."""
+    super()._CommonOptions(p, opt_v=False)
+
   def _Options(self, p):
     g = p.add_option_group('Sources')
     g.add_option('--cached',
@@ -152,6 +158,72 @@ contain a line that matches both expressions:
                  action='callback', callback=self._carry_option,
                  help='Show only file names not containing matching lines')
 
+  def _ExecuteOne(self, cmd_argv, project):
+    """Process one project."""
+    try:
+      p = GitCommand(project,
+                     cmd_argv,
+                     bare=False,
+                     capture_stdout=True,
+                     capture_stderr=True)
+    except GitError as e:
+      return (project, -1, None, str(e))
+
+    return (project, p.Wait(), p.stdout, p.stderr)
+
+  @staticmethod
+  def _ProcessResults(full_name, have_rev, _pool, out, results):
+    git_failed = False
+    bad_rev = False
+    have_match = False
+
+    for project, rc, stdout, stderr in results:
+      if rc < 0:
+        git_failed = True
+        out.project('--- project %s ---' % project.relpath)
+        out.nl()
+        out.fail('%s', stderr)
+        out.nl()
+        continue
+
+      if rc:
+        # no results
+        if stderr:
+          if have_rev and 'fatal: ambiguous argument' in stderr:
+            bad_rev = True
+          else:
+            out.project('--- project %s ---' % project.relpath)
+            out.nl()
+            out.fail('%s', stderr.strip())
+            out.nl()
+        continue
+      have_match = True
+
+      # We cut the last element, to avoid a blank line.
+      r = stdout.split('\n')
+      r = r[0:-1]
+
+      if have_rev and full_name:
+        for line in r:
+          rev, line = line.split(':', 1)
+          out.write("%s", rev)
+          out.write(':')
+          out.project(project.relpath)
+          out.write('/')
+          out.write("%s", line)
+          out.nl()
+      elif full_name:
+        for line in r:
+          out.project(project.relpath)
+          out.write('/')
+          out.write("%s", line)
+          out.nl()
+      else:
+        for line in r:
+          print(line)
+
+    return (git_failed, bad_rev, have_match)
+
   def Execute(self, opt, args):
     out = GrepColoring(self.manifest.manifestProject.config)
 
@@ -183,62 +255,13 @@ contain a line that matches both expressions:
       cmd_argv.extend(opt.revision)
     cmd_argv.append('--')
 
-    git_failed = False
-    bad_rev = False
-    have_match = False
-
-    for project in projects:
-      try:
-        p = GitCommand(project,
-                       cmd_argv,
-                       bare=False,
-                       capture_stdout=True,
-                       capture_stderr=True)
-      except GitError as e:
-        git_failed = True
-        out.project('--- project %s ---' % project.relpath)
-        out.nl()
-        out.fail('%s', str(e))
-        out.nl()
-        continue
-
-      if p.Wait() != 0:
-        # no results
-        #
-        if p.stderr:
-          if have_rev and 'fatal: ambiguous argument' in p.stderr:
-            bad_rev = True
-          else:
-            out.project('--- project %s ---' % project.relpath)
-            out.nl()
-            out.fail('%s', p.stderr.strip())
-            out.nl()
-        continue
-      have_match = True
-
-      # We cut the last element, to avoid a blank line.
-      #
-      r = p.stdout.split('\n')
-      r = r[0:-1]
-
-      if have_rev and full_name:
-        for line in r:
-          rev, line = line.split(':', 1)
-          out.write("%s", rev)
-          out.write(':')
-          out.project(project.relpath)
-          out.write('/')
-          out.write("%s", line)
-          out.nl()
-      elif full_name:
-        for line in r:
-          out.project(project.relpath)
-          out.write('/')
-          out.write("%s", line)
-          out.nl()
-      else:
-        for line in r:
-          print(line)
+    git_failed, bad_rev, have_match = self.ExecuteInParallel(
+        opt.jobs,
+        functools.partial(self._ExecuteOne, cmd_argv),
+        projects,
+        callback=functools.partial(self._ProcessResults, full_name, have_rev),
+        output=out,
+        ordered=True)
 
     if git_failed:
       sys.exit(1)

subcmds/help.py

@@ -14,16 +14,17 @@
 
 import re
 import sys
-from formatter import AbstractFormatter, DumbWriter
+import textwrap
 
 from subcmds import all_commands
 from color import Coloring
 from command import PagedCommand, MirrorSafeCommand, GitcAvailableCommand, GitcClientCommand
 import gitc_utils
+from wrapper import Wrapper
 
 
 class Help(PagedCommand, MirrorSafeCommand):
-  common = False
+  COMMON = False
   helpSummary = "Display detailed help on a command"
   helpUsage = """
 %prog [--all|command]
@@ -72,20 +73,20 @@ Displays detailed usage information about a command.
 
     commandNames = list(sorted([name
                                 for name, command in all_commands.items()
-                                if command.common and gitc_supported(command)]))
+                                if command.COMMON and gitc_supported(command)]))
     self._PrintCommands(commandNames)
 
     print(
         "See 'repo help <command>' for more information on a specific command.\n"
         "See 'repo help --all' for a complete list of recognized commands.")
+    print('Bug reports:', Wrapper().BUG_URL)
 
   def _PrintCommandHelp(self, cmd, header_prefix=''):
     class _Out(Coloring):
       def __init__(self, gc):
         Coloring.__init__(self, gc, 'help')
         self.heading = self.printer('heading', attr='bold')
-
-        self.wrap = AbstractFormatter(DumbWriter())
+        self._first = True
 
       def _PrintSection(self, heading, bodyAttr):
         try:
@@ -95,7 +96,9 @@ Displays detailed usage information about a command.
         if body == '' or body is None:
           return
 
-        self.nl()
+        if not self._first:
+          self.nl()
+        self._first = False
 
         self.heading('%s%s', header_prefix, heading)
         self.nl()
@@ -105,7 +108,8 @@ Displays detailed usage information about a command.
         body = body.strip()
         body = body.replace('%prog', me)
 
-        asciidoc_hdr = re.compile(r'^\n?#+ (.+)$')
+        # Extract the title, but skip any trailing {#anchors}.
+        asciidoc_hdr = re.compile(r'^\n?#+ ([^{]+)(\{#.+\})?$')
         for para in body.split("\n\n"):
           if para.startswith(' '):
             self.write('%s', para)
@@ -120,9 +124,12 @@ Displays detailed usage information about a command.
             self.nl()
             continue
 
-          self.wrap.add_flowing_data(para)
-          self.wrap.end_paragraph(1)
-        self.wrap.end_paragraph(0)
+          lines = textwrap.wrap(para.replace('  ', ' '), width=80,
+                                break_long_words=False, break_on_hyphens=False)
+          for line in lines:
+            self.write('%s', line)
+            self.nl()
+          self.nl()
 
     out = _Out(self.client.globalConfig)
     out._PrintSection('Summary', 'helpSummary')
@@ -131,8 +138,7 @@ Displays detailed usage information about a command.
 
   def _PrintAllCommandHelp(self):
     for name in sorted(all_commands):
-      cmd = all_commands[name]()
-      cmd.manifest = self.manifest
+      cmd = all_commands[name](manifest=self.manifest)
       self._PrintCommandHelp(cmd, header_prefix='[%s] ' % (name,))
 
   def _Options(self, p):
@@ -156,12 +162,11 @@ Displays detailed usage information about a command.
       name = args[0]
 
       try:
-        cmd = all_commands[name]()
+        cmd = all_commands[name](manifest=self.manifest)
       except KeyError:
         print("repo: '%s' is not a repo command." % name, file=sys.stderr)
         sys.exit(1)
 
-      cmd.manifest = self.manifest
       self._PrintCommandHelp(cmd)
 
     else:

subcmds/info.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import optparse
+
 from command import PagedCommand
 from color import Coloring
 from git_refs import R_M, R_HEADS
@@ -23,9 +25,9 @@ class _Coloring(Coloring):
 
 
 class Info(PagedCommand):
-  common = True
+  COMMON = True
   helpSummary = "Get info on the manifest branch, current branch or unmerged branches"
-  helpUsage = "%prog [-dl] [-o [-b]] [<project>...]"
+  helpUsage = "%prog [-dl] [-o [-c]] [<project>...]"
 
   def _Options(self, p):
     p.add_option('-d', '--diff',
@@ -34,12 +36,19 @@ class Info(PagedCommand):
     p.add_option('-o', '--overview',
                  dest='overview', action='store_true',
                  help='show overview of all local commits')
-    p.add_option('-b', '--current-branch',
+    p.add_option('-c', '--current-branch',
                  dest="current_branch", action="store_true",
                  help="consider only checked out branches")
+    p.add_option('--no-current-branch',
+                 dest='current_branch', action='store_false',
+                 help='consider all local branches')
+    # Turn this into a warning & remove this someday.
+    p.add_option('-b',
+                 dest='current_branch', action='store_true',
+                 help=optparse.SUPPRESS_HELP)
     p.add_option('-l', '--local-only',
                  dest="local", action="store_true",
-                 help="Disable all remote operations")
+                 help="disable all remote operations")
 
   def Execute(self, opt, args):
     self.out = _Coloring(self.client.globalConfig)

subcmds/init.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import optparse
 import os
 import platform
 import re
@@ -31,7 +30,7 @@ from wrapper import Wrapper
 
 
 class Init(InteractiveCommand, MirrorSafeCommand):
-  common = True
+  COMMON = True
   helpSummary = "Initialize a repo client checkout in the current directory"
   helpUsage = """
 %prog [options] [manifest url]
@@ -79,107 +78,11 @@ manifest, a subsequent `repo sync` (or `repo sync -d`) is necessary
 to update the working directory files.
 """
 
+  def _CommonOptions(self, p):
+    """Disable due to re-use of Wrapper()."""
+
   def _Options(self, p, gitc_init=False):
-    # Logging
-    g = p.add_option_group('Logging options')
-    g.add_option('-v', '--verbose',
-                 dest='output_mode', action='store_true',
-                 help='show all output')
-    g.add_option('-q', '--quiet',
-                 dest='output_mode', action='store_false',
-                 help='only show errors')
-
-    # Manifest
-    g = p.add_option_group('Manifest options')
-    g.add_option('-u', '--manifest-url',
-                 dest='manifest_url',
-                 help='manifest repository location', metavar='URL')
-    g.add_option('-b', '--manifest-branch', metavar='REVISION',
-                 help='manifest branch or revision (use HEAD for default)')
-    cbr_opts = ['--current-branch']
-    # The gitc-init subcommand allocates -c itself, but a lot of init users
-    # want -c, so try to satisfy both as best we can.
-    if not gitc_init:
-      cbr_opts += ['-c']
-    g.add_option(*cbr_opts,
-                 dest='current_branch_only', action='store_true',
-                 help='fetch only current manifest branch from server')
-    g.add_option('-m', '--manifest-name',
-                 dest='manifest_name', default='default.xml',
-                 help='initial manifest file', metavar='NAME.xml')
-    g.add_option('--mirror',
-                 dest='mirror', action='store_true',
-                 help='create a replica of the remote repositories '
-                      'rather than a client working directory')
-    g.add_option('--reference',
-                 dest='reference',
-                 help='location of mirror directory', metavar='DIR')
-    g.add_option('--dissociate',
-                 dest='dissociate', action='store_true',
-                 help='dissociate from reference mirrors after clone')
-    g.add_option('--depth', type='int', default=None,
-                 dest='depth',
-                 help='create a shallow clone with given depth; see git clone')
-    g.add_option('--partial-clone', action='store_true',
-                 dest='partial_clone',
-                 help='perform partial clone (https://git-scm.com/'
-                 'docs/gitrepository-layout#_code_partialclone_code)')
-    g.add_option('--clone-filter', action='store', default='blob:none',
-                 dest='clone_filter',
-                 help='filter for use with --partial-clone [default: %default]')
-    # TODO(vapier): Expose option with real help text once this has been in the
-    # wild for a while w/out significant bug reports.  Goal is by ~Sep 2020.
-    g.add_option('--worktree', action='store_true',
-                 help=optparse.SUPPRESS_HELP)
-    g.add_option('--archive',
-                 dest='archive', action='store_true',
-                 help='checkout an archive instead of a git repository for '
-                      'each project. See git archive.')
-    g.add_option('--submodules',
-                 dest='submodules', action='store_true',
-                 help='sync any submodules associated with the manifest repo')
-    g.add_option('--use-superproject', action='store_true',
-                 help='use the manifest superproject to sync projects')
-    g.add_option('--no-use-superproject', action='store_false',
-                 dest='use_superproject',
-                 help='disable use of manifest superprojects')
-    g.add_option('-g', '--groups',
-                 dest='groups', default='default',
-                 help='restrict manifest projects to ones with specified '
-                      'group(s) [default|all|G1,G2,G3|G4,-G5,-G6]',
-                 metavar='GROUP')
-    g.add_option('-p', '--platform',
-                 dest='platform', default='auto',
-                 help='restrict manifest projects to ones with a specified '
-                      'platform group [auto|all|none|linux|darwin|...]',
-                 metavar='PLATFORM')
-    g.add_option('--clone-bundle', action='store_true',
-                 help='force use of /clone.bundle on HTTP/HTTPS (default if not --partial-clone)')
-    g.add_option('--no-clone-bundle',
-                 dest='clone_bundle', action='store_false',
-                 help='disable use of /clone.bundle on HTTP/HTTPS (default if --partial-clone)')
-    g.add_option('--no-tags',
-                 dest='tags', default=True, action='store_false',
-                 help="don't fetch tags in the manifest")
-
-    # Tool
-    g = p.add_option_group('repo Version options')
-    g.add_option('--repo-url',
-                 dest='repo_url',
-                 help='repo repository location', metavar='URL')
-    g.add_option('--repo-rev', metavar='REV',
-                 help='repo branch or revision')
-    g.add_option('--repo-branch', dest='repo_rev',
-                 help=optparse.SUPPRESS_HELP)
-    g.add_option('--no-repo-verify',
-                 dest='repo_verify', default=True, action='store_false',
-                 help='do not verify repo source code')
-
-    # Other
-    g = p.add_option_group('Other options')
-    g.add_option('--config-name',
-                 dest='config_name', action="store_true", default=False,
-                 help='Always prompt for name/e-mail')
+    Wrapper().InitParser(p, gitc_init=gitc_init)
 
   def _RegisteredEnvironmentOptions(self):
     return {'REPO_MANIFEST_URL': 'manifest_url',
@@ -193,10 +96,17 @@ to update the working directory files.
     """
     superproject = git_superproject.Superproject(self.manifest,
                                                  self.repodir,
+                                                 self.git_event_log,
                                                  quiet=opt.quiet)
-    if not superproject.Sync():
-      print('error: git update of superproject failed', file=sys.stderr)
-      sys.exit(1)
+    sync_result = superproject.Sync()
+    if not sync_result.success:
+      print('warning: git update of superproject failed, repo sync will not '
+            'use superproject to fetch source; while this error is not fatal, '
+            'and you can continue to run repo sync, please run repo init with '
+            'the --no-use-superproject option to stop seeing this warning',
+            file=sys.stderr)
+      if sync_result.fatal and opt.use_superproject is not None:
+        sys.exit(1)
 
   def _SyncManifest(self, opt):
     m = self.manifest.manifestProject
@@ -269,7 +179,7 @@ to update the working directory files.
 
     groups = [x for x in groups if x]
     groupstr = ','.join(groups)
-    if opt.platform == 'auto' and groupstr == 'default,platform-' + platform.system().lower():
+    if opt.platform == 'auto' and groupstr == self.manifest.GetDefaultGroupsStr():
       groupstr = None
     m.config.SetString('manifest.groups', groupstr)
 
@@ -313,7 +223,7 @@ to update the working directory files.
               'in another location.', file=sys.stderr)
         sys.exit(1)
 
-    if opt.partial_clone:
+    if opt.partial_clone is not None:
       if opt.mirror:
         print('fatal: --mirror and --partial-clone are mutually exclusive',
               file=sys.stderr)
@@ -321,9 +231,14 @@ to update the working directory files.
       m.config.SetBoolean('repo.partialclone', opt.partial_clone)
       if opt.clone_filter:
         m.config.SetString('repo.clonefilter', opt.clone_filter)
+    elif m.config.GetBoolean('repo.partialclone'):
+      opt.clone_filter = m.config.GetString('repo.clonefilter')
     else:
       opt.clone_filter = None
 
+    if opt.partial_clone_exclude is not None:
+      m.config.SetString('repo.partialcloneexclude', opt.partial_clone_exclude)
+
     if opt.clone_bundle is None:
       opt.clone_bundle = False if opt.partial_clone else True
     else:
@@ -339,7 +254,8 @@ to update the working directory files.
                               clone_bundle=opt.clone_bundle,
                               current_branch_only=opt.current_branch_only,
                               tags=opt.tags, submodules=opt.submodules,
-                              clone_filter=opt.clone_filter):
+                              clone_filter=opt.clone_filter,
+                              partial_clone_exclude=self.manifest.PartialCloneExclude):
       r = m.GetRemote(m.remote.name)
       print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
 
@@ -529,9 +445,6 @@ to update the working directory files.
             % ('.'.join(str(x) for x in MIN_GIT_VERSION_SOFT),),
             file=sys.stderr)
 
-    opt.quiet = opt.output_mode is False
-    opt.verbose = opt.output_mode is True
-
     rp = self.manifest.repoProject
 
     # Handle new --repo-url requests.

subcmds/list.py

@@ -16,34 +16,42 @@ from command import Command, MirrorSafeCommand
 
 
 class List(Command, MirrorSafeCommand):
-  common = True
+  COMMON = True
   helpSummary = "List projects and their associated directories"
   helpUsage = """
 %prog [-f] [<project>...]
-%prog [-f] -r str1 [str2]..."
+%prog [-f] -r str1 [str2]...
 """
   helpDescription = """
 List all projects; pass '.' to list the project for the cwd.
 
+By default, only projects that currently exist in the checkout are shown.  If
+you want to list all projects (using the specified filter settings), use the
+--all option.  If you want to show all projects regardless of the manifest
+groups, then also pass --groups all.
+
 This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
 """
 
   def _Options(self, p):
     p.add_option('-r', '--regex',
                  dest='regex', action='store_true',
-                 help="Filter the project list based on regex or wildcard matching of strings")
+                 help='filter the project list based on regex or wildcard matching of strings')
     p.add_option('-g', '--groups',
                  dest='groups',
-                 help="Filter the project list based on the groups the project is in")
+                 help='filter the project list based on the groups the project is in')
+    p.add_option('-a', '--all',
+                 action='store_true',
+                 help='show projects regardless of checkout state')
     p.add_option('-f', '--fullpath',
                  dest='fullpath', action='store_true',
-                 help="Display the full work tree path instead of the relative path")
+                 help='display the full work tree path instead of the relative path')
     p.add_option('-n', '--name-only',
                  dest='name_only', action='store_true',
-                 help="Display only the name of the repository")
+                 help='display only the name of the repository')
     p.add_option('-p', '--path-only',
                  dest='path_only', action='store_true',
-                 help="Display only the path of the repository")
+                 help='display only the path of the repository')
 
   def ValidateOptions(self, opt, args):
     if opt.fullpath and opt.name_only:
@@ -61,7 +69,7 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
       args: Positional args.  Can be a list of projects to list, or empty.
     """
     if not opt.regex:
-      projects = self.GetProjects(args, groups=opt.groups)
+      projects = self.GetProjects(args, groups=opt.groups, missing_ok=opt.all)
     else:
       projects = self.FindProjects(args)
 
@@ -79,5 +87,6 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
       else:
         lines.append("%s : %s" % (_getpath(project), project.name))
 
-    lines.sort()
-    print('\n'.join(lines))
+    if lines:
+      lines.sort()
+      print('\n'.join(lines))

subcmds/manifest.py

@@ -20,7 +20,7 @@ from command import PagedCommand
 
 
 class Manifest(PagedCommand):
-  common = False
+  COMMON = False
   helpSummary = "Manifest inspection utility"
   helpUsage = """
 %prog [-o {-|NAME.xml}] [-m MANIFEST.xml] [-r]
@@ -53,27 +53,27 @@ to indicate the remote ref to push changes to via 'repo upload'.
   def _Options(self, p):
     p.add_option('-r', '--revision-as-HEAD',
                  dest='peg_rev', action='store_true',
-                 help='Save revisions as current HEAD')
+                 help='save revisions as current HEAD')
     p.add_option('-m', '--manifest-name',
                  help='temporary manifest to use for this sync', metavar='NAME.xml')
     p.add_option('--suppress-upstream-revision', dest='peg_rev_upstream',
                  default=True, action='store_false',
-                 help='If in -r mode, do not write the upstream field.  '
-                 'Only of use if the branch names for a sha1 manifest are '
-                 'sensitive.')
+                 help='if in -r mode, do not write the upstream field '
+                 '(only of use if the branch names for a sha1 manifest are '
+                 'sensitive)')
     p.add_option('--suppress-dest-branch', dest='peg_rev_dest_branch',
                  default=True, action='store_false',
-                 help='If in -r mode, do not write the dest-branch field.  '
-                 'Only of use if the branch names for a sha1 manifest are '
-                 'sensitive.')
+                 help='if in -r mode, do not write the dest-branch field '
+                 '(only of use if the branch names for a sha1 manifest are '
+                 'sensitive)')
     p.add_option('--json', default=False, action='store_true',
-                 help='Output manifest in JSON format (experimental).')
+                 help='output manifest in JSON format (experimental)')
     p.add_option('--pretty', default=False, action='store_true',
-                 help='Format output for humans to read.')
+                 help='format output for humans to read')
     p.add_option('-o', '--output-file',
                  dest='output_file',
                  default='-',
-                 help='File to save the manifest to',
+                 help='file to save the manifest to',
                  metavar='-|NAME.xml')
 
   def _Output(self, opt):

subcmds/overview.py

@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import optparse
+
 from color import Coloring
 from command import PagedCommand
 
 
 class Overview(PagedCommand):
-  common = True
+  COMMON = True
   helpSummary = "Display overview of unmerged project branches"
   helpUsage = """
 %prog [--current-branch] [<project>...]
@@ -26,15 +28,22 @@ class Overview(PagedCommand):
 The '%prog' command is used to display an overview of the projects branches,
 and list any local commits that have not yet been merged into the project.
 
-The -b/--current-branch option can be used to restrict the output to only
+The -c/--current-branch option can be used to restrict the output to only
 branches currently checked out in each project.  By default, all branches
 are displayed.
 """
 
   def _Options(self, p):
-    p.add_option('-b', '--current-branch',
+    p.add_option('-c', '--current-branch',
                  dest="current_branch", action="store_true",
-                 help="Consider only checked out branches")
+                 help="consider only checked out branches")
+    p.add_option('--no-current-branch',
+                 dest='current_branch', action='store_false',
+                 help='consider all local branches')
+    # Turn this into a warning & remove this someday.
+    p.add_option('-b',
+                 dest='current_branch', action='store_true',
+                 help=optparse.SUPPRESS_HELP)
 
   def Execute(self, opt, args):
     all_branches = []

subcmds/prune.py

@@ -12,21 +12,38 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import itertools
+
 from color import Coloring
-from command import PagedCommand
+from command import DEFAULT_LOCAL_JOBS, PagedCommand
 
 
 class Prune(PagedCommand):
-  common = True
+  COMMON = True
   helpSummary = "Prune (delete) already merged topics"
   helpUsage = """
 %prog [<project>...]
 """
+  PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
+
+  def _ExecuteOne(self, project):
+    """Process one project."""
+    return project.PruneHeads()
 
   def Execute(self, opt, args):
-    all_branches = []
-    for project in self.GetProjects(args):
-      all_branches.extend(project.PruneHeads())
+    projects = self.GetProjects(args)
+
+    # NB: Should be able to refactor this module to display summary as results
+    # come back from children.
+    def _ProcessResults(_pool, _output, results):
+      return list(itertools.chain.from_iterable(results))
+
+    all_branches = self.ExecuteInParallel(
+        opt.jobs,
+        self._ExecuteOne,
+        projects,
+        callback=_ProcessResults,
+        ordered=True)
 
     if not all_branches:
       return

subcmds/rebase.py

@@ -27,7 +27,7 @@ class RebaseColoring(Coloring):
 
 
 class Rebase(Command):
-  common = True
+  COMMON = True
   helpSummary = "Rebase local branches on upstream branch"
   helpUsage = """
 %prog {[<project>...] | -i <project>...}
@@ -39,36 +39,34 @@ branch but need to incorporate new upstream changes "underneath" them.
 """
 
   def _Options(self, p):
-    p.add_option('-i', '--interactive',
+    g = p.get_option_group('--quiet')
+    g.add_option('-i', '--interactive',
                  dest="interactive", action="store_true",
                  help="interactive rebase (single project only)")
 
     p.add_option('--fail-fast',
                  dest='fail_fast', action='store_true',
-                 help='Stop rebasing after first error is hit')
+                 help='stop rebasing after first error is hit')
     p.add_option('-f', '--force-rebase',
                  dest='force_rebase', action='store_true',
-                 help='Pass --force-rebase to git rebase')
+                 help='pass --force-rebase to git rebase')
     p.add_option('--no-ff',
                  dest='ff', default=True, action='store_false',
-                 help='Pass --no-ff to git rebase')
-    p.add_option('-q', '--quiet',
-                 dest='quiet', action='store_true',
-                 help='Pass --quiet to git rebase')
+                 help='pass --no-ff to git rebase')
     p.add_option('--autosquash',
                  dest='autosquash', action='store_true',
-                 help='Pass --autosquash to git rebase')
+                 help='pass --autosquash to git rebase')
     p.add_option('--whitespace',
                  dest='whitespace', action='store', metavar='WS',
-                 help='Pass --whitespace to git rebase')
+                 help='pass --whitespace to git rebase')
     p.add_option('--auto-stash',
                  dest='auto_stash', action='store_true',
-                 help='Stash local modifications before starting')
+                 help='stash local modifications before starting')
     p.add_option('-m', '--onto-manifest',
                  dest='onto_manifest', action='store_true',
-                 help='Rebase onto the manifest version instead of upstream '
-                      'HEAD.  This helps to make sure the local tree stays '
-                      'consistent if you previously synced to a manifest.')
+                 help='rebase onto the manifest version instead of upstream '
+                      'HEAD (this helps to make sure the local tree stays '
+                      'consistent if you previously synced to a manifest)')
 
   def Execute(self, opt, args):
     all_projects = self.GetProjects(args)

subcmds/selfupdate.py

@@ -21,7 +21,7 @@ from subcmds.sync import _PostRepoFetch
 
 
 class Selfupdate(Command, MirrorSafeCommand):
-  common = False
+  COMMON = False
   helpSummary = "Update repo to the latest version"
   helpUsage = """
 %prog

subcmds/smartsync.py

@@ -16,7 +16,7 @@ from subcmds.sync import Sync
 
 
 class Smartsync(Sync):
-  common = True
+  COMMON = True
   helpSummary = "Update working tree to the latest known good revision"
   helpUsage = """
 %prog [<project>...]

subcmds/stage.py

@@ -28,7 +28,7 @@ class _ProjectList(Coloring):
 
 
 class Stage(InteractiveCommand):
-  common = True
+  COMMON = True
   helpSummary = "Stage file(s) for commit"
   helpUsage = """
 %prog -i [<project>...]
@@ -38,7 +38,8 @@ The '%prog' command stages files to prepare the next commit.
 """
 
   def _Options(self, p):
-    p.add_option('-i', '--interactive',
+    g = p.get_option_group('--quiet')
+    g.add_option('-i', '--interactive',
                  dest='interactive', action='store_true',
                  help='use interactive staging')
 

subcmds/start.py

@@ -13,11 +13,10 @@
 # limitations under the License.
 
 import functools
-import multiprocessing
 import os
 import sys
 
-from command import Command, DEFAULT_LOCAL_JOBS, WORKER_BATCH_SIZE
+from command import Command, DEFAULT_LOCAL_JOBS
 from git_config import IsImmutable
 from git_command import git
 import gitc_utils
@@ -26,7 +25,7 @@ from project import SyncBuffer
 
 
 class Start(Command):
-  common = True
+  COMMON = True
   helpSummary = "Start a new branch for development"
   helpUsage = """
 %prog <newbranchname> [--all | <project>...]
@@ -38,13 +37,13 @@ revision specified in the manifest.
   PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
   def _Options(self, p):
-    super()._Options(p)
     p.add_option('--all',
                  dest='all', action='store_true',
                  help='begin branch in all projects')
     p.add_option('-r', '--rev', '--revision', dest='revision',
                  help='point branch at this revision instead of upstream')
-    p.add_option('--head', dest='revision', action='store_const', const='HEAD',
+    p.add_option('--head', '--HEAD',
+                 dest='revision', action='store_const', const='HEAD',
                  help='abbreviation for --rev HEAD')
 
   def ValidateOptions(self, opt, args):
@@ -55,7 +54,7 @@ revision specified in the manifest.
     if not git.check_ref_format('heads/%s' % nb):
       self.OptionParser.error("'%s' is not a valid name" % nb)
 
-  def _ExecuteOne(self, opt, nb, project):
+  def _ExecuteOne(self, revision, nb, project):
     """Start one project."""
     # If the current revision is immutable, such as a SHA1, a tag or
     # a change, then we can't push back to it. Substitute with
@@ -69,7 +68,7 @@ revision specified in the manifest.
 
     try:
       ret = project.StartBranch(
-          nb, branch_merge=branch_merge, revision=opt.revision)
+          nb, branch_merge=branch_merge, revision=revision)
     except Exception as e:
       print('error: unable to checkout %s: %s' % (project.name, e), file=sys.stderr)
       ret = False
@@ -106,7 +105,7 @@ revision specified in the manifest.
       if not os.path.exists(os.getcwd()):
         os.chdir(self.manifest.topdir)
 
-      pm = Progress('Syncing %s' % nb, len(all_projects))
+      pm = Progress('Syncing %s' % nb, len(all_projects), quiet=opt.quiet)
       for project in all_projects:
         gitc_project = self.gitc_manifest.paths[project.relpath]
         # Sync projects that have not been opened.
@@ -123,23 +122,18 @@ revision specified in the manifest.
         pm.update()
       pm.end()
 
-    def _ProcessResults(results):
+    def _ProcessResults(_pool, pm, results):
       for (result, project) in results:
         if not result:
           err.append(project)
         pm.update()
 
-    pm = Progress('Starting %s' % nb, len(all_projects))
-    # NB: Multiprocessing is heavy, so don't spin it up for one job.
-    if len(all_projects) == 1 or opt.jobs == 1:
-      _ProcessResults(self._ExecuteOne(opt, nb, x) for x in all_projects)
-    else:
-      with multiprocessing.Pool(opt.jobs) as pool:
-        results = pool.imap_unordered(
-            functools.partial(self._ExecuteOne, opt, nb), all_projects,
-            chunksize=WORKER_BATCH_SIZE)
-        _ProcessResults(results)
-    pm.end()
+    self.ExecuteInParallel(
+        opt.jobs,
+        functools.partial(self._ExecuteOne, opt.revision, nb),
+        all_projects,
+        callback=_ProcessResults,
+        output=Progress('Starting %s' % (nb,), len(all_projects), quiet=opt.quiet))
 
     if err:
       for p in err:

subcmds/status.py

@@ -15,17 +15,16 @@
 import functools
 import glob
 import io
-import multiprocessing
 import os
 
-from command import DEFAULT_LOCAL_JOBS, PagedCommand, WORKER_BATCH_SIZE
+from command import DEFAULT_LOCAL_JOBS, PagedCommand
 
 from color import Coloring
 import platform_utils
 
 
 class Status(PagedCommand):
-  common = True
+  COMMON = True
   helpSummary = "Show the working tree status"
   helpUsage = """
 %prog [<project>...]
@@ -80,12 +79,9 @@ the following meanings:
   PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
   def _Options(self, p):
-    super()._Options(p)
     p.add_option('-o', '--orphans',
                  dest='orphans', action='store_true',
                  help="include objects in working directory outside of repo projects")
-    p.add_option('-q', '--quiet', action='store_true',
-                 help="only print the name of modified projects")
 
   def _StatusHelper(self, quiet, project):
     """Obtains the status for a specific project.
@@ -122,22 +118,23 @@ the following meanings:
 
   def Execute(self, opt, args):
     all_projects = self.GetProjects(args)
-    counter = 0
 
-    if opt.jobs == 1:
-      for project in all_projects:
-        state = project.PrintWorkTreeStatus(quiet=opt.quiet)
+    def _ProcessResults(_pool, _output, results):
+      ret = 0
+      for (state, output) in results:
+        if output:
+          print(output, end='')
         if state == 'CLEAN':
-          counter += 1
-    else:
-      with multiprocessing.Pool(opt.jobs) as pool:
-        states = pool.imap(functools.partial(self._StatusHelper, opt.quiet),
-                           all_projects, chunksize=WORKER_BATCH_SIZE)
-        for (state, output) in states:
-          if output:
-            print(output, end='')
-          if state == 'CLEAN':
-            counter += 1
+          ret += 1
+      return ret
+
+    counter = self.ExecuteInParallel(
+        opt.jobs,
+        functools.partial(self._StatusHelper, opt.quiet),
+        all_projects,
+        callback=_ProcessResults,
+        ordered=True)
+
     if not opt.quiet and len(all_projects) == counter:
       print('nothing to commit (working directory clean)')
 

subcmds/sync.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import errno
 import functools
 import http.cookiejar as cookielib
 import io
@@ -20,9 +21,7 @@ import multiprocessing
 import netrc
 from optparse import SUPPRESS_HELP
 import os
-import re
 import socket
-import subprocess
 import sys
 import tempfile
 import time
@@ -45,13 +44,8 @@ except ImportError:
   def _rlimit_nofile():
     return (256, 256)
 
-try:
-  import multiprocessing
-except ImportError:
-  multiprocessing = None
-
 import event_log
-from git_command import GIT, git_require
+from git_command import git_require
 from git_config import GetUrlCookieFile
 from git_refs import R_HEADS, HEAD
 import git_superproject
@@ -63,19 +57,16 @@ from error import RepoChangedException, GitError, ManifestParseError
 import platform_utils
 from project import SyncBuffer
 from progress import Progress
+import ssh
 from wrapper import Wrapper
 from manifest_xml import GitcManifest
 
 _ONE_DAY_S = 24 * 60 * 60
 
 
-class _FetchError(Exception):
-  """Internal error thrown in _FetchHelper() when we don't want stack trace."""
-
-
 class Sync(Command, MirrorSafeCommand):
   jobs = 1
-  common = True
+  COMMON = True
   helpSummary = "Update working tree to the latest revision"
   helpUsage = """
 %prog [<project>...]
@@ -178,12 +169,19 @@ later is required to fix a server side protocol bug.
 """
   PARALLEL_JOBS = 1
 
+  def _CommonOptions(self, p):
+    if self.manifest:
+      try:
+        self.PARALLEL_JOBS = self.manifest.default.sync_j
+      except ManifestParseError:
+        pass
+    super()._CommonOptions(p)
+
   def _Options(self, p, show_smart=True):
-    try:
-      self.PARALLEL_JOBS = self.manifest.default.sync_j
-    except ManifestParseError:
-      pass
-    super()._Options(p)
+    p.add_option('--jobs-network', default=None, type=int, metavar='JOBS',
+                 help='number of network jobs to run in parallel (defaults to --jobs)')
+    p.add_option('--jobs-checkout', default=None, type=int, metavar='JOBS',
+                 help='number of local checkout jobs to run in parallel (defaults to --jobs)')
 
     p.add_option('-f', '--force-broken',
                  dest='force_broken', action='store_true',
@@ -217,12 +215,9 @@ later is required to fix a server side protocol bug.
     p.add_option('-c', '--current-branch',
                  dest='current_branch_only', action='store_true',
                  help='fetch only current branch from server')
-    p.add_option('-v', '--verbose',
-                 dest='output_mode', action='store_true',
-                 help='show all sync output')
-    p.add_option('-q', '--quiet',
-                 dest='output_mode', action='store_false',
-                 help='only show errors')
+    p.add_option('--no-current-branch',
+                 dest='current_branch_only', action='store_false',
+                 help='fetch all branches from server')
     p.add_option('-m', '--manifest-name',
                  dest='manifest_name',
                  help='temporary manifest to use for this sync', metavar='NAME.xml')
@@ -241,8 +236,14 @@ later is required to fix a server side protocol bug.
                  help='fetch submodules from server')
     p.add_option('--use-superproject', action='store_true',
                  help='use the manifest superproject to sync projects')
+    p.add_option('--no-use-superproject', action='store_false',
+                 dest='use_superproject',
+                 help='disable use of manifest superprojects')
+    p.add_option('--tags',
+                 action='store_false',
+                 help='fetch tags')
     p.add_option('--no-tags',
-                 dest='tags', default=True, action='store_false',
+                 dest='tags', action='store_false',
                  help="don't fetch tags")
     p.add_option('--optimized-fetch',
                  dest='optimized_fetch', action='store_true',
@@ -277,7 +278,11 @@ later is required to fix a server side protocol bug.
       branch = branch[len(R_HEADS):]
     return branch
 
-  def _UpdateProjectsRevisionId(self, opt, args):
+  def _GetCurrentBranchOnly(self, opt):
+    """Returns True if current-branch or use-superproject options are enabled."""
+    return opt.current_branch_only or git_superproject.UseSuperproject(opt, self.manifest)
+
+  def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests):
     """Update revisionId of every project with the SHA from superproject.
 
     This function updates each project's revisionId with SHA from superproject.
@@ -287,166 +292,165 @@ later is required to fix a server side protocol bug.
       opt: Program options returned from optparse.  See _Options().
       args: Arguments to pass to GetProjects. See the GetProjects
           docstring for details.
+      load_local_manifests: Whether to load local manifests.
 
     Returns:
-      Returns path to the overriding manifest file.
+      Returns path to the overriding manifest file instead of None.
     """
     superproject = git_superproject.Superproject(self.manifest,
                                                  self.repodir,
+                                                 self.git_event_log,
                                                  quiet=opt.quiet)
     all_projects = self.GetProjects(args,
                                     missing_ok=True,
                                     submodules_ok=opt.fetch_submodules)
-    manifest_path = superproject.UpdateProjectsRevisionId(all_projects)
-    if not manifest_path:
-      print('error: Update of revsionId from superproject has failed',
+    update_result = superproject.UpdateProjectsRevisionId(all_projects)
+    manifest_path = update_result.manifest_path
+    if manifest_path:
+      self._ReloadManifest(manifest_path, load_local_manifests)
+    else:
+      print('warning: Update of revisionId from superproject has failed, '
+            'repo sync will not use superproject to fetch the source. ',
+            'Please resync with the --no-use-superproject option to avoid this repo warning.',
             file=sys.stderr)
-      sys.exit(1)
-    self._ReloadManifest(manifest_path)
+      if update_result.fatal and opt.use_superproject is not None:
+        sys.exit(1)
     return manifest_path
 
-  def _FetchProjectList(self, opt, projects, sem, *args, **kwargs):
-    """Main function of the fetch threads.
+  def _FetchProjectList(self, opt, projects):
+    """Main function of the fetch worker.
+
+    The projects we're given share the same underlying git object store, so we
+    have to fetch them in serial.
 
     Delegates most of the work to _FetchHelper.
 
     Args:
       opt: Program options returned from optparse.  See _Options().
       projects: Projects to fetch.
-      sem: We'll release() this semaphore when we exit so that another thread
-          can be started up.
-      *args, **kwargs: Remaining arguments to pass to _FetchHelper. See the
-          _FetchHelper docstring for details.
     """
-    try:
-        for project in projects:
-          success = self._FetchHelper(opt, project, *args, **kwargs)
-          if not success and opt.fail_fast:
-            break
-    finally:
-        sem.release()
+    return [self._FetchOne(opt, x) for x in projects]
 
-  def _FetchHelper(self, opt, project, lock, fetched, pm, err_event,
-                   clone_filter):
+  def _FetchOne(self, opt, project):
     """Fetch git objects for a single project.
 
     Args:
       opt: Program options returned from optparse.  See _Options().
       project: Project object for the project to fetch.
-      lock: Lock for accessing objects that are shared amongst multiple
-          _FetchHelper() threads.
-      fetched: set object that we will add project.gitdir to when we're done
-          (with our lock held).
-      pm: Instance of a Project object.  We will call pm.update() (with our
-          lock held).
-      err_event: We'll set this event in the case of an error (after printing
-          out info about the error).
-      clone_filter: Filter for use in a partial clone.
 
     Returns:
       Whether the fetch was successful.
     """
-    # We'll set to true once we've locked the lock.
-    did_lock = False
-
-    # Encapsulate everything in a try/except/finally so that:
-    # - We always set err_event in the case of an exception.
-    # - We always make sure we unlock the lock if we locked it.
     start = time.time()
     success = False
     buf = io.StringIO()
-    with lock:
-      pm.start(project.name)
     try:
-      try:
-        success = project.Sync_NetworkHalf(
-            quiet=opt.quiet,
-            verbose=opt.verbose,
-            output_redir=buf,
-            current_branch_only=opt.current_branch_only,
-            force_sync=opt.force_sync,
-            clone_bundle=opt.clone_bundle,
-            tags=opt.tags, archive=self.manifest.IsArchive,
-            optimized_fetch=opt.optimized_fetch,
-            retry_fetches=opt.retry_fetches,
-            prune=opt.prune,
-            clone_filter=clone_filter)
-        self._fetch_times.Set(project, time.time() - start)
+      success = project.Sync_NetworkHalf(
+          quiet=opt.quiet,
+          verbose=opt.verbose,
+          output_redir=buf,
+          current_branch_only=self._GetCurrentBranchOnly(opt),
+          force_sync=opt.force_sync,
+          clone_bundle=opt.clone_bundle,
+          tags=opt.tags, archive=self.manifest.IsArchive,
+          optimized_fetch=opt.optimized_fetch,
+          retry_fetches=opt.retry_fetches,
+          prune=opt.prune,
+          ssh_proxy=self.ssh_proxy,
+          clone_filter=self.manifest.CloneFilter,
+          partial_clone_exclude=self.manifest.PartialCloneExclude)
 
-        # Lock around all the rest of the code, since printing, updating a set
-        # and Progress.update() are not thread safe.
-        lock.acquire()
-        did_lock = True
+      output = buf.getvalue()
+      if (opt.verbose or not success) and output:
+        print('\n' + output.rstrip())
 
-        output = buf.getvalue()
-        if opt.verbose and output:
-          pm.update(inc=0, msg=output.rstrip())
+      if not success:
+        print('error: Cannot fetch %s from %s'
+              % (project.name, project.remote.url),
+              file=sys.stderr)
+    except GitError as e:
+      print('error.GitError: Cannot fetch %s' % str(e), file=sys.stderr)
+    except Exception as e:
+      print('error: Cannot fetch %s (%s: %s)'
+            % (project.name, type(e).__name__, str(e)), file=sys.stderr)
+      raise
 
-        if not success:
-          err_event.set()
-          print('error: Cannot fetch %s from %s'
-                % (project.name, project.remote.url),
-                file=sys.stderr)
-          if opt.fail_fast:
-            raise _FetchError()
+    finish = time.time()
+    return (success, project, start, finish)
 
-        fetched.add(project.gitdir)
-      except _FetchError:
-        pass
-      except Exception as e:
-        print('error: Cannot fetch %s (%s: %s)'
-              % (project.name, type(e).__name__, str(e)), file=sys.stderr)
-        err_event.set()
-        raise
-    finally:
-      if not did_lock:
-        lock.acquire()
-      pm.finish(project.name)
-      lock.release()
-      finish = time.time()
-      self.event_log.AddSync(project, event_log.TASK_SYNC_NETWORK,
-                             start, finish, success)
+  @classmethod
+  def _FetchInitChild(cls, ssh_proxy):
+    cls.ssh_proxy = ssh_proxy
 
-    return success
+  def _Fetch(self, projects, opt, err_event, ssh_proxy):
+    ret = True
 
-  def _Fetch(self, projects, opt, err_event):
+    jobs = opt.jobs_network if opt.jobs_network else self.jobs
     fetched = set()
-    lock = _threading.Lock()
-    pm = Progress('Fetching', len(projects))
+    pm = Progress('Fetching', len(projects), delay=False, quiet=opt.quiet)
 
     objdir_project_map = dict()
     for project in projects:
       objdir_project_map.setdefault(project.objdir, []).append(project)
+    projects_list = list(objdir_project_map.values())
 
-    threads = set()
-    sem = _threading.Semaphore(self.jobs)
-    for project_list in objdir_project_map.values():
-      # Check for any errors before running any more tasks.
-      # ...we'll let existing threads finish, though.
-      if err_event.is_set() and opt.fail_fast:
-        break
+    def _ProcessResults(results_sets):
+      ret = True
+      for results in results_sets:
+        for (success, project, start, finish) in results:
+          self._fetch_times.Set(project, finish - start)
+          self.event_log.AddSync(project, event_log.TASK_SYNC_NETWORK,
+                                 start, finish, success)
+          # Check for any errors before running any more tasks.
+          # ...we'll let existing jobs finish, though.
+          if not success:
+            ret = False
+          else:
+            fetched.add(project.gitdir)
+          pm.update(msg=project.name)
+        if not ret and opt.fail_fast:
+          break
+      return ret
 
-      sem.acquire()
-      kwargs = dict(opt=opt,
-                    projects=project_list,
-                    sem=sem,
-                    lock=lock,
-                    fetched=fetched,
-                    pm=pm,
-                    err_event=err_event,
-                    clone_filter=self.manifest.CloneFilter)
-      if self.jobs > 1:
-        t = _threading.Thread(target=self._FetchProjectList,
-                              kwargs=kwargs)
-        # Ensure that Ctrl-C will not freeze the repo process.
-        t.daemon = True
-        threads.add(t)
-        t.start()
+    # We pass the ssh proxy settings via the class.  This allows multiprocessing
+    # to pickle it up when spawning children.  We can't pass it as an argument
+    # to _FetchProjectList below as multiprocessing is unable to pickle those.
+    Sync.ssh_proxy = None
+
+    # NB: Multiprocessing is heavy, so don't spin it up for one job.
+    if len(projects_list) == 1 or jobs == 1:
+      self._FetchInitChild(ssh_proxy)
+      if not _ProcessResults(self._FetchProjectList(opt, x) for x in projects_list):
+        ret = False
+    else:
+      # Favor throughput over responsiveness when quiet.  It seems that imap()
+      # will yield results in batches relative to chunksize, so even as the
+      # children finish a sync, we won't see the result until one child finishes
+      # ~chunksize jobs.  When using a large --jobs with large chunksize, this
+      # can be jarring as there will be a large initial delay where repo looks
+      # like it isn't doing anything and sits at 0%, but then suddenly completes
+      # a lot of jobs all at once.  Since this code is more network bound, we
+      # can accept a bit more CPU overhead with a smaller chunksize so that the
+      # user sees more immediate & continuous feedback.
+      if opt.quiet:
+        chunksize = WORKER_BATCH_SIZE
       else:
-        self._FetchProjectList(**kwargs)
+        pm.update(inc=0, msg='warming up')
+        chunksize = 4
+      with multiprocessing.Pool(
+          jobs, initializer=self._FetchInitChild, initargs=(ssh_proxy,)) as pool:
+        results = pool.imap_unordered(
+            functools.partial(self._FetchProjectList, opt),
+            projects_list,
+            chunksize=chunksize)
+        if not _ProcessResults(results):
+          ret = False
+          pool.close()
 
-    for t in threads:
-      t.join()
+    # Cleanup the reference now that we're done with it, and we're going to
+    # release any resources it points to.  If we don't, later multiprocessing
+    # usage (e.g. checkouts) will try to pickle and then crash.
+    del Sync.ssh_proxy
 
     pm.end()
     self._fetch_times.Save()
@@ -454,13 +458,77 @@ later is required to fix a server side protocol bug.
     if not self.manifest.IsArchive:
       self._GCProjects(projects, opt, err_event)
 
-    return fetched
+    return (ret, fetched)
 
-  def _CheckoutOne(self, opt, project):
-    """Checkout work tree for one project
+  def _FetchMain(self, opt, args, all_projects, err_event, manifest_name,
+                 load_local_manifests, ssh_proxy):
+    """The main network fetch loop.
 
     Args:
       opt: Program options returned from optparse.  See _Options().
+      args: Command line args used to filter out projects.
+      all_projects: List of all projects that should be fetched.
+      err_event: Whether an error was hit while processing.
+      manifest_name: Manifest file to be reloaded.
+      load_local_manifests: Whether to load local manifests.
+      ssh_proxy: SSH manager for clients & masters.
+
+    Returns:
+      List of all projects that should be checked out.
+    """
+    rp = self.manifest.repoProject
+
+    to_fetch = []
+    now = time.time()
+    if _ONE_DAY_S <= (now - rp.LastFetch):
+      to_fetch.append(rp)
+    to_fetch.extend(all_projects)
+    to_fetch.sort(key=self._fetch_times.Get, reverse=True)
+
+    success, fetched = self._Fetch(to_fetch, opt, err_event, ssh_proxy)
+    if not success:
+      err_event.set()
+
+    _PostRepoFetch(rp, opt.repo_verify)
+    if opt.network_only:
+      # bail out now; the rest touches the working tree
+      if err_event.is_set():
+        print('\nerror: Exited sync due to fetch errors.\n', file=sys.stderr)
+        sys.exit(1)
+      return
+
+    # Iteratively fetch missing and/or nested unregistered submodules
+    previously_missing_set = set()
+    while True:
+      self._ReloadManifest(manifest_name, load_local_manifests)
+      all_projects = self.GetProjects(args,
+                                      missing_ok=True,
+                                      submodules_ok=opt.fetch_submodules)
+      missing = []
+      for project in all_projects:
+        if project.gitdir not in fetched:
+          missing.append(project)
+      if not missing:
+        break
+      # Stop us from non-stopped fetching actually-missing repos: If set of
+      # missing repos has not been changed from last fetch, we break.
+      missing_set = set(p.name for p in missing)
+      if previously_missing_set == missing_set:
+        break
+      previously_missing_set = missing_set
+      success, new_fetched = self._Fetch(missing, opt, err_event, ssh_proxy)
+      if not success:
+        err_event.set()
+      fetched.update(new_fetched)
+
+    return all_projects
+
+  def _CheckoutOne(self, detach_head, force_sync, project):
+    """Checkout work tree for one project
+
+    Args:
+      detach_head: Whether to leave a detached HEAD.
+      force_sync: Force checking out of the repo.
       project: Project object for the project to checkout.
 
     Returns:
@@ -468,11 +536,14 @@ later is required to fix a server side protocol bug.
     """
     start = time.time()
     syncbuf = SyncBuffer(self.manifest.manifestProject.config,
-                         detach_head=opt.detach_head)
+                         detach_head=detach_head)
     success = False
     try:
-      project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
+      project.Sync_LocalHalf(syncbuf, force_sync=force_sync)
       success = syncbuf.Finish()
+    except GitError as e:
+      print('error.GitError: Cannot checkout %s: %s' %
+            (project.name, str(e)), file=sys.stderr)
     except Exception as e:
       print('error: Cannot checkout %s: %s: %s' %
             (project.name, type(e).__name__, str(e)),
@@ -492,73 +563,66 @@ later is required to fix a server side protocol bug.
       opt: Program options returned from optparse.  See _Options().
       err_results: A list of strings, paths to git repos where checkout failed.
     """
-    ret = True
-
     # Only checkout projects with worktrees.
     all_projects = [x for x in all_projects if x.worktree]
 
-    pm = Progress('Checking out', len(all_projects))
-
-    def _ProcessResults(results):
+    def _ProcessResults(pool, pm, results):
+      ret = True
       for (success, project, start, finish) in results:
         self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL,
                                start, finish, success)
         # Check for any errors before running any more tasks.
-        # ...we'll let existing threads finish, though.
+        # ...we'll let existing jobs finish, though.
         if not success:
+          ret = False
           err_results.append(project.relpath)
           if opt.fail_fast:
-            return False
+            if pool:
+              pool.close()
+            return ret
         pm.update(msg=project.name)
-      return True
+      return ret
 
-    # NB: Multiprocessing is heavy, so don't spin it up for one job.
-    if len(all_projects) == 1 or opt.jobs == 1:
-      if not _ProcessResults(self._CheckoutOne(opt, x) for x in all_projects):
-        ret = False
-    else:
-      with multiprocessing.Pool(opt.jobs) as pool:
-        results = pool.imap_unordered(
-            functools.partial(self._CheckoutOne, opt),
-            all_projects,
-            chunksize=WORKER_BATCH_SIZE)
-        if not _ProcessResults(results):
-          ret = False
-          pool.close()
-
-    pm.end()
-
-    return ret
+    return self.ExecuteInParallel(
+        opt.jobs_checkout if opt.jobs_checkout else self.jobs,
+        functools.partial(self._CheckoutOne, opt.detach_head, opt.force_sync),
+        all_projects,
+        callback=_ProcessResults,
+        output=Progress('Checking out', len(all_projects), quiet=opt.quiet)) and not err_results
 
   def _GCProjects(self, projects, opt, err_event):
+    pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet)
+    pm.update(inc=0, msg='prescan')
+
     gc_gitdirs = {}
     for project in projects:
       # Make sure pruning never kicks in with shared projects.
       if (not project.use_git_worktrees and
               len(project.manifest.GetProjectsWithName(project.name)) > 1):
         if not opt.quiet:
-          print('%s: Shared project %s found, disabling pruning.' %
+          print('\r%s: Shared project %s found, disabling pruning.' %
                 (project.relpath, project.name))
         if git_require((2, 7, 0)):
           project.EnableRepositoryExtension('preciousObjects')
         else:
           # This isn't perfect, but it's the best we can do with old git.
-          print('%s: WARNING: shared projects are unreliable when using old '
+          print('\r%s: WARNING: shared projects are unreliable when using old '
                 'versions of git; please upgrade to git-2.7.0+.'
                 % (project.relpath,),
                 file=sys.stderr)
           project.config.SetString('gc.pruneExpire', 'never')
       gc_gitdirs[project.gitdir] = project.bare_git
 
-    if multiprocessing:
-      cpu_count = multiprocessing.cpu_count()
-    else:
-      cpu_count = 1
+    pm.update(inc=len(projects) - len(gc_gitdirs), msg='warming up')
+
+    cpu_count = os.cpu_count()
     jobs = min(self.jobs, cpu_count)
 
     if jobs < 2:
       for bare_git in gc_gitdirs.values():
+        pm.update(msg=bare_git._project.name)
         bare_git.gc('--auto')
+      pm.end()
       return
 
     config = {'pack.threads': cpu_count // jobs if cpu_count > jobs else 1}
@@ -567,6 +631,7 @@ later is required to fix a server side protocol bug.
     sem = _threading.Semaphore(jobs)
 
     def GC(bare_git):
+      pm.start(bare_git._project.name)
       try:
         try:
           bare_git.gc('--auto', config=config)
@@ -576,6 +641,7 @@ later is required to fix a server side protocol bug.
           err_event.set()
           raise
       finally:
+        pm.finish(bare_git._project.name)
         sem.release()
 
     for bare_git in gc_gitdirs.values():
@@ -589,11 +655,20 @@ later is required to fix a server side protocol bug.
 
     for t in threads:
       t.join()
+    pm.end()
 
-  def _ReloadManifest(self, manifest_name=None):
+  def _ReloadManifest(self, manifest_name=None, load_local_manifests=True):
+    """Reload the manfiest from the file specified by the |manifest_name|.
+
+    It unloads the manifest if |manifest_name| is None.
+
+    Args:
+      manifest_name: Manifest file to be reloaded.
+      load_local_manifests: Whether to load local manifests.
+    """
     if manifest_name:
       # Override calls _Unload already
-      self.manifest.Override(manifest_name)
+      self.manifest.Override(manifest_name, load_local_manifests=load_local_manifests)
     else:
       self.manifest._Unload()
 
@@ -640,6 +715,60 @@ later is required to fix a server side protocol bug.
       fd.write('\n')
     return 0
 
+  def UpdateCopyLinkfileList(self):
+    """Save all dests of copyfile and linkfile, and update them if needed.
+
+    Returns:
+      Whether update was successful.
+    """
+    new_paths = {}
+    new_linkfile_paths = []
+    new_copyfile_paths = []
+    for project in self.GetProjects(None, missing_ok=True):
+      new_linkfile_paths.extend(x.dest for x in project.linkfiles)
+      new_copyfile_paths.extend(x.dest for x in project.copyfiles)
+
+    new_paths = {
+        'linkfile': new_linkfile_paths,
+        'copyfile': new_copyfile_paths,
+    }
+
+    copylinkfile_name = 'copy-link-files.json'
+    copylinkfile_path = os.path.join(self.manifest.repodir, copylinkfile_name)
+    old_copylinkfile_paths = {}
+
+    if os.path.exists(copylinkfile_path):
+      with open(copylinkfile_path, 'rb') as fp:
+        try:
+          old_copylinkfile_paths = json.load(fp)
+        except:
+          print('error: %s is not a json formatted file.' %
+                copylinkfile_path, file=sys.stderr)
+          platform_utils.remove(copylinkfile_path)
+          return False
+
+      need_remove_files = []
+      need_remove_files.extend(
+          set(old_copylinkfile_paths.get('linkfile', [])) -
+          set(new_linkfile_paths))
+      need_remove_files.extend(
+          set(old_copylinkfile_paths.get('copyfile', [])) -
+          set(new_copyfile_paths))
+
+      for need_remove_file in need_remove_files:
+        try:
+          platform_utils.remove(need_remove_file)
+        except OSError as e:
+          if e.errno == errno.ENOENT:
+            # Try to remove the updated copyfile or linkfile.
+            # So, if the file is not exist, nothing need to do.
+            pass
+
+    # Create copy-link-files.json, save dest path of "copyfile" and "linkfile".
+    with open(copylinkfile_path, 'w', encoding='utf-8') as fp:
+      json.dump(new_paths, fp)
+    return True
+
   def _SmartSyncSetup(self, opt, smart_sync_manifest_path):
     if not self.manifest.manifest_server:
       print('error: cannot smart sync: no manifest server defined in '
@@ -735,13 +864,14 @@ later is required to fix a server side protocol bug.
     if not opt.local_only:
       start = time.time()
       success = mp.Sync_NetworkHalf(quiet=opt.quiet, verbose=opt.verbose,
-                                    current_branch_only=opt.current_branch_only,
+                                    current_branch_only=self._GetCurrentBranchOnly(opt),
                                     force_sync=opt.force_sync,
                                     tags=opt.tags,
                                     optimized_fetch=opt.optimized_fetch,
                                     retry_fetches=opt.retry_fetches,
                                     submodules=self.manifest.HasSubmodules,
-                                    clone_filter=self.manifest.CloneFilter)
+                                    clone_filter=self.manifest.CloneFilter,
+                                    partial_clone_exclude=self.manifest.PartialCloneExclude)
       finish = time.time()
       self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK,
                              start, finish, success)
@@ -755,7 +885,7 @@ later is required to fix a server side protocol bug.
                              start, time.time(), clean)
       if not clean:
         sys.exit(1)
-      self._ReloadManifest(opt.manifest_name)
+      self._ReloadManifest(manifest_name)
       if opt.jobs is None:
         self.jobs = self.manifest.default.sync_j
 
@@ -784,9 +914,6 @@ later is required to fix a server side protocol bug.
       soft_limit, _ = _rlimit_nofile()
       self.jobs = min(self.jobs, (soft_limit - 5) // 3)
 
-    opt.quiet = opt.output_mode is False
-    opt.verbose = opt.output_mode is True
-
     if opt.manifest_name:
       self.manifest.Override(opt.manifest_name)
 
@@ -807,7 +934,7 @@ later is required to fix a server side protocol bug.
           print('error: failed to remove existing smart sync override manifest: %s' %
                 e, file=sys.stderr)
 
-    err_event = _threading.Event()
+    err_event = multiprocessing.Event()
 
     rp = self.manifest.repoProject
     rp.PreSync()
@@ -830,10 +957,9 @@ later is required to fix a server side protocol bug.
     else:
       self._UpdateManifestProject(opt, mp, manifest_name)
 
-    if (opt.use_superproject or
-        self.manifest.manifestProject.config.GetBoolean(
-            'repo.superproject')):
-      manifest_name = self._UpdateProjectsRevisionId(opt, args)
+    load_local_manifests = not self.manifest.HasLocalManifests
+    if git_superproject.UseSuperproject(opt, self.manifest):
+      manifest_name = self._UpdateProjectsRevisionId(opt, args, load_local_manifests) or opt.manifest_name
 
     if self.gitc_manifest:
       gitc_manifest_projects = self.GetProjects(args,
@@ -879,44 +1005,17 @@ later is required to fix a server side protocol bug.
 
     self._fetch_times = _FetchTimes(self.manifest)
     if not opt.local_only:
-      to_fetch = []
-      now = time.time()
-      if _ONE_DAY_S <= (now - rp.LastFetch):
-        to_fetch.append(rp)
-      to_fetch.extend(all_projects)
-      to_fetch.sort(key=self._fetch_times.Get, reverse=True)
+      with multiprocessing.Manager() as manager:
+        with ssh.ProxyManager(manager) as ssh_proxy:
+          # Initialize the socket dir once in the parent.
+          ssh_proxy.sock()
+          all_projects = self._FetchMain(opt, args, all_projects, err_event,
+                                         manifest_name, load_local_manifests,
+                                         ssh_proxy)
 
-      fetched = self._Fetch(to_fetch, opt, err_event)
-
-      _PostRepoFetch(rp, opt.repo_verify)
       if opt.network_only:
-        # bail out now; the rest touches the working tree
-        if err_event.is_set():
-          print('\nerror: Exited sync due to fetch errors.\n', file=sys.stderr)
-          sys.exit(1)
         return
 
-      # Iteratively fetch missing and/or nested unregistered submodules
-      previously_missing_set = set()
-      while True:
-        self._ReloadManifest(manifest_name)
-        all_projects = self.GetProjects(args,
-                                        missing_ok=True,
-                                        submodules_ok=opt.fetch_submodules)
-        missing = []
-        for project in all_projects:
-          if project.gitdir not in fetched:
-            missing.append(project)
-        if not missing:
-          break
-        # Stop us from non-stopped fetching actually-missing repos: If set of
-        # missing repos has not been changed from last fetch, we break.
-        missing_set = set(p.name for p in missing)
-        if previously_missing_set == missing_set:
-          break
-        previously_missing_set = missing_set
-        fetched.update(self._Fetch(missing, opt, err_event))
-
       # If we saw an error, exit with code 1 so that other scripts can check.
       if err_event.is_set():
         err_network_sync = True
@@ -939,6 +1038,13 @@ later is required to fix a server side protocol bug.
         print('\nerror: Local checkouts *not* updated.', file=sys.stderr)
         sys.exit(1)
 
+    err_update_linkfiles = not self.UpdateCopyLinkfileList()
+    if err_update_linkfiles:
+      err_event.set()
+      if opt.fail_fast:
+        print('\nerror: Local update copyfile or linkfile failed.', file=sys.stderr)
+        sys.exit(1)
+
     err_results = []
     # NB: We don't exit here because this is the last step.
     err_checkout = not self._Checkout(all_projects, opt, err_results)
@@ -957,6 +1063,8 @@ later is required to fix a server side protocol bug.
         print('error: Downloading network changes failed.', file=sys.stderr)
       if err_update_projects:
         print('error: Updating local project lists failed.', file=sys.stderr)
+      if err_update_linkfiles:
+        print('error: Updating copyfiles or linkfiles failed.', file=sys.stderr)
       if err_checkout:
         print('error: Checking out local projects failed.', file=sys.stderr)
         if err_results:
@@ -981,12 +1089,25 @@ def _PostRepoUpgrade(manifest, quiet=False):
 def _PostRepoFetch(rp, repo_verify=True, verbose=False):
   if rp.HasChanges:
     print('info: A new version of repo is available', file=sys.stderr)
-    print(file=sys.stderr)
-    if not repo_verify or _VerifyTag(rp):
-      syncbuf = SyncBuffer(rp.config)
-      rp.Sync_LocalHalf(syncbuf)
-      if not syncbuf.Finish():
-        sys.exit(1)
+    wrapper = Wrapper()
+    try:
+      rev = rp.bare_git.describe(rp.GetRevisionId())
+    except GitError:
+      rev = None
+    _, new_rev = wrapper.check_repo_rev(rp.gitdir, rev, repo_verify=repo_verify)
+    # See if we're held back due to missing signed tag.
+    current_revid = rp.bare_git.rev_parse('HEAD')
+    new_revid = rp.bare_git.rev_parse('--verify', new_rev)
+    if current_revid != new_revid:
+      # We want to switch to the new rev, but also not trash any uncommitted
+      # changes.  This helps with local testing/hacking.
+      # If a local change has been made, we will throw that away.
+      # We also have to make sure this will switch to an older commit if that's
+      # the latest tag in order to support release rollback.
+      try:
+        rp.work_git.reset('--keep', new_rev)
+      except GitError as e:
+        sys.exit(str(e))
       print('info: Restarting repo with latest version', file=sys.stderr)
       raise RepoChangedException(['--repo-upgraded'])
     else:
@@ -997,45 +1118,6 @@ def _PostRepoFetch(rp, repo_verify=True, verbose=False):
             file=sys.stderr)
 
 
-def _VerifyTag(project):
-  gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
-  if not os.path.exists(gpg_dir):
-    print('warning: GnuPG was not available during last "repo init"\n'
-          'warning: Cannot automatically authenticate repo."""',
-          file=sys.stderr)
-    return True
-
-  try:
-    cur = project.bare_git.describe(project.GetRevisionId())
-  except GitError:
-    cur = None
-
-  if not cur \
-     or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
-    rev = project.revisionExpr
-    if rev.startswith(R_HEADS):
-      rev = rev[len(R_HEADS):]
-
-    print(file=sys.stderr)
-    print("warning: project '%s' branch '%s' is not signed"
-          % (project.name, rev), file=sys.stderr)
-    return False
-
-  env = os.environ.copy()
-  env['GIT_DIR'] = project.gitdir
-  env['GNUPGHOME'] = gpg_dir
-
-  cmd = [GIT, 'tag', '-v', cur]
-  result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
-                          env=env, check=False)
-  if result.returncode:
-    print(file=sys.stderr)
-    print(result.stdout, file=sys.stderr)
-    print(file=sys.stderr)
-    return False
-  return True
-
-
 class _FetchTimes(object):
   _ALPHA = 0.5
 

subcmds/upload.py

@@ -13,10 +13,12 @@
 # limitations under the License.
 
 import copy
+import functools
+import optparse
 import re
 import sys
 
-from command import InteractiveCommand
+from command import DEFAULT_LOCAL_JOBS, InteractiveCommand
 from editor import Editor
 from error import UploadError
 from git_command import GitCommand
@@ -53,7 +55,7 @@ def _SplitEmails(values):
 
 
 class Upload(InteractiveCommand):
-  common = True
+  COMMON = True
   helpSummary = "Upload changes for code review"
   helpUsage = """
 %prog [--re --cc] [<project>]...
@@ -145,58 +147,66 @@ https://gerrit-review.googlesource.com/Documentation/user-upload.html#notify
 Gerrit Code Review:  https://www.gerritcodereview.com/
 
 """
+  PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
   def _Options(self, p):
     p.add_option('-t',
                  dest='auto_topic', action='store_true',
-                 help='Send local branch name to Gerrit Code Review')
+                 help='send local branch name to Gerrit Code Review')
     p.add_option('--hashtag', '--ht',
                  dest='hashtags', action='append', default=[],
-                 help='Add hashtags (comma delimited) to the review.')
+                 help='add hashtags (comma delimited) to the review')
     p.add_option('--hashtag-branch', '--htb',
                  action='store_true',
-                 help='Add local branch name as a hashtag.')
+                 help='add local branch name as a hashtag')
     p.add_option('-l', '--label',
                  dest='labels', action='append', default=[],
-                 help='Add a label when uploading.')
+                 help='add a label when uploading')
     p.add_option('--re', '--reviewers',
                  type='string', action='append', dest='reviewers',
-                 help='Request reviews from these people.')
+                 help='request reviews from these people')
     p.add_option('--cc',
                  type='string', action='append', dest='cc',
-                 help='Also send email to these email addresses.')
-    p.add_option('--br',
+                 help='also send email to these email addresses')
+    p.add_option('--br', '--branch',
                  type='string', action='store', dest='branch',
-                 help='Branch to upload.')
-    p.add_option('--cbr', '--current-branch',
+                 help='(local) branch to upload')
+    p.add_option('-c', '--current-branch',
                  dest='current_branch', action='store_true',
-                 help='Upload current git branch.')
+                 help='upload current git branch')
+    p.add_option('--no-current-branch',
+                 dest='current_branch', action='store_false',
+                 help='upload all git branches')
+    # Turn this into a warning & remove this someday.
+    p.add_option('--cbr',
+                 dest='current_branch', action='store_true',
+                 help=optparse.SUPPRESS_HELP)
     p.add_option('--ne', '--no-emails',
                  action='store_false', dest='notify', default=True,
-                 help='If specified, do not send emails on upload.')
+                 help='do not send e-mails on upload')
     p.add_option('-p', '--private',
                  action='store_true', dest='private', default=False,
-                 help='If specified, upload as a private change.')
+                 help='upload as a private change (deprecated; use --wip)')
     p.add_option('-w', '--wip',
                  action='store_true', dest='wip', default=False,
-                 help='If specified, upload as a work-in-progress change.')
+                 help='upload as a work-in-progress change')
     p.add_option('-o', '--push-option',
                  type='string', action='append', dest='push_options',
                  default=[],
-                 help='Additional push options to transmit')
+                 help='additional push options to transmit')
     p.add_option('-D', '--destination', '--dest',
                  type='string', action='store', dest='dest_branch',
                  metavar='BRANCH',
-                 help='Submit for review on this target branch.')
+                 help='submit for review on this target branch')
     p.add_option('-n', '--dry-run',
                  dest='dryrun', default=False, action='store_true',
-                 help='Do everything except actually upload the CL.')
+                 help='do everything except actually upload the CL')
     p.add_option('-y', '--yes',
                  default=False, action='store_true',
-                 help='Answer yes to all safe prompts.')
+                 help='answer yes to all safe prompts')
     p.add_option('--no-cert-checks',
                  dest='validate_certs', action='store_false', default=True,
-                 help='Disable verifying ssl certs (unsafe).')
+                 help='disable verifying ssl certs (unsafe)')
     RepoHook.AddOptionGroup(p, 'pre-upload')
 
   def _SingleBranch(self, opt, branch, people):
@@ -502,40 +512,46 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     merge_branch = p.stdout.strip()
     return merge_branch
 
+  @staticmethod
+  def _GatherOne(opt, project):
+    """Figure out the upload status for |project|."""
+    if opt.current_branch:
+      cbr = project.CurrentBranch
+      up_branch = project.GetUploadableBranch(cbr)
+      avail = [up_branch] if up_branch else None
+    else:
+      avail = project.GetUploadableBranches(opt.branch)
+    return (project, avail)
+
   def Execute(self, opt, args):
-    project_list = self.GetProjects(args)
-    pending = []
-    reviewers = []
-    cc = []
-    branch = None
+    projects = self.GetProjects(args)
 
-    if opt.branch:
-      branch = opt.branch
-
-    for project in project_list:
-      if opt.current_branch:
-        cbr = project.CurrentBranch
-        up_branch = project.GetUploadableBranch(cbr)
-        if up_branch:
-          avail = [up_branch]
-        else:
-          avail = None
-          print('repo: error: Unable to upload branch "%s". '
+    def _ProcessResults(_pool, _out, results):
+      pending = []
+      for result in results:
+        project, avail = result
+        if avail is None:
+          print('repo: error: %s: Unable to upload branch "%s". '
                 'You might be able to fix the branch by running:\n'
                 '  git branch --set-upstream-to m/%s' %
-                (str(cbr), self.manifest.branch),
+                (project.relpath, project.CurrentBranch, self.manifest.branch),
                 file=sys.stderr)
-      else:
-        avail = project.GetUploadableBranches(branch)
-      if avail:
-        pending.append((project, avail))
+        elif avail:
+          pending.append(result)
+      return pending
+
+    pending = self.ExecuteInParallel(
+        opt.jobs,
+        functools.partial(self._GatherOne, opt),
+        projects,
+        callback=_ProcessResults)
 
     if not pending:
-      if branch is None:
+      if opt.branch is None:
         print('repo: error: no branches ready for upload', file=sys.stderr)
       else:
         print('repo: error: no branches named "%s" ready for upload' %
-              (branch,), file=sys.stderr)
+              (opt.branch,), file=sys.stderr)
       return 1
 
     pending_proj_names = [project.name for (project, available) in pending]
@@ -548,10 +564,8 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
         worktree_list=pending_worktrees):
       return 1
 
-    if opt.reviewers:
-      reviewers = _SplitEmails(opt.reviewers)
-    if opt.cc:
-      cc = _SplitEmails(opt.cc)
+    reviewers = _SplitEmails(opt.reviewers) if opt.reviewers else []
+    cc = _SplitEmails(opt.cc) if opt.cc else []
     people = (reviewers, cc)
 
     if len(pending) == 1 and len(pending[0][1]) == 1:

subcmds/version.py

@@ -18,13 +18,14 @@ import sys
 from command import Command, MirrorSafeCommand
 from git_command import git, RepoSourceVersion, user_agent
 from git_refs import HEAD
+from wrapper import Wrapper
 
 
 class Version(Command, MirrorSafeCommand):
   wrapper_version = None
   wrapper_path = None
 
-  common = False
+  COMMON = False
   helpSummary = "Display the version of repo"
   helpUsage = """
 %prog
@@ -62,3 +63,4 @@ class Version(Command, MirrorSafeCommand):
       print('OS %s %s (%s)' % (uname.system, uname.release, uname.version))
       print('CPU %s (%s)' %
             (uname.machine, uname.processor if uname.processor else 'unknown'))
+    print('Bug reports:', Wrapper().BUG_URL)

tests/test_git_command.py

@@ -26,33 +26,6 @@ import git_command
 import wrapper
 
 
-class SSHUnitTest(unittest.TestCase):
-  """Tests the ssh functions."""
-
-  def test_ssh_version(self):
-    """Check ssh_version() handling."""
-    ver = git_command._parse_ssh_version('Unknown\n')
-    self.assertEqual(ver, ())
-    ver = git_command._parse_ssh_version('OpenSSH_1.0\n')
-    self.assertEqual(ver, (1, 0))
-    ver = git_command._parse_ssh_version('OpenSSH_6.6.1p1 Ubuntu-2ubuntu2.13, OpenSSL 1.0.1f 6 Jan 2014\n')
-    self.assertEqual(ver, (6, 6, 1))
-    ver = git_command._parse_ssh_version('OpenSSH_7.6p1 Ubuntu-4ubuntu0.3, OpenSSL 1.0.2n  7 Dec 2017\n')
-    self.assertEqual(ver, (7, 6))
-
-  def test_ssh_sock(self):
-    """Check ssh_sock() function."""
-    with mock.patch('tempfile.mkdtemp', return_value='/tmp/foo'):
-      # old ssh version uses port
-      with mock.patch('git_command.ssh_version', return_value=(6, 6)):
-        self.assertTrue(git_command.ssh_sock().endswith('%p'))
-      git_command._ssh_sock_path = None
-      # new ssh version uses hash
-      with mock.patch('git_command.ssh_version', return_value=(6, 7)):
-        self.assertTrue(git_command.ssh_sock().endswith('%C'))
-      git_command._ssh_sock_path = None
-
-
 class GitCallUnitTest(unittest.TestCase):
   """Tests the _GitCall class (via git_command.git)."""
 

tests/test_git_superproject.py

@@ -14,19 +14,28 @@
 
 """Unittests for the git_superproject.py module."""
 
+import json
 import os
+import platform
 import tempfile
 import unittest
 from unittest import mock
 
 import git_superproject
+import git_trace2_event_log
 import manifest_xml
 import platform_utils
+from test_manifest_xml import sort_attributes
 
 
 class SuperprojectTestCase(unittest.TestCase):
   """TestCase for the Superproject module."""
 
+  PARENT_SID_KEY = 'GIT_TRACE2_PARENT_SID'
+  PARENT_SID_VALUE = 'parent_sid'
+  SELF_SID_REGEX = r'repo-\d+T\d+Z-.*'
+  FULL_SID_REGEX = r'^%s/%s' % (PARENT_SID_VALUE, SELF_SID_REGEX)
+
   def setUp(self):
     """Set up superproject every time."""
     self.tempdir = tempfile.mkdtemp(prefix='repo_tests')
@@ -34,6 +43,14 @@ class SuperprojectTestCase(unittest.TestCase):
     self.manifest_file = os.path.join(
         self.repodir, manifest_xml.MANIFEST_FILE_NAME)
     os.mkdir(self.repodir)
+    self.platform = platform.system().lower()
+
+    # By default we initialize with the expected case where
+    # repo launches us (so GIT_TRACE2_PARENT_SID is set).
+    env = {
+        self.PARENT_SID_KEY: self.PARENT_SID_VALUE,
+    }
+    self.git_event_log = git_trace2_event_log.EventLog(env=env)
 
     # The manifest parsing really wants a git repo currently.
     gitdir = os.path.join(self.repodir, 'manifests.git')
@@ -48,10 +65,11 @@ class SuperprojectTestCase(unittest.TestCase):
   <remote name="default-remote" fetch="http://localhost" />
   <default remote="default-remote" revision="refs/heads/main" />
   <superproject name="superproject"/>
-  <project path="art" name="platform/art" />
-</manifest>
+  <project path="art" name="platform/art" groups="notdefault,platform-""" + self.platform + """
+  " /></manifest>
 """)
-    self._superproject = git_superproject.Superproject(manifest, self.repodir)
+    self._superproject = git_superproject.Superproject(manifest, self.repodir,
+                                                       self.git_event_log)
 
   def tearDown(self):
     """Tear down superproject every time."""
@@ -63,14 +81,56 @@ class SuperprojectTestCase(unittest.TestCase):
       fp.write(data)
     return manifest_xml.XmlManifest(self.repodir, self.manifest_file)
 
+  def verifyCommonKeys(self, log_entry, expected_event_name, full_sid=True):
+    """Helper function to verify common event log keys."""
+    self.assertIn('event', log_entry)
+    self.assertIn('sid', log_entry)
+    self.assertIn('thread', log_entry)
+    self.assertIn('time', log_entry)
+
+    # Do basic data format validation.
+    self.assertEqual(expected_event_name, log_entry['event'])
+    if full_sid:
+      self.assertRegex(log_entry['sid'], self.FULL_SID_REGEX)
+    else:
+      self.assertRegex(log_entry['sid'], self.SELF_SID_REGEX)
+    self.assertRegex(log_entry['time'], r'^\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z$')
+
+  def readLog(self, log_path):
+    """Helper function to read log data into a list."""
+    log_data = []
+    with open(log_path, mode='rb') as f:
+      for line in f:
+        log_data.append(json.loads(line))
+    return log_data
+
+  def verifyErrorEvent(self):
+    """Helper to verify that error event is written."""
+
+    with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
+      log_path = self.git_event_log.Write(path=tempdir)
+      self.log_data = self.readLog(log_path)
+
+    self.assertEqual(len(self.log_data), 2)
+    error_event = self.log_data[1]
+    self.verifyCommonKeys(self.log_data[0], expected_event_name='version')
+    self.verifyCommonKeys(error_event, expected_event_name='error')
+    # Check for 'error' event specific fields.
+    self.assertIn('msg', error_event)
+    self.assertIn('fmt', error_event)
+
   def test_superproject_get_superproject_no_superproject(self):
     """Test with no url."""
     manifest = self.getXmlManifest("""
 <manifest>
 </manifest>
 """)
-    superproject = git_superproject.Superproject(manifest, self.repodir)
-    self.assertFalse(superproject.Sync())
+    superproject = git_superproject.Superproject(manifest, self.repodir, self.git_event_log)
+    # Test that exit condition is false when there is no superproject tag.
+    sync_result = superproject.Sync()
+    self.assertFalse(sync_result.success)
+    self.assertFalse(sync_result.fatal)
+    self.verifyErrorEvent()
 
   def test_superproject_get_superproject_invalid_url(self):
     """Test with an invalid url."""
@@ -81,8 +141,10 @@ class SuperprojectTestCase(unittest.TestCase):
   <superproject name="superproject"/>
 </manifest>
 """)
-    superproject = git_superproject.Superproject(manifest, self.repodir)
-    self.assertFalse(superproject.Sync())
+    superproject = git_superproject.Superproject(manifest, self.repodir, self.git_event_log)
+    sync_result = superproject.Sync()
+    self.assertFalse(sync_result.success)
+    self.assertTrue(sync_result.fatal)
 
   def test_superproject_get_superproject_invalid_branch(self):
     """Test with an invalid branch."""
@@ -93,21 +155,28 @@ class SuperprojectTestCase(unittest.TestCase):
   <superproject name="superproject"/>
 </manifest>
 """)
-    superproject = git_superproject.Superproject(manifest, self.repodir)
+    self._superproject = git_superproject.Superproject(manifest, self.repodir,
+                                                       self.git_event_log)
     with mock.patch.object(self._superproject, '_GetBranch', return_value='junk'):
-      self.assertFalse(superproject.Sync())
+      sync_result = self._superproject.Sync()
+      self.assertFalse(sync_result.success)
+      self.assertTrue(sync_result.fatal)
 
   def test_superproject_get_superproject_mock_init(self):
     """Test with _Init failing."""
     with mock.patch.object(self._superproject, '_Init', return_value=False):
-      self.assertFalse(self._superproject.Sync())
+      sync_result = self._superproject.Sync()
+      self.assertFalse(sync_result.success)
+      self.assertTrue(sync_result.fatal)
 
   def test_superproject_get_superproject_mock_fetch(self):
     """Test with _Fetch failing."""
     with mock.patch.object(self._superproject, '_Init', return_value=True):
       os.mkdir(self._superproject._superproject_path)
       with mock.patch.object(self._superproject, '_Fetch', return_value=False):
-        self.assertFalse(self._superproject.Sync())
+        sync_result = self._superproject.Sync()
+        self.assertFalse(sync_result.success)
+        self.assertTrue(sync_result.fatal)
 
   def test_superproject_get_all_project_commit_ids_mock_ls_tree(self):
     """Test with LsTree being a mock."""
@@ -119,12 +188,13 @@ class SuperprojectTestCase(unittest.TestCase):
     with mock.patch.object(self._superproject, '_Init', return_value=True):
       with mock.patch.object(self._superproject, '_Fetch', return_value=True):
         with mock.patch.object(self._superproject, '_LsTree', return_value=data):
-          commit_ids = self._superproject._GetAllProjectsCommitIds()
-          self.assertEqual(commit_ids, {
+          commit_ids_result = self._superproject._GetAllProjectsCommitIds()
+          self.assertEqual(commit_ids_result.commit_ids, {
               'art': '2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea',
               'bootable/recovery': 'e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06',
               'build/bazel': 'ade9b7a0d874e25fff4bf2552488825c6f111928'
           })
+          self.assertFalse(commit_ids_result.fatal)
 
   def test_superproject_write_manifest_file(self):
     """Test with writing manifest to a file after setting revisionId."""
@@ -136,14 +206,15 @@ class SuperprojectTestCase(unittest.TestCase):
     manifest_path = self._superproject._WriteManfiestFile()
     self.assertIsNotNone(manifest_path)
     with open(manifest_path, 'r') as fp:
-      manifest_xml = fp.read()
+      manifest_xml_data = fp.read()
     self.assertEqual(
-        manifest_xml,
-        '<?xml version="1.0" ?><manifest>' +
-        '<remote name="default-remote" fetch="http://localhost"/>' +
-        '<default remote="default-remote" revision="refs/heads/main"/>' +
-        '<project name="platform/art" path="art" revision="ABCDEF"/>' +
-        '<superproject name="superproject"/>' +
+        sort_attributes(manifest_xml_data),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch="http://localhost" name="default-remote"/>'
+        '<default remote="default-remote" revision="refs/heads/main"/>'
+        '<project groups="notdefault,platform-' + self.platform + '" '
+        'name="platform/art" path="art" revision="ABCDEF" upstream="refs/heads/main"/>'
+        '<superproject name="superproject"/>'
         '</manifest>')
 
   def test_superproject_update_project_revision_id(self):
@@ -159,18 +230,145 @@ class SuperprojectTestCase(unittest.TestCase):
                                return_value=data):
           # Create temporary directory so that it can write the file.
           os.mkdir(self._superproject._superproject_path)
-          manifest_path = self._superproject.UpdateProjectsRevisionId(projects)
-          self.assertIsNotNone(manifest_path)
-          with open(manifest_path, 'r') as fp:
-            manifest_xml = fp.read()
+          update_result = self._superproject.UpdateProjectsRevisionId(projects)
+          self.assertIsNotNone(update_result.manifest_path)
+          self.assertFalse(update_result.fatal)
+          with open(update_result.manifest_path, 'r') as fp:
+            manifest_xml_data = fp.read()
           self.assertEqual(
-              manifest_xml,
-              '<?xml version="1.0" ?><manifest>' +
-              '<remote name="default-remote" fetch="http://localhost"/>' +
-              '<default remote="default-remote" revision="refs/heads/main"/>' +
-              '<project name="platform/art" path="art" ' +
-              'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea"/>' +
-              '<superproject name="superproject"/>' +
+              sort_attributes(manifest_xml_data),
+              '<?xml version="1.0" ?><manifest>'
+              '<remote fetch="http://localhost" name="default-remote"/>'
+              '<default remote="default-remote" revision="refs/heads/main"/>'
+              '<project groups="notdefault,platform-' + self.platform + '" '
+              'name="platform/art" path="art" '
+              'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea" upstream="refs/heads/main"/>'
+              '<superproject name="superproject"/>'
+              '</manifest>')
+
+  def test_superproject_update_project_revision_id_no_superproject_tag(self):
+    """Test update of commit ids of a manifest without superproject tag."""
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="test-name"/>
+</manifest>
+""")
+    self.maxDiff = None
+    self._superproject = git_superproject.Superproject(manifest, self.repodir,
+                                                       self.git_event_log)
+    self.assertEqual(len(self._superproject._manifest.projects), 1)
+    projects = self._superproject._manifest.projects
+    project = projects[0]
+    project.SetRevisionId('ABCDEF')
+    update_result = self._superproject.UpdateProjectsRevisionId(projects)
+    self.assertIsNone(update_result.manifest_path)
+    self.assertFalse(update_result.fatal)
+    self.verifyErrorEvent()
+    self.assertEqual(
+        sort_attributes(manifest.ToXml().toxml()),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch="http://localhost" name="default-remote"/>'
+        '<default remote="default-remote" revision="refs/heads/main"/>'
+        '<project name="test-name" revision="ABCDEF" upstream="refs/heads/main"/>'
+        '</manifest>')
+
+  def test_superproject_update_project_revision_id_from_local_manifest_group(self):
+    """Test update of commit ids of a manifest that have local manifest no superproject group."""
+    local_group = manifest_xml.LOCAL_MANIFEST_GROUP_PREFIX + ':local'
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <remote name="goog" fetch="http://localhost2" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <superproject name="superproject"/>
+  <project path="vendor/x" name="platform/vendor/x" remote="goog"
+           groups=\"""" + local_group + """
+         " revision="master-with-vendor" clone-depth="1" />
+  <project path="art" name="platform/art" groups="notdefault,platform-""" + self.platform + """
+  " /></manifest>
+""")
+    self.maxDiff = None
+    self._superproject = git_superproject.Superproject(manifest, self.repodir,
+                                                       self.git_event_log)
+    self.assertEqual(len(self._superproject._manifest.projects), 2)
+    projects = self._superproject._manifest.projects
+    data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00')
+    with mock.patch.object(self._superproject, '_Init', return_value=True):
+      with mock.patch.object(self._superproject, '_Fetch', return_value=True):
+        with mock.patch.object(self._superproject,
+                               '_LsTree',
+                               return_value=data):
+          # Create temporary directory so that it can write the file.
+          os.mkdir(self._superproject._superproject_path)
+          update_result = self._superproject.UpdateProjectsRevisionId(projects)
+          self.assertIsNotNone(update_result.manifest_path)
+          self.assertFalse(update_result.fatal)
+          with open(update_result.manifest_path, 'r') as fp:
+            manifest_xml_data = fp.read()
+          # Verify platform/vendor/x's project revision hasn't changed.
+          self.assertEqual(
+              sort_attributes(manifest_xml_data),
+              '<?xml version="1.0" ?><manifest>'
+              '<remote fetch="http://localhost" name="default-remote"/>'
+              '<remote fetch="http://localhost2" name="goog"/>'
+              '<default remote="default-remote" revision="refs/heads/main"/>'
+              '<project groups="notdefault,platform-' + self.platform + '" '
+              'name="platform/art" path="art" '
+              'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea" upstream="refs/heads/main"/>'
+              '<project clone-depth="1" groups="' + local_group + '" '
+              'name="platform/vendor/x" path="vendor/x" remote="goog" '
+              'revision="master-with-vendor"/>'
+              '<superproject name="superproject"/>'
+              '</manifest>')
+
+  def test_superproject_update_project_revision_id_with_pinned_manifest(self):
+    """Test update of commit ids of a pinned manifest."""
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <superproject name="superproject"/>
+  <project path="vendor/x" name="platform/vendor/x" revision="" />
+  <project path="vendor/y" name="platform/vendor/y"
+           revision="52d3c9f7c107839ece2319d077de0cd922aa9d8f" />
+  <project path="art" name="platform/art" groups="notdefault,platform-""" + self.platform + """
+  " /></manifest>
+""")
+    self.maxDiff = None
+    self._superproject = git_superproject.Superproject(manifest, self.repodir,
+                                                       self.git_event_log)
+    self.assertEqual(len(self._superproject._manifest.projects), 3)
+    projects = self._superproject._manifest.projects
+    data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00'
+            '160000 commit e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06\tvendor/x\x00')
+    with mock.patch.object(self._superproject, '_Init', return_value=True):
+      with mock.patch.object(self._superproject, '_Fetch', return_value=True):
+        with mock.patch.object(self._superproject,
+                               '_LsTree',
+                               return_value=data):
+          # Create temporary directory so that it can write the file.
+          os.mkdir(self._superproject._superproject_path)
+          update_result = self._superproject.UpdateProjectsRevisionId(projects)
+          self.assertIsNotNone(update_result.manifest_path)
+          self.assertFalse(update_result.fatal)
+          with open(update_result.manifest_path, 'r') as fp:
+            manifest_xml_data = fp.read()
+          # Verify platform/vendor/x's project revision hasn't changed.
+          self.assertEqual(
+              sort_attributes(manifest_xml_data),
+              '<?xml version="1.0" ?><manifest>'
+              '<remote fetch="http://localhost" name="default-remote"/>'
+              '<default remote="default-remote" revision="refs/heads/main"/>'
+              '<project groups="notdefault,platform-' + self.platform + '" '
+              'name="platform/art" path="art" '
+              'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea" upstream="refs/heads/main"/>'
+              '<project name="platform/vendor/x" path="vendor/x" '
+              'revision="e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06" upstream="refs/heads/main"/>'
+              '<project name="platform/vendor/y" path="vendor/y" '
+              'revision="52d3c9f7c107839ece2319d077de0cd922aa9d8f"/>'
+              '<superproject name="superproject"/>'
               '</manifest>')
 
 

tests/test_git_trace2_event_log.py

@@ -161,6 +161,103 @@ class EventLogTestCase(unittest.TestCase):
     self.assertIn('code', exit_event)
     self.assertEqual(exit_event['code'], 2)
 
+  def test_command_event(self):
+    """Test and validate 'command' event data is valid.
+
+    Expected event log:
+    <version event>
+    <command event>
+    """
+    name = 'repo'
+    subcommands = ['init' 'this']
+    self._event_log_module.CommandEvent(name='repo', subcommands=subcommands)
+    with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
+      log_path = self._event_log_module.Write(path=tempdir)
+      self._log_data = self.readLog(log_path)
+
+    self.assertEqual(len(self._log_data), 2)
+    command_event = self._log_data[1]
+    self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
+    self.verifyCommonKeys(command_event, expected_event_name='command')
+    # Check for 'command' event specific fields.
+    self.assertIn('name', command_event)
+    self.assertIn('subcommands', command_event)
+    self.assertEqual(command_event['name'], name)
+    self.assertEqual(command_event['subcommands'], subcommands)
+
+  def test_def_params_event_repo_config(self):
+    """Test 'def_params' event data outputs only repo config keys.
+
+    Expected event log:
+    <version event>
+    <def_param event>
+    <def_param event>
+    """
+    config = {
+        'git.foo': 'bar',
+        'repo.partialclone': 'true',
+        'repo.partialclonefilter': 'blob:none',
+    }
+    self._event_log_module.DefParamRepoEvents(config)
+
+    with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
+      log_path = self._event_log_module.Write(path=tempdir)
+      self._log_data = self.readLog(log_path)
+
+    self.assertEqual(len(self._log_data), 3)
+    def_param_events = self._log_data[1:]
+    self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
+
+    for event in def_param_events:
+      self.verifyCommonKeys(event, expected_event_name='def_param')
+      # Check for 'def_param' event specific fields.
+      self.assertIn('param', event)
+      self.assertIn('value', event)
+      self.assertTrue(event['param'].startswith('repo.'))
+
+  def test_def_params_event_no_repo_config(self):
+    """Test 'def_params' event data won't output non-repo config keys.
+
+    Expected event log:
+    <version event>
+    """
+    config = {
+        'git.foo': 'bar',
+        'git.core.foo2': 'baz',
+    }
+    self._event_log_module.DefParamRepoEvents(config)
+
+    with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
+      log_path = self._event_log_module.Write(path=tempdir)
+      self._log_data = self.readLog(log_path)
+
+    self.assertEqual(len(self._log_data), 1)
+    self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
+
+  def test_error_event(self):
+    """Test and validate 'error' event data is valid.
+
+    Expected event log:
+    <version event>
+    <error event>
+    """
+    msg = 'invalid option: --cahced'
+    fmt = 'invalid option: %s'
+    self._event_log_module.ErrorEvent(msg, fmt)
+    with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
+      log_path = self._event_log_module.Write(path=tempdir)
+      self._log_data = self.readLog(log_path)
+
+    self.assertEqual(len(self._log_data), 2)
+    error_event = self._log_data[1]
+    self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
+    self.verifyCommonKeys(error_event, expected_event_name='error')
+    # Check for 'error' event specific fields.
+    self.assertIn('msg', error_event)
+    self.assertIn('fmt', error_event)
+    self.assertEqual(error_event['msg'], msg)
+    self.assertEqual(error_event['fmt'], fmt)
+
   def test_write_with_filename(self):
     """Test Write() with a path to a file exits with None."""
     self.assertIsNone(self._event_log_module.Write(path='path/to/file'))

tests/test_manifest_xml.py

@@ -15,6 +15,8 @@
 """Unittests for the manifest_xml.py module."""
 
 import os
+import platform
+import re
 import shutil
 import tempfile
 import unittest
@@ -31,6 +33,7 @@ INVALID_FS_PATHS = (
     '..',
     '../',
     './',
+    './/',
     'foo/',
     './foo',
     '../foo',
@@ -50,6 +53,9 @@ INVALID_FS_PATHS = (
     'blah/foo~',
     # Block Unicode characters that get normalized out by filesystems.
     u'foo\u200Cbar',
+    # Block newlines.
+    'f\n/bar',
+    'f\r/bar',
 )
 
 # Make sure platforms that use path separators (e.g. Windows) are also
@@ -58,6 +64,30 @@ if os.path.sep != '/':
   INVALID_FS_PATHS += tuple(x.replace('/', os.path.sep) for x in INVALID_FS_PATHS)
 
 
+def sort_attributes(manifest):
+  """Sort the attributes of all elements alphabetically.
+
+  This is needed because different versions of the toxml() function from
+  xml.dom.minidom outputs the attributes of elements in different orders.
+  Before Python 3.8 they were output alphabetically, later versions preserve
+  the order specified by the user.
+
+  Args:
+    manifest: String containing an XML manifest.
+
+  Returns:
+    The XML manifest with the attributes of all elements sorted alphabetically.
+  """
+  new_manifest = ''
+  # This will find every element in the XML manifest, whether they have
+  # attributes or not. This simplifies recreating the manifest below.
+  matches = re.findall(r'(<[/?]?[a-z-]+\s*)((?:\S+?="[^"]+"\s*?)*)(\s*[/?]?>)', manifest)
+  for head, attrs, tail in matches:
+    m = re.findall(r'\S+?="[^"]+"', attrs)
+    new_manifest += head + ' '.join(sorted(m)) + tail
+  return new_manifest
+
+
 class ManifestParseTestCase(unittest.TestCase):
   """TestCase for parsing manifests."""
 
@@ -89,6 +119,11 @@ class ManifestParseTestCase(unittest.TestCase):
       fp.write(data)
     return manifest_xml.XmlManifest(self.repodir, self.manifest_file)
 
+  @staticmethod
+  def encodeXmlAttr(attr):
+    """Encode |attr| using XML escape rules."""
+    return attr.replace('\r', '&#x000d;').replace('\n', '&#x000a;')
+
 
 class ManifestValidateFilePaths(unittest.TestCase):
   """Check _ValidateFilePaths helper.
@@ -244,11 +279,11 @@ class XmlManifestTests(ManifestParseTestCase):
     self.assertEqual(manifest.superproject['name'], 'superproject')
     self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
     self.assertEqual(
-        manifest.ToXml().toxml(),
-        '<?xml version="1.0" ?><manifest>' +
-        '<remote name="test-remote" fetch="http://localhost"/>' +
-        '<default remote="test-remote" revision="refs/heads/main"/>' +
-        '<superproject name="superproject"/>' +
+        sort_attributes(manifest.ToXml().toxml()),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch="http://localhost" name="test-remote"/>'
+        '<default remote="test-remote" revision="refs/heads/main"/>'
+        '<superproject name="superproject"/>'
         '</manifest>')
 
 
@@ -301,6 +336,7 @@ class IncludeElementTests(ManifestParseTestCase):
   def test_allow_bad_name_from_user(self):
     """Check handling of bad name attribute from the user's input."""
     def parse(name):
+      name = self.encodeXmlAttr(name)
       manifest = self.getXmlManifest(f"""
 <manifest>
   <remote name="default-remote" fetch="http://localhost" />
@@ -325,6 +361,7 @@ class IncludeElementTests(ManifestParseTestCase):
   def test_bad_name_checks(self):
     """Check handling of bad name attribute."""
     def parse(name):
+      name = self.encodeXmlAttr(name)
       # Setup target of the include.
       with open(os.path.join(self.manifest_dir, 'target.xml'), 'w') as fp:
         fp.write(f'<manifest><include name="{name}"/></manifest>')
@@ -377,6 +414,11 @@ class ProjectElementTests(ManifestParseTestCase):
     self.assertCountEqual(
         result['extras'],
         ['g1', 'g2', 'g1', 'name:extras', 'all', 'path:path'])
+    groupstr = 'default,platform-' + platform.system().lower()
+    self.assertEqual(groupstr, manifest.GetGroupsStr())
+    groupstr = 'g1,g2,g1'
+    manifest.manifestProject.config.SetString('manifest.groups', groupstr)
+    self.assertEqual(groupstr, manifest.GetGroupsStr())
 
   def test_set_revision_id(self):
     """Check setting of project's revisionId."""
@@ -391,16 +433,18 @@ class ProjectElementTests(ManifestParseTestCase):
     project = manifest.projects[0]
     project.SetRevisionId('ABCDEF')
     self.assertEqual(
-        manifest.ToXml().toxml(),
-        '<?xml version="1.0" ?><manifest>' +
-        '<remote name="default-remote" fetch="http://localhost"/>' +
-        '<default remote="default-remote" revision="refs/heads/main"/>' +
-        '<project name="test-name" revision="ABCDEF"/>' +
+        sort_attributes(manifest.ToXml().toxml()),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch="http://localhost" name="default-remote"/>'
+        '<default remote="default-remote" revision="refs/heads/main"/>'
+        '<project name="test-name" revision="ABCDEF" upstream="refs/heads/main"/>'
         '</manifest>')
 
   def test_trailing_slash(self):
     """Check handling of trailing slashes in attributes."""
     def parse(name, path):
+      name = self.encodeXmlAttr(name)
+      path = self.encodeXmlAttr(path)
       return self.getXmlManifest(f"""
 <manifest>
   <remote name="default-remote" fetch="http://localhost" />
@@ -421,9 +465,35 @@ class ProjectElementTests(ManifestParseTestCase):
     self.assertEqual(manifest.projects[0].objdir,
                      os.path.join(self.tempdir, '.repo/project-objects/a/path.git'))
 
+    manifest = parse('a/path', 'foo//////')
+    self.assertEqual(manifest.projects[0].gitdir,
+                     os.path.join(self.tempdir, '.repo/projects/foo.git'))
+    self.assertEqual(manifest.projects[0].objdir,
+                     os.path.join(self.tempdir, '.repo/project-objects/a/path.git'))
+
+  def test_toplevel_path(self):
+    """Check handling of path=. specially."""
+    def parse(name, path):
+      name = self.encodeXmlAttr(name)
+      path = self.encodeXmlAttr(path)
+      return self.getXmlManifest(f"""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="{name}" path="{path}" />
+</manifest>
+""")
+
+    for path in ('.', './', './/', './//'):
+      manifest = parse('server/path', path)
+      self.assertEqual(manifest.projects[0].gitdir,
+                       os.path.join(self.tempdir, '.repo/projects/..git'))
+
   def test_bad_path_name_checks(self):
     """Check handling of bad path & name attributes."""
     def parse(name, path):
+      name = self.encodeXmlAttr(name)
+      path = self.encodeXmlAttr(path)
       manifest = self.getXmlManifest(f"""
 <manifest>
   <remote name="default-remote" fetch="http://localhost" />
@@ -448,8 +518,11 @@ class ProjectElementTests(ManifestParseTestCase):
 
       with self.assertRaises(error.ManifestInvalidPathError):
         parse(path, 'ok')
-      with self.assertRaises(error.ManifestInvalidPathError):
-        parse('ok', path)
+
+      # We have a dedicated test for path=".".
+      if path not in {'.'}:
+        with self.assertRaises(error.ManifestInvalidPathError):
+          parse('ok', path)
 
 
 class SuperProjectElementTests(ManifestParseTestCase):
@@ -468,11 +541,11 @@ class SuperProjectElementTests(ManifestParseTestCase):
     self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
     self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
     self.assertEqual(
-        manifest.ToXml().toxml(),
-        '<?xml version="1.0" ?><manifest>' +
-        '<remote name="test-remote" fetch="http://localhost"/>' +
-        '<default remote="test-remote" revision="refs/heads/main"/>' +
-        '<superproject name="superproject"/>' +
+        sort_attributes(manifest.ToXml().toxml()),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch="http://localhost" name="test-remote"/>'
+        '<default remote="test-remote" revision="refs/heads/main"/>'
+        '<superproject name="superproject"/>'
         '</manifest>')
 
   def test_remote(self):
@@ -489,12 +562,12 @@ class SuperProjectElementTests(ManifestParseTestCase):
     self.assertEqual(manifest.superproject['remote'].name, 'superproject-remote')
     self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/platform/superproject')
     self.assertEqual(
-        manifest.ToXml().toxml(),
-        '<?xml version="1.0" ?><manifest>' +
-        '<remote name="default-remote" fetch="http://localhost"/>' +
-        '<remote name="superproject-remote" fetch="http://localhost"/>' +
-        '<default remote="default-remote" revision="refs/heads/main"/>' +
-        '<superproject name="platform/superproject" remote="superproject-remote"/>' +
+        sort_attributes(manifest.ToXml().toxml()),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch="http://localhost" name="default-remote"/>'
+        '<remote fetch="http://localhost" name="superproject-remote"/>'
+        '<default remote="default-remote" revision="refs/heads/main"/>'
+        '<superproject name="platform/superproject" remote="superproject-remote"/>'
         '</manifest>')
 
   def test_defalut_remote(self):
@@ -509,9 +582,109 @@ class SuperProjectElementTests(ManifestParseTestCase):
     self.assertEqual(manifest.superproject['name'], 'superproject')
     self.assertEqual(manifest.superproject['remote'].name, 'default-remote')
     self.assertEqual(
-        manifest.ToXml().toxml(),
-        '<?xml version="1.0" ?><manifest>' +
-        '<remote name="default-remote" fetch="http://localhost"/>' +
-        '<default remote="default-remote" revision="refs/heads/main"/>' +
-        '<superproject name="superproject"/>' +
+        sort_attributes(manifest.ToXml().toxml()),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch="http://localhost" name="default-remote"/>'
+        '<default remote="default-remote" revision="refs/heads/main"/>'
+        '<superproject name="superproject"/>'
         '</manifest>')
+
+
+class ContactinfoElementTests(ManifestParseTestCase):
+  """Tests for <contactinfo>."""
+
+  def test_contactinfo(self):
+    """Check contactinfo settings."""
+    bugurl = 'http://localhost/contactinfo'
+    manifest = self.getXmlManifest(f"""
+<manifest>
+  <contactinfo bugurl="{bugurl}"/>
+</manifest>
+""")
+    self.assertEqual(manifest.contactinfo.bugurl, bugurl)
+    self.assertEqual(
+        manifest.ToXml().toxml(),
+        '<?xml version="1.0" ?><manifest>'
+        f'<contactinfo bugurl="{bugurl}"/>'
+        '</manifest>')
+
+
+class DefaultElementTests(ManifestParseTestCase):
+  """Tests for <default>."""
+
+  def test_default(self):
+    """Check default settings."""
+    a = manifest_xml._Default()
+    a.revisionExpr = 'foo'
+    a.remote = manifest_xml._XmlRemote(name='remote')
+    b = manifest_xml._Default()
+    b.revisionExpr = 'bar'
+    self.assertEqual(a, a)
+    self.assertNotEqual(a, b)
+    self.assertNotEqual(b, a.remote)
+    self.assertNotEqual(a, 123)
+    self.assertNotEqual(a, None)
+
+
+class RemoteElementTests(ManifestParseTestCase):
+  """Tests for <remote>."""
+
+  def test_remote(self):
+    """Check remote settings."""
+    a = manifest_xml._XmlRemote(name='foo')
+    b = manifest_xml._XmlRemote(name='bar')
+    self.assertEqual(a, a)
+    self.assertNotEqual(a, b)
+    self.assertNotEqual(a, manifest_xml._Default())
+    self.assertNotEqual(a, 123)
+    self.assertNotEqual(a, None)
+
+
+class RemoveProjectElementTests(ManifestParseTestCase):
+  """Tests for <remove-project>."""
+
+  def test_remove_one_project(self):
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="myproject" />
+  <remove-project name="myproject" />
+</manifest>
+""")
+    self.assertEqual(manifest.projects, [])
+
+  def test_remove_one_project_one_remains(self):
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="myproject" />
+  <project name="yourproject" />
+  <remove-project name="myproject" />
+</manifest>
+""")
+
+    self.assertEqual(len(manifest.projects), 1)
+    self.assertEqual(manifest.projects[0].name, 'yourproject')
+
+  def test_remove_one_project_doesnt_exist(self):
+    with self.assertRaises(manifest_xml.ManifestParseError):
+      manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <remove-project name="myproject" />
+</manifest>
+""")
+      manifest.projects
+
+  def test_remove_one_optional_project_doesnt_exist(self):
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <remove-project name="myproject" optional="true" />
+</manifest>
+""")
+    self.assertEqual(manifest.projects, [])

tests/test_project.py

@@ -46,7 +46,7 @@ def TempGitTree():
       templatedir = tempfile.mkdtemp(prefix='.test-template')
       with open(os.path.join(templatedir, 'HEAD'), 'w') as fp:
         fp.write('ref: refs/heads/main\n')
-      cmd += ['--template=', templatedir]
+      cmd += ['--template', templatedir]
     subprocess.check_call(cmd, cwd=tempdir)
     yield tempdir
   finally:

tests/test_ssh.py

@@ -0,0 +1,74 @@
+# Copyright 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for the ssh.py module."""
+
+import multiprocessing
+import subprocess
+import unittest
+from unittest import mock
+
+import ssh
+
+
+class SshTests(unittest.TestCase):
+  """Tests the ssh functions."""
+
+  def test_parse_ssh_version(self):
+    """Check _parse_ssh_version() handling."""
+    ver = ssh._parse_ssh_version('Unknown\n')
+    self.assertEqual(ver, ())
+    ver = ssh._parse_ssh_version('OpenSSH_1.0\n')
+    self.assertEqual(ver, (1, 0))
+    ver = ssh._parse_ssh_version('OpenSSH_6.6.1p1 Ubuntu-2ubuntu2.13, OpenSSL 1.0.1f 6 Jan 2014\n')
+    self.assertEqual(ver, (6, 6, 1))
+    ver = ssh._parse_ssh_version('OpenSSH_7.6p1 Ubuntu-4ubuntu0.3, OpenSSL 1.0.2n  7 Dec 2017\n')
+    self.assertEqual(ver, (7, 6))
+
+  def test_version(self):
+    """Check version() handling."""
+    with mock.patch('ssh._run_ssh_version', return_value='OpenSSH_1.2\n'):
+      self.assertEqual(ssh.version(), (1, 2))
+
+  def test_context_manager_empty(self):
+    """Verify context manager with no clients works correctly."""
+    with multiprocessing.Manager() as manager:
+      with ssh.ProxyManager(manager):
+        pass
+
+  def test_context_manager_child_cleanup(self):
+    """Verify orphaned clients & masters get cleaned up."""
+    with multiprocessing.Manager() as manager:
+      with ssh.ProxyManager(manager) as ssh_proxy:
+        client = subprocess.Popen(['sleep', '964853320'])
+        ssh_proxy.add_client(client)
+        master = subprocess.Popen(['sleep', '964853321'])
+        ssh_proxy.add_master(master)
+    # If the process still exists, these will throw timeout errors.
+    client.wait(0)
+    master.wait(0)
+
+  def test_ssh_sock(self):
+    """Check sock() function."""
+    manager = multiprocessing.Manager()
+    proxy = ssh.ProxyManager(manager)
+    with mock.patch('tempfile.mkdtemp', return_value='/tmp/foo'):
+      # old ssh version uses port
+      with mock.patch('ssh.version', return_value=(6, 6)):
+        self.assertTrue(proxy.sock().endswith('%p'))
+
+      proxy._sock_path = None
+      # new ssh version uses hash
+      with mock.patch('ssh.version', return_value=(6, 7)):
+        self.assertTrue(proxy.sock().endswith('%C'))

tests/test_subcmds.py

@@ -14,6 +14,7 @@
 
 """Unittests for the subcmds module (mostly __init__.py than subcommands)."""
 
+import optparse
 import unittest
 
 import subcmds
@@ -41,3 +42,32 @@ class AllCommands(unittest.TestCase):
 
       # Reject internal python paths like "__init__".
       self.assertFalse(cmd.startswith('__'))
+
+  def test_help_desc_style(self):
+    """Force some consistency in option descriptions.
+
+    Python's optparse & argparse has a few default options like --help.  Their
+    option description text uses lowercase sentence fragments, so enforce our
+    options follow the same style so UI is consistent.
+
+    We enforce:
+    * Text starts with lowercase.
+    * Text doesn't end with period.
+    """
+    for name, cls in subcmds.all_commands.items():
+      cmd = cls()
+      parser = cmd.OptionParser
+      for option in parser.option_list:
+        if option.help == optparse.SUPPRESS_HELP:
+          continue
+
+        c = option.help[0]
+        self.assertEqual(
+            c.lower(), c,
+            msg=f'subcmds/{name}.py: {option.get_opt_string()}: help text '
+                f'should start with lowercase: "{option.help}"')
+
+        self.assertNotEqual(
+            option.help[-1], '.',
+            msg=f'subcmds/{name}.py: {option.get_opt_string()}: help text '
+                f'should not end in a period: "{option.help}"')

tox.ini

@@ -15,13 +15,14 @@
 # https://tox.readthedocs.io/
 
 [tox]
-envlist = py36, py37, py38
+envlist = py36, py37, py38, py39
 
 [gh-actions]
 python =
     3.6: py36
     3.7: py37
     3.8: py38
+    3.9: py39
 
 [testenv]
 deps = pytest