Fix Projects.shareable_dirs

If this tree is not using alternates for object sharing, then we need to
continue to call it a shared directory.

Bug: https://bugs.chromium.org/p/gerrit/issues/detail?id=15982
Test: manual
Change-Id: I1750f10b192504ac67f552222f8ddb9809d344fe
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/338974
Tested-by: LaMont Jones <lamontjones@google.com>
Reviewed-by: Mike Frysinger <vapier@google.com>

.gitreview

@@ -0,0 +1,5 @@
+[gerrit]
+host=gerrit-review.googlesource.com
+scheme=https
+project=git-repo.git
+defaultbranch=main

SUBMITTING_PATCHES.md

@@ -3,7 +3,7 @@
 # Short Version
 
  - Make small logical changes.
- - Provide a meaningful commit message.
+ - [Provide a meaningful commit message][commit-message-style].
  - Check for coding errors and style nits with flake8.
  - Make sure all code is under the Apache License, 2.0.
  - Publish your changes for review.
@@ -26,10 +26,11 @@ yourself with the following relevant bits.
 
 ## Make separate commits for logically separate changes.
 
-Unless your patch is really trivial, you should not be sending
-out a patch that was generated between your working tree and your
-commit head.  Instead, always make a commit with complete commit
-message and generate a series of patches from your repository.
+Unless your patch is really trivial, you should not be sending out a patch that
+was generated between your working tree and your commit head.
+Instead, always make a commit with a complete
+[commit message][commit-message-style] and generate a series of patches from
+your repository.
 It is a good discipline.
 
 Describe the technical detail of the change(s).
@@ -171,3 +172,6 @@ After you receive a Code-Review+2 from the maintainer, select the Verified
 button on the gerrit page for the change. This verifies that you have tested
 your changes and notifies the maintainer that they are ready to be submitted.
 The maintainer will then submit your changes to the repository.
+
+
+[commit-message-style]: https://chris.beams.io/posts/git-commit/

command.py

@@ -61,13 +61,21 @@ class Command(object):
   # it is the number of parallel jobs to default to.
   PARALLEL_JOBS = None
 
+  # Whether this command supports Multi-manifest.  If False, then main.py will
+  # iterate over the manifests and invoke the command once per (sub)manifest.
+  # This is only checked after calling ValidateOptions, so that partially
+  # migrated subcommands can set it to False.
+  MULTI_MANIFEST_SUPPORT = True
+
   def __init__(self, repodir=None, client=None, manifest=None, gitc_manifest=None,
-               git_event_log=None):
+               git_event_log=None, outer_client=None, outer_manifest=None):
     self.repodir = repodir
     self.client = client
+    self.outer_client = outer_client or client
     self.manifest = manifest
     self.gitc_manifest = gitc_manifest
     self.git_event_log = git_event_log
+    self.outer_manifest = outer_manifest
 
     # Cache for the OptionParser property.
     self._optparse = None
@@ -135,6 +143,17 @@ class Command(object):
           type=int, default=self.PARALLEL_JOBS,
           help=f'number of jobs to run in parallel (default: {default})')
 
+    m = p.add_option_group('Multi-manifest options')
+    m.add_option('--outer-manifest', action='store_true', default=None,
+                 help='operate starting at the outermost manifest')
+    m.add_option('--no-outer-manifest', dest='outer_manifest',
+                 action='store_false', help='do not operate on outer manifests')
+    m.add_option('--this-manifest-only', action='store_true', default=None,
+                 help='only operate on this (sub)manifest')
+    m.add_option('--no-this-manifest-only', '--all-manifests',
+                 dest='this_manifest_only', action='store_false',
+                 help='operate on this manifest and its submanifests')
+
   def _Options(self, p):
     """Initialize the option parser with subcommand-specific options."""
 
@@ -166,6 +185,10 @@ class Command(object):
     """Validate common options."""
     opt.quiet = opt.output_mode is False
     opt.verbose = opt.output_mode is True
+    if opt.outer_manifest is None:
+      # By default, treat multi-manifest instances as a single manifest from
+      # the user's perspective.
+      opt.outer_manifest = True
 
   def ValidateOptions(self, opt, args):
     """Validate the user options & arguments before executing.
@@ -252,15 +275,30 @@ class Command(object):
     return project
 
   def GetProjects(self, args, manifest=None, groups='', missing_ok=False,
-                  submodules_ok=False):
+                  submodules_ok=False, all_manifests=False):
     """A list of projects that match the arguments.
-    """
-    if not manifest:
-      manifest = self.manifest
-    all_projects_list = manifest.projects
-    result = []
 
-    mp = manifest.manifestProject
+    Args:
+      args: a list of (case-insensitive) strings, projects to search for.
+      manifest: an XmlManifest, the manifest to use, or None for default.
+      groups: a string, the manifest groups in use.
+      missing_ok: a boolean, whether to allow missing projects.
+      submodules_ok: a boolean, whether to allow submodules.
+      all_manifests: a boolean, if True then all manifests and submanifests are
+                     used.  If False, then only the local (sub)manifest is used.
+
+    Returns:
+      A list of matching Project instances.
+    """
+    if all_manifests:
+      if not manifest:
+        manifest = self.manifest.outer_client
+      all_projects_list = manifest.all_projects
+    else:
+      if not manifest:
+        manifest = self.manifest
+      all_projects_list = manifest.projects
+    result = []
 
     if not groups:
       groups = manifest.GetGroupsStr()
@@ -282,12 +320,19 @@ class Command(object):
       for arg in args:
         # We have to filter by manifest groups in case the requested project is
         # checked out multiple times or differently based on them.
-        projects = [project for project in manifest.GetProjectsWithName(arg)
+        projects = [project for project in manifest.GetProjectsWithName(
+                        arg, all_manifests=all_manifests)
                     if project.MatchesGroups(groups)]
 
         if not projects:
           path = os.path.abspath(arg).replace('\\', '/')
-          project = self._GetProjectByPath(manifest, path)
+          tree = manifest
+          if all_manifests:
+            # Look for the deepest matching submanifest.
+            for tree in reversed(list(manifest.all_manifests)):
+              if path.startswith(tree.topdir):
+                break
+          project = self._GetProjectByPath(tree, path)
 
           # If it's not a derived project, update path->project mapping and
           # search again, as arg might actually point to a derived subproject.
@@ -308,7 +353,8 @@ class Command(object):
 
         for project in projects:
           if not missing_ok and not project.Exists:
-            raise NoSuchProjectError('%s (%s)' % (arg, project.relpath))
+            raise NoSuchProjectError('%s (%s)' % (
+                arg, project.RelPath(local=not all_manifests)))
           if not project.MatchesGroups(groups):
             raise InvalidProjectGroupsError(arg)
 
@@ -319,12 +365,22 @@ class Command(object):
     result.sort(key=_getpath)
     return result
 
-  def FindProjects(self, args, inverse=False):
+  def FindProjects(self, args, inverse=False, all_manifests=False):
+    """Find projects from command line arguments.
+
+    Args:
+      args: a list of (case-insensitive) strings, projects to search for.
+      inverse: a boolean, if True, then projects not matching any |args| are
+               returned.
+      all_manifests: a boolean, if True then all manifests and submanifests are
+                     used.  If False, then only the local (sub)manifest is used.
+    """
     result = []
     patterns = [re.compile(r'%s' % a, re.IGNORECASE) for a in args]
-    for project in self.GetProjects(''):
+    for project in self.GetProjects('', all_manifests=all_manifests):
+      paths = [project.name, project.RelPath(local=not all_manifests)]
       for pattern in patterns:
-        match = pattern.search(project.name) or pattern.search(project.relpath)
+        match = any(pattern.search(x) for x in paths)
         if not inverse and match:
           result.append(project)
           break
@@ -333,9 +389,24 @@ class Command(object):
       else:
         if inverse:
           result.append(project)
-    result.sort(key=lambda project: project.relpath)
+    result.sort(key=lambda project: (project.manifest.path_prefix,
+                                     project.relpath))
     return result
 
+  def ManifestList(self, opt):
+    """Yields all of the manifests to traverse.
+
+    Args:
+      opt: The command options.
+    """
+    top = self.outer_manifest
+    if not opt.outer_manifest or opt.this_manifest_only:
+      top = self.manifest
+    yield top
+    if not opt.this_manifest_only:
+      for child in top.all_children:
+        yield child
+
 
 class InteractiveCommand(Command):
   """Command which requires user interaction on the tty and

docs/internal-fs-layout.md

@@ -50,6 +50,10 @@ For example, if you want to change the manifest branch, you can simply run
 For more documentation on the manifest format, including the local_manifests
 support, see the [manifest-format.md] file.
 
+*   `submanifests/{submanifest.path}/`: The path prefix to the manifest state of
+    a submanifest included in a multi-manifest checkout.  The outermost manifest
+    manifest state is found adjacent to `submanifests/`.
+
 *   `manifests/`: A git checkout of the manifest project.  Its `.git/` state
     points to the `manifest.git` bare checkout (see below).  It tracks the git
     branch specified at `repo init` time via `--manifest-branch`.
@@ -163,6 +167,7 @@ User controlled settings are initialized when running `repo init`.
 | repo.clonefilter         | `--clone-filter`          | Filter setting when using [partial git clones] |
 | repo.depth               | `--depth`                 | Create shallow checkouts when cloning |
 | repo.dissociate          | `--dissociate`            | Dissociate from any reference/mirrors after initial clone |
+| repo.git-lfs             | `--git-lfs`               | Enable [Git LFS] support |
 | repo.mirror              | `--mirror`                | Checkout is a repo mirror |
 | repo.partialclone        | `--partial-clone`         | Create [partial git clones] |
 | repo.partialcloneexclude | `--partial-clone-exclude` | Comma-delimited list of project names (not paths) to exclude while using [partial git clones] |
@@ -254,6 +259,7 @@ Repo will create & maintain a few files in the user's home directory.
 
 
 [git-config]: https://git-scm.com/docs/git-config
+[Git LFS]: https://git-lfs.github.com/
 [git worktree]: https://git-scm.com/docs/git-worktree
 [gitsubmodules]: https://git-scm.com/docs/gitsubmodules
 [manifest-format.md]: ./manifest-format.md

docs/manifest-format.md

@@ -26,6 +26,7 @@ following DTD:
                       remote*,
                       default?,
                       manifest-server?,
+                      submanifest*?,
                       remove-project*,
                       project*,
                       extend-project*,
@@ -57,6 +58,16 @@ following DTD:
   <!ELEMENT manifest-server EMPTY>
   <!ATTLIST manifest-server url CDATA #REQUIRED>
 
+  <!ELEMENT submanifest EMPTY>
+  <!ATTLIST submanifest name           ID #REQUIRED>
+  <!ATTLIST submanifest remote         IDREF #IMPLIED>
+  <!ATTLIST submanifest project        CDATA #IMPLIED>
+  <!ATTLIST submanifest manifest-name  CDATA #IMPLIED>
+  <!ATTLIST submanifest revision       CDATA #IMPLIED>
+  <!ATTLIST submanifest path           CDATA #IMPLIED>
+  <!ATTLIST submanifest groups         CDATA #IMPLIED>
+  <!ATTLIST submanifest default-groups CDATA #IMPLIED>
+
   <!ELEMENT project (annotation*,
                      project*,
                      copyfile*,
@@ -236,6 +247,66 @@ the specified tag. This is used by repo sync when the --smart-tag option
 is given.
 
 
+### Element submanifest
+
+One or more submanifest elements may be specified.  Each element describes a
+single manifest to be checked out as a child.
+
+Attribute `name`: A unique name (within the current (sub)manifest) for this
+submanifest. It acts as a default for `revision` below.  The same name can be
+used for submanifests with different parent (sub)manifests.
+
+Attribute `remote`: Name of a previously defined remote element.
+If not supplied the remote given by the default element is used.
+
+Attribute `project`: The manifest project name.  The project's name is appended
+onto its remote's fetch URL to generate the actual URL to configure the Git
+remote with.  The URL gets formed as:
+
+    ${remote_fetch}/${project_name}.git
+
+where ${remote_fetch} is the remote's fetch attribute and
+${project_name} is the project's name attribute.  The suffix ".git"
+is always appended as repo assumes the upstream is a forest of
+bare Git repositories.  If the project has a parent element, its
+name will be prefixed by the parent's.
+
+The project name must match the name Gerrit knows, if Gerrit is
+being used for code reviews.
+
+`project` must not be empty, and may not be an absolute path or use "." or ".."
+path components.  It is always interpreted relative to the remote's fetch
+settings, so if a different base path is needed, declare a different remote
+with the new settings needed.
+
+If not supplied the remote and project for this manifest will be used: `remote`
+cannot be supplied.
+
+Projects from a submanifest and its submanifests are added to the
+submanifest::path:<path_prefix> group.
+
+Attribute `manifest-name`: The manifest filename in the manifest project.  If
+not supplied, `default.xml` is used.
+
+Attribute `revision`: Name of a Git branch (e.g. "main" or "refs/heads/main"),
+tag (e.g. "refs/tags/stable"), or a commit hash.  If not supplied, `name` is
+used.
+
+Attribute `path`: An optional path relative to the top directory
+of the repo client where the submanifest repo client top directory
+should be placed.  If not supplied, `revision` is used.
+
+`path` may not be an absolute path or use "." or ".." path components.
+
+Attribute `groups`: List of additional groups to which all projects
+in the included submanifest belong. This appends and recurses, meaning
+all projects in submanifests carry all parent submanifest groups.
+Same syntax as the corresponding element of `project`.
+
+Attribute `default-groups`: The list of manifest groups to sync if no
+`--groups=` parameter was specified at init.  When that list is empty, use this
+list instead of "default" as the list of groups to sync.
+
 ### Element project
 
 One or more project elements may be specified.  Each element
@@ -471,7 +542,7 @@ These restrictions are not enforced for [Local Manifests].
 
 Attribute `groups`: List of additional groups to which all projects
 in the included manifest belong. This appends and recurses, meaning
-all projects in sub-manifests carry all parent include groups.
+all projects in included manifests carry all parent include groups.
 Same syntax as the corresponding element of `project`.
 
 ## Local Manifests {#local-manifests}

fetch.py

@@ -17,8 +17,10 @@
 import subprocess
 import sys
 from urllib.parse import urlparse
+from urllib.request import urlopen
 
-def fetch_file(url):
+
+def fetch_file(url, verbose=False):
   """Fetch a file from the specified source using the appropriate protocol.
 
   Returns:
@@ -29,13 +31,15 @@ def fetch_file(url):
     cmd = ['gsutil', 'cat', url]
     try:
       result = subprocess.run(
-          cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+          cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+          check=True)
+      if result.stderr and verbose:
+        print('warning: non-fatal error running "gsutil": %s' % result.stderr,
+              file=sys.stderr)
       return result.stdout
     except subprocess.CalledProcessError as e:
-      print('fatal: error running "gsutil": %s' % e.output,
+      print('fatal: error running "gsutil": %s' % e.stderr,
             file=sys.stderr)
     sys.exit(1)
-  if scheme == 'file':
-    with open(url[len('file://'):], 'rb') as f:
-      return f.read()
-  raise ValueError('unsupported url %s' % url)
+  with urlopen(url) as f:
+    return f.read()

git_command.py

@@ -169,7 +169,8 @@ class GitCommand(object):
                disable_editor=False,
                ssh_proxy=None,
                cwd=None,
-               gitdir=None):
+               gitdir=None,
+               objdir=None):
     env = self._GetBasicEnv()
 
     if disable_editor:
@@ -194,13 +195,24 @@ class GitCommand(object):
         cwd = project.worktree
       if not gitdir:
         gitdir = project.gitdir
+    # Git on Windows wants its paths only using / for reliability.
+    if platform_utils.isWindows():
+      if objdir:
+        objdir = objdir.replace('\\', '/')
+      if gitdir:
+        gitdir = gitdir.replace('\\', '/')
+
+    if objdir:
+      # Set to the place we want to save the objects.
+      env['GIT_OBJECT_DIRECTORY'] = objdir
+      if gitdir:
+        # Allow git to search the original place in case of local or unique refs
+        # that git will attempt to resolve even if we aren't fetching them.
+        env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = gitdir + '/objects'
 
     command = [GIT]
     if bare:
       if gitdir:
-        # Git on Windows wants its paths only using / for reliability.
-        if platform_utils.isWindows():
-          gitdir = gitdir.replace('\\', '/')
         env[GIT_DIR] = gitdir
       cwd = None
     command.append(cmdv[0])
@@ -234,6 +246,11 @@ class GitCommand(object):
         dbg += ': export GIT_DIR=%s\n' % env[GIT_DIR]
         LAST_GITDIR = env[GIT_DIR]
 
+      if 'GIT_OBJECT_DIRECTORY' in env:
+        dbg += ': export GIT_OBJECT_DIRECTORY=%s\n' % env['GIT_OBJECT_DIRECTORY']
+      if 'GIT_ALTERNATE_OBJECT_DIRECTORIES' in env:
+        dbg += ': export GIT_ALTERNATE_OBJECT_DIRECTORIES=%s\n' % env['GIT_ALTERNATE_OBJECT_DIRECTORIES']
+
       dbg += ': '
       dbg += ' '.join(command)
       if stdin == subprocess.PIPE:

git_config.py

@@ -352,7 +352,7 @@ class GitConfig(object):
       Trace(': parsing %s', self.file)
       with open(self._json) as fd:
         return json.load(fd)
-    except (IOError, ValueErrorl):
+    except (IOError, ValueError):
       platform_utils.remove(self._json, missing_ok=True)
       return None
 

git_superproject.py

@@ -18,7 +18,7 @@ For more information on superproject, check out:
 https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects
 
 Examples:
-  superproject = Superproject()
+  superproject = Superproject(manifest, name, remote, revision)
   UpdateProjectsResult = superproject.UpdateProjectsRevisionId(projects)
 """
 
@@ -32,7 +32,6 @@ from typing import NamedTuple
 from git_command import git_require, GitCommand
 from git_config import RepoConfig
 from git_refs import R_HEADS
-from manifest_xml import LOCAL_MANIFEST_GROUP_PREFIX
 
 _SUPERPROJECT_GIT_NAME = 'superproject.git'
 _SUPERPROJECT_MANIFEST_NAME = 'superproject_override.xml'
@@ -72,41 +71,50 @@ class Superproject(object):
   lookup of commit ids for all projects. It contains _project_commit_ids which
   is a dictionary with project/commit id entries.
   """
-  def __init__(self, manifest, repodir, git_event_log,
-               superproject_dir='exp-superproject', quiet=False, print_messages=False):
+  def __init__(self, manifest, name, remote, revision,
+               superproject_dir='exp-superproject'):
     """Initializes superproject.
 
     Args:
       manifest: A Manifest object that is to be written to a file.
-      repodir: Path to the .repo/ dir for holding all internal checkout state.
-          It must be in the top directory of the repo client checkout.
-      git_event_log: A git trace2 event log to log events.
-      superproject_dir: Relative path under |repodir| to checkout superproject.
-      quiet:  If True then only print the progress messages.
-      print_messages: if True then print error/warning messages.
+      name: The unique name of the superproject
+      remote: The RemoteSpec for the remote.
+      revision: The name of the git branch to track.
+      superproject_dir: Relative path under |manifest.subdir| to checkout
+          superproject.
     """
     self._project_commit_ids = None
     self._manifest = manifest
-    self._git_event_log = git_event_log
-    self._quiet = quiet
-    self._print_messages = print_messages
-    self._branch = manifest.branch
-    self._repodir = os.path.abspath(repodir)
+    self.name = name
+    self.remote = remote
+    self.revision = self._branch = revision
+    self._repodir = manifest.repodir
     self._superproject_dir = superproject_dir
-    self._superproject_path = os.path.join(self._repodir, superproject_dir)
+    self._superproject_path = manifest.SubmanifestInfoDir(manifest.path_prefix,
+                                                          superproject_dir)
     self._manifest_path = os.path.join(self._superproject_path,
                                        _SUPERPROJECT_MANIFEST_NAME)
-    git_name = ''
-    if self._manifest.superproject:
-      remote = self._manifest.superproject['remote']
-      git_name = hashlib.md5(remote.name.encode('utf8')).hexdigest() + '-'
-      self._branch = self._manifest.superproject['revision']
-      self._remote_url = remote.url
-    else:
-      self._remote_url = None
+    git_name = hashlib.md5(remote.name.encode('utf8')).hexdigest() + '-'
+    self._remote_url = remote.url
     self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME
     self._work_git = os.path.join(self._superproject_path, self._work_git_name)
 
+    # The following are command arguemnts, rather than superproject attributes,
+    # and were included here originally.  They should eventually become
+    # arguments that are passed down from the public methods, instead of being
+    # treated as attributes.
+    self._git_event_log = None
+    self._quiet = False
+    self._print_messages = False
+
+  def SetQuiet(self, value):
+    """Set the _quiet attribute."""
+    self._quiet = value
+
+  def SetPrintMessages(self, value):
+    """Set the _print_messages attribute."""
+    self._print_messages = value
+
   @property
   def project_commit_ids(self):
     """Returns a dictionary of projects and their commit ids."""
@@ -215,19 +223,23 @@ class Superproject(object):
                        f'return code: {retval}, stderr: {p.stderr}')
     return data
 
-  def Sync(self):
+  def Sync(self, git_event_log):
     """Gets a local copy of a superproject for the manifest.
 
+    Args:
+      git_event_log: an EventLog, for git tracing.
+
     Returns:
       SyncResult
     """
+    self._git_event_log = git_event_log
     if not self._manifest.superproject:
       self._LogWarning(f'superproject tag is not defined in manifest: '
                        f'{self._manifest.manifestFile}')
       return SyncResult(False, False)
 
-    print('NOTICE: --use-superproject is in beta; report any issues to the '
-          'address described in `repo version`', file=sys.stderr)
+    _PrintBetaNotice()
+
     should_exit = True
     if not self._remote_url:
       self._LogWarning(f'superproject URL is not defined in manifest: '
@@ -248,7 +260,7 @@ class Superproject(object):
     Returns:
       CommitIdsResult
     """
-    sync_result = self.Sync()
+    sync_result = self.Sync(self._git_event_log)
     if not sync_result.success:
       return CommitIdsResult(None, sync_result.fatal)
 
@@ -311,17 +323,19 @@ class Superproject(object):
     if project.revisionId:
       return True
     # Skip the project if it comes from the local manifest.
-    return any(s.startswith(LOCAL_MANIFEST_GROUP_PREFIX) for s in project.groups)
+    return project.manifest.IsFromLocalManifest(project)
 
-  def UpdateProjectsRevisionId(self, projects):
+  def UpdateProjectsRevisionId(self, projects, git_event_log):
     """Update revisionId of every project in projects with the commit id.
 
     Args:
-      projects: List of projects whose revisionId needs to be updated.
+      projects: a list of projects whose revisionId needs to be updated.
+      git_event_log: an EventLog, for git tracing.
 
     Returns:
       UpdateProjectsResult
     """
+    self._git_event_log = git_event_log
     commit_ids_result = self._GetAllProjectsCommitIds()
     commit_ids = commit_ids_result.commit_ids
     if not commit_ids:
@@ -351,6 +365,13 @@ class Superproject(object):
     return UpdateProjectsResult(manifest_path, False)
 
 
+@functools.lru_cache(maxsize=10)
+def _PrintBetaNotice():
+  """Print the notice of beta status."""
+  print('NOTICE: --use-superproject is in beta; report any issues to the '
+        'address described in `repo version`', file=sys.stderr)
+
+
 @functools.lru_cache(maxsize=None)
 def _UseSuperprojectFromConfiguration():
   """Returns the user choice of whether to use superproject."""
@@ -395,21 +416,37 @@ def _UseSuperprojectFromConfiguration():
   return False
 
 
-def PrintMessages(opt, manifest):
-  """Returns a boolean if error/warning messages are to be printed."""
-  return opt.use_superproject is not None or manifest.superproject
+def PrintMessages(use_superproject, manifest):
+  """Returns a boolean if error/warning messages are to be printed.
+
+  Args:
+    use_superproject: option value from optparse.
+    manifest: manifest to use.
+  """
+  return use_superproject is not None or bool(manifest.superproject)
 
 
-def UseSuperproject(opt, manifest):
-  """Returns a boolean if use-superproject option is enabled."""
+def UseSuperproject(use_superproject, manifest):
+  """Returns a boolean if use-superproject option is enabled.
 
-  if opt.use_superproject is not None:
-    return opt.use_superproject
+  Args:
+    use_superproject: option value from optparse.
+    manifest: manifest to use.
+
+  Returns:
+    Whether the superproject should be used.
+  """
+
+  if not manifest.superproject:
+    # This (sub) manifest does not have a superproject definition.
+    return False
+  elif use_superproject is not None:
+    return use_superproject
   else:
-    client_value = manifest.manifestProject.config.GetBoolean('repo.superproject')
+    client_value = manifest.manifestProject.use_superproject
     if client_value is not None:
       return client_value
-    else:
-      if not manifest.superproject:
-        return False
+    elif manifest.superproject:
       return _UseSuperprojectFromConfiguration()
+    else:
+      return False

git_trace2_event_log.py

@@ -29,8 +29,10 @@ https://git-scm.com/docs/api-trace2#_the_event_format_target
 
 
 import datetime
+import errno
 import json
 import os
+import socket
 import sys
 import tempfile
 import threading
@@ -218,20 +220,39 @@ class EventLog(object):
           retval, p.stderr), file=sys.stderr)
     return path
 
+  def _WriteLog(self, write_fn):
+    """Writes the log out using a provided writer function.
+
+    Generate compact JSON output for each item in the log, and write it using
+    write_fn.
+
+    Args:
+      write_fn: A function that accepts byts and writes them to a destination.
+    """
+
+    for e in self._log:
+      # Dump in compact encoding mode.
+      # See 'Compact encoding' in Python docs:
+      # https://docs.python.org/3/library/json.html#module-json
+      write_fn(json.dumps(e, indent=None, separators=(',', ':')).encode('utf-8') + b'\n')
+
   def Write(self, path=None):
-    """Writes the log out to a file.
+    """Writes the log out to a file or socket.
 
     Log is only written if 'path' or 'git config --get trace2.eventtarget'
-    provide a valid path to write logs to.
+    provide a valid path (or socket) to write logs to.
 
     Logging filename format follows the git trace2 style of being a unique
     (exclusive writable) file.
 
     Args:
-      path: Path to where logs should be written.
+      path: Path to where logs should be written. The path may have a prefix of
+          the form "af_unix:[{stream|dgram}:]", in which case the path is
+          treated as a Unix domain socket. See
+          https://git-scm.com/docs/api-trace2#_enabling_a_target for details.
 
     Returns:
-      log_path: Path to the log file if log is written, otherwise None
+      log_path: Path to the log file or socket if log is written, otherwise None
     """
     log_path = None
     # If no logging path is specified, get the path from 'trace2.eventtarget'.
@@ -242,29 +263,66 @@ class EventLog(object):
     if path is None:
       return None
 
+    path_is_socket = False
+    socket_type = None
     if isinstance(path, str):
-      # Get absolute path.
-      path = os.path.abspath(os.path.expanduser(path))
+      parts = path.split(':', 1)
+      if parts[0] == 'af_unix' and len(parts) == 2:
+        path_is_socket = True
+        path = parts[1]
+        parts = path.split(':', 1)
+        if parts[0] == 'stream' and len(parts) == 2:
+          socket_type = socket.SOCK_STREAM
+          path = parts[1]
+        elif parts[0] == 'dgram' and len(parts) == 2:
+          socket_type = socket.SOCK_DGRAM
+          path = parts[1]
+      else:
+        # Get absolute path.
+        path = os.path.abspath(os.path.expanduser(path))
     else:
       raise TypeError('path: str required but got %s.' % type(path))
 
     # Git trace2 requires a directory to write log to.
 
     # TODO(https://crbug.com/gerrit/13706): Support file (append) mode also.
-    if not os.path.isdir(path):
+    if not (path_is_socket or os.path.isdir(path)):
       return None
+
+    if path_is_socket:
+      if socket_type == socket.SOCK_STREAM or socket_type is None:
+        try:
+          with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
+            sock.connect(path)
+            self._WriteLog(sock.sendall)
+          return f'af_unix:stream:{path}'
+        except OSError as err:
+          # If we tried to connect to a DGRAM socket using STREAM, ignore the
+          # attempt and continue to DGRAM below. Otherwise, issue a warning.
+          if err.errno != errno.EPROTOTYPE:
+            print(f'repo: warning: git trace2 logging failed: {err}', file=sys.stderr)
+            return None
+      if socket_type == socket.SOCK_DGRAM or socket_type is None:
+        try:
+          with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
+            self._WriteLog(lambda bs: sock.sendto(bs, path))
+            return f'af_unix:dgram:{path}'
+        except OSError as err:
+          print(f'repo: warning: git trace2 logging failed: {err}', file=sys.stderr)
+          return None
+      # Tried to open a socket but couldn't connect (SOCK_STREAM) or write
+      # (SOCK_DGRAM).
+      print('repo: warning: git trace2 logging failed: could not write to socket', file=sys.stderr)
+      return None
+
+    # Path is an absolute path
     # Use NamedTemporaryFile to generate a unique filename as required by git trace2.
     try:
-      with tempfile.NamedTemporaryFile(mode='x', prefix=self._sid, dir=path,
+      with tempfile.NamedTemporaryFile(mode='xb', prefix=self._sid, dir=path,
                                        delete=False) as f:
         # TODO(https://crbug.com/gerrit/13706): Support writing events as they
         # occur.
-        for e in self._log:
-          # Dump in compact encoding mode.
-          # See 'Compact encoding' in Python docs:
-          # https://docs.python.org/3/library/json.html#module-json
-          json.dump(e, f, indent=None, separators=(',', ':'))
-          f.write('\n')
+        self._WriteLog(f.write)
         log_path = f.name
     except FileExistsError as err:
       print('repo: warning: git trace2 logging failed: %r' % err,

main.py

@@ -127,6 +127,8 @@ global_options.add_option('--event-log',
                           help='filename of event log to append timeline to')
 global_options.add_option('--git-trace2-event-log', action='store',
                           help='directory to write git trace2 event log to')
+global_options.add_option('--submanifest-path', action='store',
+                          metavar='REL_PATH', help='submanifest path')
 
 
 class _Repo(object):
@@ -217,7 +219,12 @@ class _Repo(object):
     SetDefaultColoring(gopts.color)
 
     git_trace2_event_log = EventLog()
-    repo_client = RepoClient(self.repodir)
+    outer_client = RepoClient(self.repodir)
+    repo_client = outer_client
+    if gopts.submanifest_path:
+      repo_client = RepoClient(self.repodir,
+                               submanifest_path=gopts.submanifest_path,
+                               outer_client=outer_client)
     gitc_manifest = None
     gitc_client_name = gitc_utils.parse_clientdir(os.getcwd())
     if gitc_client_name:
@@ -229,6 +236,8 @@ class _Repo(object):
           repodir=self.repodir,
           client=repo_client,
           manifest=repo_client.manifest,
+          outer_client=outer_client,
+          outer_manifest=outer_client.manifest,
           gitc_manifest=gitc_manifest,
           git_event_log=git_trace2_event_log)
     except KeyError:
@@ -283,7 +292,36 @@ class _Repo(object):
     try:
       cmd.CommonValidateOptions(copts, cargs)
       cmd.ValidateOptions(copts, cargs)
-      result = cmd.Execute(copts, cargs)
+
+      this_manifest_only = copts.this_manifest_only
+      outer_manifest = copts.outer_manifest
+      if cmd.MULTI_MANIFEST_SUPPORT or this_manifest_only:
+        result = cmd.Execute(copts, cargs)
+      elif outer_manifest and repo_client.manifest.is_submanifest:
+        # The command does not support multi-manifest, we are using a
+        # submanifest, and the command line is for the outermost manifest.
+        # Re-run using the outermost manifest, which will recurse through the
+        # submanifests.
+        gopts.submanifest_path = ''
+        result = self._Run(name, gopts, argv)
+      else:
+        # No multi-manifest support. Run the command in the current
+        # (sub)manifest, and then any child submanifests.
+        result = cmd.Execute(copts, cargs)
+        for submanifest in repo_client.manifest.submanifests.values():
+          spec = submanifest.ToSubmanifestSpec()
+          gopts.submanifest_path = submanifest.repo_client.path_prefix
+          child_argv = argv[:]
+          child_argv.append('--no-outer-manifest')
+          # Not all subcommands support the 3 manifest options, so only add them
+          # if the original command includes them.
+          if hasattr(copts, 'manifest_url'):
+            child_argv.extend(['--manifest-url', spec.manifestUrl])
+          if hasattr(copts, 'manifest_name'):
+            child_argv.extend(['--manifest-name', spec.manifestName])
+          if hasattr(copts, 'manifest_branch'):
+            child_argv.extend(['--manifest-branch', spec.revision])
+          result = self._Run(name, gopts, child_argv) or result
     except (DownloadError, ManifestInvalidRevisionError,
             NoManifestException) as e:
       print('error: in `%s`: %s' % (' '.join([name] + argv), str(e)),

man/repo-gitc-init.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "September 2021" "repo gitc-init" "Repo Manual"
+.TH REPO "1" "November 2021" "repo gitc-init" "Repo Manual"
 .SH NAME
 repo \- repo gitc-init - manual page for repo gitc-init
 .SH SYNOPSIS
@@ -31,10 +31,6 @@ manifest branch or revision (use HEAD for default)
 \fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
 initial manifest file
 .TP
-\fB\-\-standalone\-manifest\fR
-download the manifest as a static file rather then
-create a git checkout of the manifest repo
-.TP
 \fB\-g\fR GROUP, \fB\-\-groups\fR=\fI\,GROUP\/\fR
 restrict manifest projects to ones with specified
 group(s) [default|all|G1,G2,G3|G4,\-G5,\-G6]
@@ -45,6 +41,10 @@ platform group [auto|all|none|linux|darwin|...]
 .TP
 \fB\-\-submodules\fR
 sync any submodules associated with the manifest repo
+.TP
+\fB\-\-standalone\-manifest\fR
+download the manifest as a static file rather then
+create a git checkout of the manifest repo
 .SS Manifest (only) checkout options:
 .TP
 \fB\-\-current\-branch\fR
@@ -96,7 +96,8 @@ filter for use with \fB\-\-partial\-clone\fR [default:
 blob:none]
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects

man/repo-init.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "September 2021" "repo init" "Repo Manual"
+.TH REPO "1" "November 2021" "repo init" "Repo Manual"
 .SH NAME
 repo \- repo init - manual page for repo init
 .SH SYNOPSIS
@@ -31,10 +31,6 @@ manifest branch or revision (use HEAD for default)
 \fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
 initial manifest file
 .TP
-\fB\-\-standalone\-manifest\fR
-download the manifest as a static file rather then
-create a git checkout of the manifest repo
-.TP
 \fB\-g\fR GROUP, \fB\-\-groups\fR=\fI\,GROUP\/\fR
 restrict manifest projects to ones with specified
 group(s) [default|all|G1,G2,G3|G4,\-G5,\-G6]
@@ -45,6 +41,10 @@ platform group [auto|all|none|linux|darwin|...]
 .TP
 \fB\-\-submodules\fR
 sync any submodules associated with the manifest repo
+.TP
+\fB\-\-standalone\-manifest\fR
+download the manifest as a static file rather then
+create a git checkout of the manifest repo
 .SS Manifest (only) checkout options:
 .TP
 \fB\-c\fR, \fB\-\-current\-branch\fR
@@ -96,7 +96,8 @@ filter for use with \fB\-\-partial\-clone\fR [default:
 blob:none]
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects

man/repo-manifest.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo manifest" "Repo Manual"
+.TH REPO "1" "November 2021" "repo manifest" "Repo Manual"
 .SH NAME
 repo \- repo manifest - manual page for repo manifest
 .SH SYNOPSIS
@@ -161,6 +161,7 @@ CDATA #IMPLIED>
 <!ELEMENT extend\-project EMPTY>
 <!ATTLIST extend\-project name CDATA #REQUIRED>
 <!ATTLIST extend\-project path CDATA #IMPLIED>
+<!ATTLIST extend\-project dest\-path CDATA #IMPLIED>
 <!ATTLIST extend\-project groups CDATA #IMPLIED>
 <!ATTLIST extend\-project revision CDATA #IMPLIED>
 <!ATTLIST extend\-project remote CDATA #IMPLIED>
@@ -174,8 +175,9 @@ CDATA #IMPLIED>
 <!ATTLIST repo\-hooks enabled\-list CDATA #REQUIRED>
 .IP
 <!ELEMENT superproject EMPTY>
-<!ATTLIST superproject name    CDATA #REQUIRED>
-<!ATTLIST superproject remote  IDREF #IMPLIED>
+<!ATTLIST superproject name     CDATA #REQUIRED>
+<!ATTLIST superproject remote   IDREF #IMPLIED>
+<!ATTLIST superproject revision CDATA #IMPLIED>
 .IP
 <!ELEMENT contactinfo EMPTY>
 <!ATTLIST contactinfo bugurl  CDATA #REQUIRED>
@@ -385,6 +387,11 @@ original manifest.
 Attribute `path`: If specified, limit the change to projects checked out at the
 specified path, rather than all projects with the given name.
 .PP
+Attribute `dest\-path`: If specified, a path relative to the top directory of the
+repo client where the Git working directory for this project should be placed.
+This is used to move a project in the checkout by overriding the existing `path`
+setting.
+.PP
 Attribute `groups`: List of additional groups to which this project belongs.
 Same syntax as the corresponding element of `project`.
 .PP
@@ -477,6 +484,10 @@ project](#element\-project) for more information.
 Attribute `remote`: Name of a previously defined remote element. If not supplied
 the remote given by the default element is used.
 .PP
+Attribute `revision`: Name of the Git branch the manifest wants to track for
+this superproject. If not supplied the revision given by the remote element is
+used if applicable, else the default element is used.
+.PP
 Element contactinfo
 .PP
 *** *Note*: This is currently a WIP. ***

man/repo-smartsync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo smartsync" "Repo Manual"
+.TH REPO "1" "November 2021" "repo smartsync" "Repo Manual"
 .SH NAME
 repo \- repo smartsync - manual page for repo smartsync
 .SH SYNOPSIS
@@ -80,7 +80,8 @@ password to authenticate with the manifest server
 fetch submodules from server
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects
@@ -89,7 +90,7 @@ disable use of manifest superprojects
 fetch tags
 .TP
 \fB\-\-no\-tags\fR
-don't fetch tags
+don't fetch tags (default)
 .TP
 \fB\-\-optimized\-fetch\fR
 only fetch projects fixed to sha1 if revision does not
@@ -100,6 +101,10 @@ number of times to retry fetches on transient errors
 .TP
 \fB\-\-prune\fR
 delete refs that no longer exist on the remote
+(default)
+.TP
+\fB\-\-no\-prune\fR
+do not delete refs that no longer exist on the remote
 .SS Logging options:
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR

man/repo-sync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo sync" "Repo Manual"
+.TH REPO "1" "November 2021" "repo sync" "Repo Manual"
 .SH NAME
 repo \- repo sync - manual page for repo sync
 .SH SYNOPSIS
@@ -80,7 +80,8 @@ password to authenticate with the manifest server
 fetch submodules from server
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects
@@ -89,7 +90,7 @@ disable use of manifest superprojects
 fetch tags
 .TP
 \fB\-\-no\-tags\fR
-don't fetch tags
+don't fetch tags (default)
 .TP
 \fB\-\-optimized\-fetch\fR
 only fetch projects fixed to sha1 if revision does not
@@ -100,6 +101,10 @@ number of times to retry fetches on transient errors
 .TP
 \fB\-\-prune\fR
 delete refs that no longer exist on the remote
+(default)
+.TP
+\fB\-\-no\-prune\fR
+do not delete refs that no longer exist on the remote
 .TP
 \fB\-s\fR, \fB\-\-smart\-sync\fR
 smart sync using manifest from the latest known good

man/repo.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo" "Repo Manual"
+.TH REPO "1" "November 2021" "repo" "Repo Manual"
 .SH NAME
 repo \- repository management tool built on top of git
 .SH SYNOPSIS
@@ -43,7 +43,7 @@ filename of event log to append timeline to
 .TP
 \fB\-\-git\-trace2\-event\-log\fR=\fI\,GIT_TRACE2_EVENT_LOG\/\fR
 directory to write git trace2 event log to
-.SS "The complete list of recognized repo commands are:"
+.SS "The complete list of recognized repo commands is:"
 .TP
 abandon
 Permanently abandon a development branch

manifest_xml.py

@@ -24,8 +24,10 @@ import urllib.parse
 import gitc_utils
 from git_config import GitConfig, IsId
 from git_refs import R_HEADS, HEAD
+from git_superproject import Superproject
 import platform_utils
-from project import Annotation, RemoteSpec, Project, MetaProject
+from project import (Annotation, RemoteSpec, Project, RepoProject,
+                     ManifestProject)
 from error import (ManifestParseError, ManifestInvalidPathError,
                    ManifestInvalidRevisionError)
 from wrapper import Wrapper
@@ -33,6 +35,11 @@ from wrapper import Wrapper
 MANIFEST_FILE_NAME = 'manifest.xml'
 LOCAL_MANIFEST_NAME = 'local_manifest.xml'
 LOCAL_MANIFESTS_DIR_NAME = 'local_manifests'
+SUBMANIFEST_DIR = 'submanifests'
+# Limit submanifests to an arbitrary depth for loop detection.
+MAX_SUBMANIFEST_DEPTH = 8
+# Add all projects from sub manifest into a group.
+SUBMANIFEST_GROUP_PREFIX = 'submanifest:'
 
 # Add all projects from local manifest into a group.
 LOCAL_MANIFEST_GROUP_PREFIX = 'local:'
@@ -197,10 +204,141 @@ class _XmlRemote(object):
     self.annotations.append(Annotation(name, value, keep))
 
 
+class _XmlSubmanifest:
+  """Manage the <submanifest> element specified in the manifest.
+
+  Attributes:
+    name: a string, the name for this submanifest.
+    remote: a string, the remote.name for this submanifest.
+    project: a string, the name of the manifest project.
+    revision: a string, the commitish.
+    manifestName: a string, the submanifest file name.
+    groups: a list of strings, the groups to add to all projects in the submanifest.
+    default_groups: a list of strings, the default groups to sync.
+    path: a string, the relative path for the submanifest checkout.
+    parent: an XmlManifest, the parent manifest.
+    annotations: (derived) a list of annotations.
+    present: (derived) a boolean, whether the sub manifest file is present.
+  """
+  def __init__(self,
+               name,
+               remote=None,
+               project=None,
+               revision=None,
+               manifestName=None,
+               groups=None,
+               default_groups=None,
+               path=None,
+               parent=None):
+    self.name = name
+    self.remote = remote
+    self.project = project
+    self.revision = revision
+    self.manifestName = manifestName
+    self.groups = groups
+    self.default_groups = default_groups
+    self.path = path
+    self.parent = parent
+    self.annotations = []
+    outer_client = parent._outer_client or parent
+    if self.remote and not self.project:
+      raise ManifestParseError(
+          f'Submanifest {name}: must specify project when remote is given.')
+    # Construct the absolute path to the manifest file using the parent's
+    # method, so that we can correctly create our repo_client.
+    manifestFile = parent.SubmanifestInfoDir(
+        os.path.join(parent.path_prefix, self.relpath),
+        os.path.join('manifests', manifestName or 'default.xml'))
+    linkFile = parent.SubmanifestInfoDir(
+        os.path.join(parent.path_prefix, self.relpath), MANIFEST_FILE_NAME)
+    rc = self.repo_client = RepoClient(
+        parent.repodir, linkFile, parent_groups=','.join(groups) or '',
+        submanifest_path=self.relpath, outer_client=outer_client,
+        default_groups=default_groups)
+
+    self.present = os.path.exists(manifestFile)
+
+  def __eq__(self, other):
+    if not isinstance(other, _XmlSubmanifest):
+      return False
+    return (
+        self.name == other.name and
+        self.remote == other.remote and
+        self.project == other.project and
+        self.revision == other.revision and
+        self.manifestName == other.manifestName and
+        self.groups == other.groups and
+        self.default_groups == other.default_groups and
+        self.path == other.path and
+        sorted(self.annotations) == sorted(other.annotations))
+
+  def __ne__(self, other):
+    return not self.__eq__(other)
+
+  def ToSubmanifestSpec(self):
+    """Return a SubmanifestSpec object, populating attributes"""
+    mp = self.parent.manifestProject
+    remote = self.parent.remotes[self.remote or self.parent.default.remote.name]
+    # If a project was given, generate the url from the remote and project.
+    # If not, use this manifestProject's url.
+    if self.project:
+      manifestUrl = remote.ToRemoteSpec(self.project).url
+    else:
+      manifestUrl = mp.GetRemote(mp.remote.name).url
+    manifestName = self.manifestName or 'default.xml'
+    revision = self.revision or self.name
+    path = self.path or revision.split('/')[-1]
+    groups = self.groups or []
+    default_groups = self.default_groups or []
+
+    return SubmanifestSpec(self.name, manifestUrl, manifestName, revision, path,
+                           groups)
+
+  @property
+  def relpath(self):
+    """The path of this submanifest relative to the parent manifest."""
+    revision = self.revision or self.name
+    return self.path or revision.split('/')[-1]
+
+  def GetGroupsStr(self):
+    """Returns the `groups` given for this submanifest."""
+    if self.groups:
+      return ','.join(self.groups)
+    return ''
+
+  def GetDefaultGroupsStr(self):
+    """Returns the `default-groups` given for this submanifest."""
+    return ','.join(self.default_groups or [])
+
+  def AddAnnotation(self, name, value, keep):
+    """Add annotations to the submanifest."""
+    self.annotations.append(Annotation(name, value, keep))
+
+
+class SubmanifestSpec:
+  """The submanifest element, with all fields expanded."""
+
+  def __init__(self,
+               name,
+               manifestUrl,
+               manifestName,
+               revision,
+               path,
+               groups):
+    self.name = name
+    self.manifestUrl = manifestUrl
+    self.manifestName = manifestName
+    self.revision = revision
+    self.path = path
+    self.groups = groups or []
+
+
 class XmlManifest(object):
   """manages the repo configuration file"""
 
-  def __init__(self, repodir, manifest_file, local_manifests=None):
+  def __init__(self, repodir, manifest_file, local_manifests=None,
+               outer_client=None, parent_groups='', submanifest_path='',
+               default_groups=None):
     """Initialize.
 
     Args:
@@ -210,33 +348,56 @@ class XmlManifest(object):
           be |repodir|/|MANIFEST_FILE_NAME|.
       local_manifests: Full path to the directory of local override manifests.
           This will usually be |repodir|/|LOCAL_MANIFESTS_DIR_NAME|.
+      outer_client: RepoClient of the outer manifest.
+      parent_groups: a string, the groups to apply to this projects.
+      submanifest_path: The submanifest root relative to the repo root.
+      default_groups: a string, the default manifest groups to use.
     """
     # TODO(vapier): Move this out of this class.
     self.globalConfig = GitConfig.ForUser()
 
     self.repodir = os.path.abspath(repodir)
-    self.topdir = os.path.dirname(self.repodir)
+    self._CheckLocalPath(submanifest_path)
+    self.topdir = os.path.join(os.path.dirname(self.repodir), submanifest_path)
+    if manifest_file != os.path.abspath(manifest_file):
+      raise ManifestParseError('manifest_file must be abspath')
     self.manifestFile = manifest_file
+    if not outer_client or outer_client == self:
+      # manifestFileOverrides only exists in the outer_client's manifest, since
+      # that is the only instance left when Unload() is called on the outer
+      # manifest.
+      self.manifestFileOverrides = {}
     self.local_manifests = local_manifests
     self._load_local_manifests = True
+    self.parent_groups = parent_groups
+    self.default_groups = default_groups
 
-    self.repoProject = MetaProject(self, 'repo',
+    if outer_client and self.isGitcClient:
+      raise ManifestParseError('Multi-manifest is incompatible with `gitc-init`')
+
+    if submanifest_path and not outer_client:
+      # If passing a submanifest_path, there must be an outer_client.
+      raise ManifestParseError(f'Bad call to {self.__class__.__name__}')
+
+    # If self._outer_client is None, this is not a checkout that supports
+    # multi-tree.
+    self._outer_client = outer_client or self
+
+    self.repoProject = RepoProject(self, 'repo',
                                    gitdir=os.path.join(repodir, 'repo/.git'),
                                    worktree=os.path.join(repodir, 'repo'))
 
-    mp = MetaProject(self, 'manifests',
-                     gitdir=os.path.join(repodir, 'manifests.git'),
-                     worktree=os.path.join(repodir, 'manifests'))
+    mp = self.SubmanifestProject(self.path_prefix)
     self.manifestProject = mp
 
     # This is a bit hacky, but we're in a chicken & egg situation: all the
     # normal repo settings live in the manifestProject which we just setup
     # above, so we couldn't easily query before that.  We assume Project()
     # init doesn't care if this changes afterwards.
-    if os.path.exists(mp.gitdir) and mp.config.GetBoolean('repo.worktree'):
+    if os.path.exists(mp.gitdir) and mp.use_worktree:
       mp.use_git_worktrees = True
 
-    self._Unload()
+    self.Unload()
 
   def Override(self, name, load_local_manifests=True):
     """Use a different manifest, just for the current instantiation.
@@ -255,14 +416,10 @@ class XmlManifest(object):
       if not os.path.isfile(path):
         raise ManifestParseError('manifest %s not found' % name)
 
-    old = self.manifestFile
-    try:
-      self._load_local_manifests = load_local_manifests
-      self.manifestFile = path
-      self._Unload()
-      self._Load()
-    finally:
-      self.manifestFile = old
+    self._load_local_manifests = load_local_manifests
+    self._outer_client.manifestFileOverrides[self.path_prefix] = path
+    self.Unload()
+    self._Load()
 
   def Link(self, name):
     """Update the repo metadata to use a different manifest.
@@ -311,6 +468,33 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         ae.setAttribute('value', a.value)
         e.appendChild(ae)
 
+  def _SubmanifestToXml(self, r, doc, root):
+    """Generate XML <submanifest/> node."""
+    e = doc.createElement('submanifest')
+    root.appendChild(e)
+    e.setAttribute('name', r.name)
+    if r.remote is not None:
+      e.setAttribute('remote', r.remote)
+    if r.project is not None:
+      e.setAttribute('project', r.project)
+    if r.manifestName is not None:
+      e.setAttribute('manifest-name', r.manifestName)
+    if r.revision is not None:
+      e.setAttribute('revision', r.revision)
+    if r.path is not None:
+      e.setAttribute('path', r.path)
+    if r.groups:
+      e.setAttribute('groups', r.GetGroupsStr())
+    if r.default_groups:
+      e.setAttribute('default-groups', r.GetDefaultGroupsStr())
+
+    for a in r.annotations:
+      if a.keep == 'true':
+        ae = doc.createElement('annotation')
+        ae.setAttribute('name', a.name)
+        ae.setAttribute('value', a.value)
+        e.appendChild(ae)
+
   def _ParseList(self, field):
     """Parse fields that contain flattened lists.
 
@@ -323,12 +507,14 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     mp = self.manifestProject
 
     if groups is None:
-      groups = mp.config.GetString('manifest.groups')
+      groups = mp.manifest_groups
     if groups:
       groups = self._ParseList(groups)
 
     doc = xml.dom.minidom.Document()
     root = doc.createElement('manifest')
+    if self.is_submanifest:
+      root.setAttribute('path', self.path_prefix)
     doc.appendChild(root)
 
     # Save out the notice.  There's a little bit of work here to give it the
@@ -383,6 +569,11 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
       root.appendChild(e)
       root.appendChild(doc.createTextNode(''))
 
+    for r in sorted(self.submanifests):
+      self._SubmanifestToXml(self.submanifests[r], doc, root)
+    if self.submanifests:
+      root.appendChild(doc.createTextNode(''))
+
     def output_projects(parent, parent_node, projects):
       for project_name in projects:
         for project in self._projects[project_name]:
@@ -498,17 +689,17 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     if self._superproject:
       root.appendChild(doc.createTextNode(''))
       e = doc.createElement('superproject')
-      e.setAttribute('name', self._superproject['name'])
+      e.setAttribute('name', self._superproject.name)
       remoteName = None
       if d.remote:
         remoteName = d.remote.name
-      remote = self._superproject.get('remote')
+      remote = self._superproject.remote
       if not d.remote or remote.orig_name != remoteName:
         remoteName = remote.orig_name
         e.setAttribute('remote', remoteName)
       revision = remote.revision or d.revisionExpr
-      if not revision or revision != self._superproject['revision']:
-        e.setAttribute('revision', self._superproject['revision'])
+      if not revision or revision != self._superproject.revision:
+        e.setAttribute('revision', self._superproject.revision)
       root.appendChild(e)
 
     if self._contactinfo.bugurl != Wrapper().BUG_URL:
@@ -537,6 +728,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         'project',
         'extend-project',
         'include',
+        'submanifest',
         # These are children of 'project' nodes.
         'annotation',
         'project',
@@ -574,26 +766,109 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
   def _output_manifest_project_extras(self, p, e):
     """Manifests can modify e if they support extra project attributes."""
 
+  @property
+  def is_multimanifest(self):
+    """Whether this is a multimanifest checkout.
+
+    This is safe to use as long as the outermost manifest XML has been parsed.
+    """
+    return bool(self._outer_client._submanifests)
+
+  @property
+  def is_submanifest(self):
+    """Whether this manifest is a submanifest.
+
+    This is safe to use as long as the outermost manifest XML has been parsed.
+    """
+    return self._outer_client and self._outer_client != self
+
+  @property
+  def outer_client(self):
+    """The instance of the outermost manifest client."""
+    self._Load()
+    return self._outer_client
+
+  @property
+  def all_manifests(self):
+    """Generator yielding all (sub)manifests, in depth-first order."""
+    self._Load()
+    outer = self._outer_client
+    yield outer
+    for tree in outer.all_children:
+      yield tree
+
+  @property
+  def all_children(self):
+    """Generator yielding all (present) child submanifests."""
+    self._Load()
+    for child in self._submanifests.values():
+      if child.repo_client:
+        yield child.repo_client
+        for tree in child.repo_client.all_children:
+          yield tree
+
+  @property
+  def path_prefix(self):
+    """The path of this submanifest, relative to the outermost manifest."""
+    if not self._outer_client or self == self._outer_client:
+      return ''
+    return os.path.relpath(self.topdir, self._outer_client.topdir)
+
+  @property
+  def all_paths(self):
+    """All project paths for all (sub)manifests.
+
+    See also `paths`.
+
+    Returns:
+      A dictionary of {path: Project()}.  `path` is relative to the outer
+      manifest.
+    """
+    ret = {}
+    for tree in self.all_manifests:
+      prefix = tree.path_prefix
+      ret.update({os.path.join(prefix, k): v for k, v in tree.paths.items()})
+    return ret
+
+  @property
+  def all_projects(self):
+    """All projects for all (sub)manifests.  See `projects`."""
+    return list(itertools.chain.from_iterable(x._paths.values() for x in self.all_manifests))
+
   @property
   def paths(self):
+    """Return all paths for this manifest.
+
+    Returns:
+      A dictionary of {path: Project()}.  `path` is relative to this manifest.
+    """
     self._Load()
     return self._paths
 
   @property
   def projects(self):
+    """Return a list of all Projects in this manifest."""
     self._Load()
     return list(self._paths.values())
 
   @property
   def remotes(self):
+    """Return a list of remotes for this manifest."""
     self._Load()
     return self._remotes
 
   @property
   def default(self):
+    """Return default values for this manifest."""
     self._Load()
     return self._default
 
+  @property
+  def submanifests(self):
+    """All submanifests in this manifest."""
+    self._Load()
+    return self._submanifests
+
   @property
   def repo_hooks_project(self):
     self._Load()
@@ -621,24 +896,27 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
   @property
   def CloneBundle(self):
-    clone_bundle = self.manifestProject.config.GetBoolean('repo.clonebundle')
+    clone_bundle = self.manifestProject.clone_bundle
     if clone_bundle is None:
-      return False if self.manifestProject.config.GetBoolean('repo.partialclone') else True
+      return False if self.manifestProject.partial_clone else True
     else:
       return clone_bundle
 
   @property
   def CloneFilter(self):
-    if self.manifestProject.config.GetBoolean('repo.partialclone'):
-      return self.manifestProject.config.GetString('repo.clonefilter')
+    if self.manifestProject.partial_clone:
+      return self.manifestProject.clone_filter
     return None
 
   @property
   def PartialCloneExclude(self):
-    exclude = self.manifest.manifestProject.config.GetString(
-        'repo.partialcloneexclude') or ''
+    exclude = self.manifest.manifestProject.partial_clone_exclude or ''
     return set(x.strip() for x in exclude.split(','))
 
+  def SetManifestOverride(self, path):
+    """Override manifestFile.  The caller must call Unload()"""
+    self._outer_client.manifest.manifestFileOverrides[self.path_prefix] = path
+
   @property
   def UseLocalManifests(self):
     return self._load_local_manifests
@@ -650,88 +928,195 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
   def HasLocalManifests(self):
     return self._load_local_manifests and self.local_manifests
 
+  def IsFromLocalManifest(self, project):
+    """Is the project from a local manifest?"""
+    return any(x.startswith(LOCAL_MANIFEST_GROUP_PREFIX)
+               for x in project.groups)
+
   @property
   def IsMirror(self):
-    return self.manifestProject.config.GetBoolean('repo.mirror')
+    return self.manifestProject.mirror
 
   @property
   def UseGitWorktrees(self):
-    return self.manifestProject.config.GetBoolean('repo.worktree')
+    return self.manifestProject.use_worktree
 
   @property
   def IsArchive(self):
-    return self.manifestProject.config.GetBoolean('repo.archive')
+    return self.manifestProject.archive
 
   @property
   def HasSubmodules(self):
-    return self.manifestProject.config.GetBoolean('repo.submodules')
+    return self.manifestProject.submodules
 
-  def GetDefaultGroupsStr(self):
-    """Returns the default group string for the platform."""
-    return 'default,platform-' + platform.system().lower()
+  @property
+  def EnableGitLfs(self):
+    return self.manifestProject.git_lfs
+
+  def FindManifestByPath(self, path):
+    """Returns the manifest containing path."""
+    path = os.path.abspath(path)
+    manifest = self._outer_client or self
+    old = None
+    while manifest._submanifests and manifest != old:
+      old = manifest
+      for name in manifest._submanifests:
+        tree = manifest._submanifests[name]
+        if path.startswith(tree.repo_client.manifest.topdir):
+          manifest = tree.repo_client
+          break
+    return manifest
+
+  @property
+  def subdir(self):
+    """Returns the path for per-submanifest objects for this manifest."""
+    return self.SubmanifestInfoDir(self.path_prefix)
+
+  def SubmanifestInfoDir(self, submanifest_path, object_path=''):
+    """Return the path to submanifest-specific info for a submanifest.
+
+    Return the full path of the directory in which to put per-manifest objects.
+
+    Args:
+      submanifest_path: a string, the path of the submanifest, relative to the
+                        outermost topdir.  If empty, then repodir is returned.
+      object_path: a string, relative path to append to the submanifest info
+                   directory path.
+    """
+    if submanifest_path:
+      return os.path.join(self.repodir, SUBMANIFEST_DIR, submanifest_path,
+                          object_path)
+    else:
+      return os.path.join(self.repodir, object_path)
+
+  def SubmanifestProject(self, submanifest_path):
+    """Return a manifestProject for a submanifest."""
+    subdir = self.SubmanifestInfoDir(submanifest_path)
+    mp = ManifestProject(self, 'manifests',
+                         gitdir=os.path.join(subdir, 'manifests.git'),
+                         worktree=os.path.join(subdir, 'manifests'))
+    return mp
+
+  def GetDefaultGroupsStr(self, with_platform=True):
+    """Returns the default group string to use.
+
+    Args:
+      with_platform: a boolean, whether to include the group for the
+                     underlying platform.
+    """
+    groups = ','.join(self.default_groups or ['default'])
+    if with_platform:
+      groups += f',platform-{platform.system().lower()}'
+    return groups
 
   def GetGroupsStr(self):
     """Returns the manifest group string that should be synced."""
-    groups = self.manifestProject.config.GetString('manifest.groups')
-    if not groups:
-      groups = self.GetDefaultGroupsStr()
-    return groups
+    return self.manifestProject.manifest_groups or self.GetDefaultGroupsStr()
 
-  def _Unload(self):
+  def Unload(self):
+    """Unload the manifest.
+
+    If the manifest files have been changed since Load() was called, this will
+    cause the new/updated manifest to be used.
+
+    """
     self._loaded = False
     self._projects = {}
     self._paths = {}
     self._remotes = {}
     self._default = None
+    self._submanifests = {}
     self._repo_hooks_project = None
-    self._superproject = {}
+    self._superproject = None
     self._contactinfo = ContactInfo(Wrapper().BUG_URL)
     self._notice = None
     self.branch = None
     self._manifest_server = None
 
-  def _Load(self):
+  def Load(self):
+    """Read the manifest into memory."""
+    # Do not expose internal arguments.
+    self._Load()
+
+  def _Load(self, initial_client=None, submanifest_depth=0):
+    if submanifest_depth > MAX_SUBMANIFEST_DEPTH:
+      raise ManifestParseError('maximum submanifest depth %d exceeded.' %
+                               MAX_SUBMANIFEST_DEPTH)
     if not self._loaded:
-      m = self.manifestProject
-      b = m.GetBranch(m.CurrentBranch).merge
-      if b is not None and b.startswith(R_HEADS):
-        b = b[len(R_HEADS):]
-      self.branch = b
+      if self._outer_client and self._outer_client != self:
+        # This will load all clients.
+        self._outer_client._Load(initial_client=self)
 
-      # The manifestFile was specified by the user which is why we allow include
-      # paths to point anywhere.
-      nodes = []
-      nodes.append(self._ParseManifestXml(
-          self.manifestFile, self.manifestProject.worktree,
-          restrict_includes=False))
-
-      if self._load_local_manifests and self.local_manifests:
-        try:
-          for local_file in sorted(platform_utils.listdir(self.local_manifests)):
-            if local_file.endswith('.xml'):
-              local = os.path.join(self.local_manifests, local_file)
-              # Since local manifests are entirely managed by the user, allow
-              # them to point anywhere the user wants.
-              nodes.append(self._ParseManifestXml(
-                  local, self.repodir,
-                  parent_groups=f'{LOCAL_MANIFEST_GROUP_PREFIX}:{local_file[:-4]}',
-                  restrict_includes=False))
-        except OSError:
-          pass
+      savedManifestFile = self.manifestFile
+      override = self._outer_client.manifestFileOverrides.get(self.path_prefix)
+      if override:
+        self.manifestFile = override
 
       try:
-        self._ParseManifest(nodes)
-      except ManifestParseError as e:
-        # There was a problem parsing, unload ourselves in case they catch
-        # this error and try again later, we will show the correct error
-        self._Unload()
-        raise e
+        m = self.manifestProject
+        b = m.GetBranch(m.CurrentBranch).merge
+        if b is not None and b.startswith(R_HEADS):
+          b = b[len(R_HEADS):]
+        self.branch = b
 
-      if self.IsMirror:
-        self._AddMetaProjectMirror(self.repoProject)
-        self._AddMetaProjectMirror(self.manifestProject)
+        parent_groups = self.parent_groups
+        if self.path_prefix:
+          parent_groups = f'{SUBMANIFEST_GROUP_PREFIX}:path:{self.path_prefix},{parent_groups}'
 
-      self._loaded = True
+        # The manifestFile was specified by the user which is why we allow include
+        # paths to point anywhere.
+        nodes = []
+        nodes.append(self._ParseManifestXml(
+            self.manifestFile, self.manifestProject.worktree,
+            parent_groups=parent_groups, restrict_includes=False))
+
+        if self._load_local_manifests and self.local_manifests:
+          try:
+            for local_file in sorted(platform_utils.listdir(self.local_manifests)):
+              if local_file.endswith('.xml'):
+                local = os.path.join(self.local_manifests, local_file)
+                # Since local manifests are entirely managed by the user, allow
+                # them to point anywhere the user wants.
+                local_group = f'{LOCAL_MANIFEST_GROUP_PREFIX}:{local_file[:-4]}'
+                nodes.append(self._ParseManifestXml(
+                    local, self.subdir,
+                    parent_groups=f'{local_group},{parent_groups}',
+                    restrict_includes=False))
+          except OSError:
+            pass
+
+        try:
+          self._ParseManifest(nodes)
+        except ManifestParseError as e:
+          # There was a problem parsing, unload ourselves in case they catch
+          # this error and try again later, we will show the correct error
+          self.Unload()
+          raise e
+
+        if self.IsMirror:
+          self._AddMetaProjectMirror(self.repoProject)
+          self._AddMetaProjectMirror(self.manifestProject)
+
+        self._loaded = True
+      finally:
+        if override:
+          self.manifestFile = savedManifestFile
+
+      # Now that we have loaded this manifest, load any submanifests as well.
+      # We need to do this after self._loaded is set to avoid looping.
+      for name in self._submanifests:
+        tree = self._submanifests[name]
+        spec = tree.ToSubmanifestSpec()
+        present = os.path.exists(os.path.join(self.subdir, MANIFEST_FILE_NAME))
+        if present and tree.present and not tree.repo_client:
+          if initial_client and initial_client.topdir == self.topdir:
+            tree.repo_client = self
+            tree.present = present
+          elif not os.path.exists(self.subdir):
+            tree.present = False
+        if present and tree.present:
+          tree.repo_client._Load(initial_client=initial_client,
+                                 submanifest_depth=submanifest_depth + 1)
 
   def _ParseManifestXml(self, path, include_root, parent_groups='',
                         restrict_includes=True):
@@ -822,6 +1207,20 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     if self._default is None:
       self._default = _Default()
 
+    submanifest_paths = set()
+    for node in itertools.chain(*node_list):
+      if node.nodeName == 'submanifest':
+        submanifest = self._ParseSubmanifest(node)
+        if submanifest:
+          if submanifest.name in self._submanifests:
+            if submanifest != self._submanifests[submanifest.name]:
+              raise ManifestParseError(
+                  'submanifest %s already exists with different attributes' %
+                  (submanifest.name))
+          else:
+            self._submanifests[submanifest.name] = submanifest
+            submanifest_paths.add(submanifest.relpath)
+
     for node in itertools.chain(*node_list):
       if node.nodeName == 'notice':
         if self._notice is not None:
@@ -849,6 +1248,11 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         raise ManifestParseError(
             'duplicate path %s in %s' %
             (project.relpath, self.manifestFile))
+      for tree in submanifest_paths:
+        if project.relpath.startswith(tree):
+          raise ManifestParseError(
+              'project %s conflicts with submanifest path %s' %
+              (project.relpath, tree))
       self._paths[project.relpath] = project
       projects.append(project)
       for subproject in project.subprojects:
@@ -873,8 +1277,10 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         if groups:
           groups = self._ParseList(groups)
         revision = node.getAttribute('revision')
-        remote = node.getAttribute('remote')
-        if remote:
+        remote_name = node.getAttribute('remote')
+        if not remote_name:
+          remote = self._default.remote
+        else:
           remote = self._get_remote(node)
 
         named_projects = self._projects[name]
@@ -889,12 +1295,13 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
           if revision:
             p.SetRevision(revision)
 
-          if remote:
+          if remote_name:
             p.remote = remote.ToRemoteSpec(name)
 
           if dest_path:
             del self._paths[p.relpath]
-            relpath, worktree, gitdir, objdir, _ = self.GetProjectPaths(name, dest_path)
+            relpath, worktree, gitdir, objdir, _ = self.GetProjectPaths(
+                name, dest_path, remote.name)
             p.UpdatePaths(relpath, worktree, gitdir, objdir)
             self._paths[p.relpath] = p
 
@@ -911,11 +1318,10 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
       if node.nodeName == 'superproject':
         name = self._reqatt(node, 'name')
         # There can only be one superproject.
-        if self._superproject.get('name'):
+        if self._superproject:
           raise ManifestParseError(
               'duplicate superproject in %s' %
               (self.manifestFile))
-        self._superproject['name'] = name
         remote_name = node.getAttribute('remote')
         if not remote_name:
           remote = self._default.remote
@@ -924,14 +1330,16 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         if remote is None:
           raise ManifestParseError("no remote for superproject %s within %s" %
                                    (name, self.manifestFile))
-        self._superproject['remote'] = remote.ToRemoteSpec(name)
         revision = node.getAttribute('revision') or remote.revision
         if not revision:
           revision = self._default.revisionExpr
         if not revision:
           raise ManifestParseError('no revision for superproject %s within %s' %
                                    (name, self.manifestFile))
-        self._superproject['revision'] = revision
+        self._superproject = Superproject(self,
+                                          name=name,
+                                          remote=remote.ToRemoteSpec(name),
+                                          revision=revision)
       if node.nodeName == 'contactinfo':
         bugurl = self._reqatt(node, 'bugurl')
         # This element can be repeated, later entries will clobber earlier ones.
@@ -1099,6 +1507,54 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
     return '\n'.join(cleanLines)
 
+  def _ParseSubmanifest(self, node):
+    """Reads a <submanifest> element from the manifest file."""
+    name = self._reqatt(node, 'name')
+    remote = node.getAttribute('remote')
+    if remote == '':
+      remote = None
+    project = node.getAttribute('project')
+    if project == '':
+      project = None
+    revision = node.getAttribute('revision')
+    if revision == '':
+      revision = None
+    manifestName = node.getAttribute('manifest-name')
+    if manifestName == '':
+      manifestName = None
+    groups = ''
+    if node.hasAttribute('groups'):
+      groups = node.getAttribute('groups')
+    groups = self._ParseList(groups)
+    default_groups = self._ParseList(node.getAttribute('default-groups'))
+    path = node.getAttribute('path')
+    if path == '':
+      path = None
+      if revision:
+        msg = self._CheckLocalPath(revision.split('/')[-1])
+        if msg:
+          raise ManifestInvalidPathError(
+              '<submanifest> invalid "revision": %s: %s' % (revision, msg))
+      else:
+        msg = self._CheckLocalPath(name)
+        if msg:
+          raise ManifestInvalidPathError(
+              '<submanifest> invalid "name": %s: %s' % (name, msg))
+    else:
+      msg = self._CheckLocalPath(path)
+      if msg:
+        raise ManifestInvalidPathError(
+            '<submanifest> invalid "path": %s: %s' % (path, msg))
+
+    submanifest = _XmlSubmanifest(name, remote, project, revision, manifestName,
+                                  groups, default_groups, path, self)
+
+    for n in node.childNodes:
+      if n.nodeName == 'annotation':
+        self._ParseAnnotation(submanifest, n)
+
+    return submanifest
+
   def _JoinName(self, parent_name, name):
     return os.path.join(parent_name, name)
 
@@ -1162,7 +1618,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
     if parent is None:
       relpath, worktree, gitdir, objdir, use_git_worktrees = \
-          self.GetProjectPaths(name, path)
+          self.GetProjectPaths(name, path, remote.name)
     else:
       use_git_worktrees = False
       relpath, worktree, gitdir, objdir = \
@@ -1208,31 +1664,61 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
     return project
 
-  def GetProjectPaths(self, name, path):
+  def GetProjectPaths(self, name, path, remote):
+    """Return the paths for a project.
+
+    Args:
+      name: a string, the name of the project.
+      path: a string, the path of the project.
+      remote: a string, the remote.name of the project.
+
+    Returns:
+      A tuple of (relpath, worktree, gitdir, objdir, use_git_worktrees) for the
+      project with |name| and |path|.
+    """
     # The manifest entries might have trailing slashes.  Normalize them to avoid
     # unexpected filesystem behavior since we do string concatenation below.
     path = path.rstrip('/')
     name = name.rstrip('/')
+    remote = remote.rstrip('/')
     use_git_worktrees = False
+    use_remote_name = self.is_multimanifest
     relpath = path
     if self.IsMirror:
       worktree = None
       gitdir = os.path.join(self.topdir, '%s.git' % name)
       objdir = gitdir
     else:
+      if use_remote_name:
+        namepath = os.path.join(remote, f'{name}.git')
+      else:
+        namepath = f'{name}.git'
       worktree = os.path.join(self.topdir, path).replace('\\', '/')
-      gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path)
+      gitdir = os.path.join(self.subdir, 'projects', '%s.git' % path)
       # We allow people to mix git worktrees & non-git worktrees for now.
       # This allows for in situ migration of repo clients.
       if os.path.exists(gitdir) or not self.UseGitWorktrees:
-        objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name)
+        objdir = os.path.join(self.repodir, 'project-objects', namepath)
       else:
         use_git_worktrees = True
-        gitdir = os.path.join(self.repodir, 'worktrees', '%s.git' % name)
+        gitdir = os.path.join(self.repodir, 'worktrees', namepath)
         objdir = gitdir
     return relpath, worktree, gitdir, objdir, use_git_worktrees
 
-  def GetProjectsWithName(self, name):
+  def GetProjectsWithName(self, name, all_manifests=False):
+    """All projects with |name|.
+
+    Args:
+      name: a string, the name of the project.
+      all_manifests: a boolean, if True, then all manifests are searched. If
+                     False, then only this manifest is searched.
+
+    Returns:
+      A list of Project instances with name |name|.
+    """
+    if all_manifests:
+      return list(itertools.chain.from_iterable(
+          x._projects.get(name, []) for x in self.all_manifests))
     return self._projects.get(name, [])
 
   def GetSubprojectName(self, parent, submodule_path):
@@ -1488,19 +1974,36 @@ class GitcManifest(XmlManifest):
 class RepoClient(XmlManifest):
   """Manages a repo client checkout."""
 
-  def __init__(self, repodir, manifest_file=None):
-    self.isGitcClient = False
+  def __init__(self, repodir, manifest_file=None, submanifest_path='', **kwargs):
+    """Initialize.
 
-    if os.path.exists(os.path.join(repodir, LOCAL_MANIFEST_NAME)):
+    Args:
+      repodir: Path to the .repo/ dir for holding all internal checkout state.
+          It must be in the top directory of the repo client checkout.
+      manifest_file: Full path to the manifest file to parse.  This will usually
+          be |repodir|/|MANIFEST_FILE_NAME|.
+      submanifest_path: The submanifest root relative to the repo root.
+      **kwargs: Additional keyword arguments, passed to XmlManifest.
+    """
+    self.isGitcClient = False
+    submanifest_path = submanifest_path or ''
+    if submanifest_path:
+      self._CheckLocalPath(submanifest_path)
+      prefix = os.path.join(repodir, SUBMANIFEST_DIR, submanifest_path)
+    else:
+      prefix = repodir
+
+    if os.path.exists(os.path.join(prefix, LOCAL_MANIFEST_NAME)):
       print('error: %s is not supported; put local manifests in `%s` instead' %
-            (LOCAL_MANIFEST_NAME, os.path.join(repodir, LOCAL_MANIFESTS_DIR_NAME)),
+            (LOCAL_MANIFEST_NAME, os.path.join(prefix, LOCAL_MANIFESTS_DIR_NAME)),
             file=sys.stderr)
       sys.exit(1)
 
     if manifest_file is None:
-      manifest_file = os.path.join(repodir, MANIFEST_FILE_NAME)
-    local_manifests = os.path.abspath(os.path.join(repodir, LOCAL_MANIFESTS_DIR_NAME))
-    super().__init__(repodir, manifest_file, local_manifests)
+        manifest_file = os.path.join(prefix, MANIFEST_FILE_NAME)
+    local_manifests = os.path.abspath(os.path.join(prefix, LOCAL_MANIFESTS_DIR_NAME))
+    super().__init__(repodir, manifest_file, local_manifests,
+                     submanifest_path=submanifest_path, **kwargs)
 
     # TODO: Completely separate manifest logic out of the client.
     self.manifest = self

progress.py

@@ -24,6 +24,11 @@ _NOT_TTY = not os.isatty(2)
 # column 0.
 CSI_ERASE_LINE = '\x1b[2K'
 
+# This will erase all content in the current line after the cursor.  This is
+# useful for partial updates & progress messages as the terminal can display
+# it better.
+CSI_ERASE_LINE_AFTER = '\x1b[K'
+
 
 def duration_str(total):
   """A less noisy timedelta.__str__.
@@ -85,10 +90,10 @@ class Progress(object):
         return
 
     if self._total <= 0:
-      sys.stderr.write('%s\r%s: %d,' % (
-          CSI_ERASE_LINE,
+      sys.stderr.write('\r%s: %d,%s' % (
           self._title,
-          self._done))
+          self._done,
+          CSI_ERASE_LINE_AFTER))
       sys.stderr.flush()
     else:
       p = (100 * self._done) / self._total
@@ -96,14 +101,14 @@ class Progress(object):
         jobs = '[%d job%s] ' % (self._active, 's' if self._active > 1 else '')
       else:
         jobs = ''
-      sys.stderr.write('%s\r%s: %2d%% %s(%d%s/%d%s)%s%s%s' % (
-          CSI_ERASE_LINE,
+      sys.stderr.write('\r%s: %2d%% %s(%d%s/%d%s)%s%s%s%s' % (
           self._title,
           p,
           jobs,
           self._done, self._units,
           self._total, self._units,
           ' ' if msg else '', msg,
+          CSI_ERASE_LINE_AFTER,
           '\n' if self._print_newline else ''))
       sys.stderr.flush()
 
@@ -113,19 +118,19 @@ class Progress(object):
 
     duration = duration_str(time() - self._start)
     if self._total <= 0:
-      sys.stderr.write('%s\r%s: %d, done in %s\n' % (
-          CSI_ERASE_LINE,
+      sys.stderr.write('\r%s: %d, done in %s%s\n' % (
           self._title,
           self._done,
-          duration))
+          duration,
+          CSI_ERASE_LINE_AFTER))
       sys.stderr.flush()
     else:
       p = (100 * self._done) / self._total
-      sys.stderr.write('%s\r%s: %3d%% (%d%s/%d%s), done in %s\n' % (
-          CSI_ERASE_LINE,
+      sys.stderr.write('\r%s: %3d%% (%d%s/%d%s), done in %s%s\n' % (
           self._title,
           p,
           self._done, self._units,
           self._total, self._units,
-          duration))
+          duration,
+          CSI_ERASE_LINE_AFTER))
       sys.stderr.flush()

release/sign-launcher.py

@@ -20,6 +20,7 @@ This is intended to be run only by the official Repo release managers.
 
 import argparse
 import os
+import re
 import subprocess
 import sys
 
@@ -49,18 +50,37 @@ def check(opts):
   util.run(opts, ['gpg', '--verify', f'{opts.launcher}.asc'])
 
 
-def postmsg(opts):
+def get_version(opts):
+  """Get the version from |launcher|."""
+  # Make sure we don't search $PATH when signing the "repo" file in the cwd.
+  launcher = os.path.join('.', opts.launcher)
+  cmd = [launcher, '--version']
+  ret = util.run(opts, cmd, encoding='utf-8', stdout=subprocess.PIPE)
+  m = re.search(r'repo launcher version ([0-9.]+)', ret.stdout)
+  if not m:
+    sys.exit(f'{opts.launcher}: unable to detect repo version')
+  return m.group(1)
+
+
+def postmsg(opts, version):
   """Helpful info to show at the end for release manager."""
   print(f"""
 Repo launcher bucket:
   gs://git-repo-downloads/
 
-To upload this launcher directly:
-  gsutil cp -a public-read {opts.launcher} {opts.launcher}.asc gs://git-repo-downloads/
+You should first upload it with a specific version:
+  gsutil cp -a public-read {opts.launcher} gs://git-repo-downloads/repo-{version}
+  gsutil cp -a public-read {opts.launcher}.asc gs://git-repo-downloads/repo-{version}.asc
 
-NB: You probably want to upload it with a specific version first, e.g.:
-  gsutil cp -a public-read {opts.launcher} gs://git-repo-downloads/repo-3.0
-  gsutil cp -a public-read {opts.launcher}.asc gs://git-repo-downloads/repo-3.0.asc
+Then to make it the public default:
+  gsutil cp -a public-read gs://git-repo-downloads/repo-{version} gs://git-repo-downloads/repo
+  gsutil cp -a public-read gs://git-repo-downloads/repo-{version}.asc gs://git-repo-downloads/repo.asc
+
+NB: If a rollback is necessary, the GS bucket archives old versions, and may be
+    accessed by specifying their unique id number.
+  gsutil ls -la gs://git-repo-downloads/repo gs://git-repo-downloads/repo.asc
+  gsutil cp -a public-read gs://git-repo-downloads/repo#<unique id> gs://git-repo-downloads/repo
+  gsutil cp -a public-read gs://git-repo-downloads/repo.asc#<unique id> gs://git-repo-downloads/repo.asc
 """)
 
 
@@ -103,9 +123,10 @@ def main(argv):
     opts.keys = [util.KEYID_DSA, util.KEYID_RSA, util.KEYID_ECC]
     util.import_release_key(opts)
 
+  version = get_version(opts)
   sign(opts)
   check(opts)
-  postmsg(opts)
+  postmsg(opts, version)
 
   return 0
 

repo

@@ -149,7 +149,7 @@ if not REPO_REV:
 BUG_URL = 'https://bugs.chromium.org/p/gerrit/issues/entry?template=Repo+tool+issue'
 
 # increment this whenever we make important changes to this script
-VERSION = (2, 17)
+VERSION = (2, 21)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (2, 3)
@@ -372,7 +372,7 @@ def InitParser(parser, gitc_init=False):
                    help='filter for use with --partial-clone '
                         '[default: %default]')
   group.add_option('--use-superproject', action='store_true', default=None,
-                   help='use the manifest superproject to sync projects')
+                   help='use the manifest superproject to sync projects; implies -c')
   group.add_option('--no-use-superproject', action='store_false',
                    dest='use_superproject',
                    help='disable use of manifest superprojects')
@@ -382,6 +382,11 @@ def InitParser(parser, gitc_init=False):
   group.add_option('--no-clone-bundle',
                    dest='clone_bundle', action='store_false',
                    help='disable use of /clone.bundle on HTTP/HTTPS (default if --partial-clone)')
+  group.add_option('--git-lfs', action='store_true',
+                   help='enable Git LFS support')
+  group.add_option('--no-git-lfs',
+                   dest='git_lfs', action='store_false',
+                   help='disable Git LFS support')
 
   # Tool.
   group = parser.add_option_group('repo Version options')
@@ -618,6 +623,7 @@ def _Init(args, gitc_init=False):
             "REPO_URL set correctly?" % url, file=sys.stderr)
 
   except CloneFailure:
+    print('fatal: double check your --repo-rev setting.', file=sys.stderr)
     if opt.quiet:
       print('fatal: repo init failed; run without --quiet to see why',
             file=sys.stderr)

ssh.py

@@ -52,6 +52,9 @@ def version():
   """return ssh version as a tuple"""
   try:
     return _parse_ssh_version()
+  except FileNotFoundError:
+    print('fatal: ssh not installed', file=sys.stderr)
+    sys.exit(1)
   except subprocess.CalledProcessError:
     print('fatal: unable to detect ssh version', file=sys.stderr)
     sys.exit(1)

subcmds/abandon.py

@@ -69,7 +69,8 @@ It is equivalent to "git branch -D <branchname>".
     nb = args[0]
     err = defaultdict(list)
     success = defaultdict(list)
-    all_projects = self.GetProjects(args[1:])
+    all_projects = self.GetProjects(args[1:], all_manifests=not opt.this_manifest_only)
+    _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
 
     def _ProcessResults(_pool, pm, states):
       for (results, project) in states:
@@ -94,7 +95,7 @@ It is equivalent to "git branch -D <branchname>".
         err_msg = "error: cannot abandon %s" % br
         print(err_msg, file=sys.stderr)
         for proj in err[br]:
-          print(' ' * len(err_msg) + " | %s" % proj.relpath, file=sys.stderr)
+          print(' ' * len(err_msg) + " | %s" % _RelPath(proj), file=sys.stderr)
       sys.exit(1)
     elif not success:
       print('error: no project has local branch(es) : %s' % nb,
@@ -110,5 +111,5 @@ It is equivalent to "git branch -D <branchname>".
           result = "all project"
         else:
           result = "%s" % (
-              ('\n' + ' ' * width + '| ').join(p.relpath for p in success[br]))
+              ('\n' + ' ' * width + '| ').join(_RelPath(p) for p in success[br]))
         print("%s%s| %s\n" % (br, ' ' * (width - len(br)), result))

subcmds/branches.py

@@ -98,7 +98,7 @@ is shown, then the branch appears in all projects.
   PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
   def Execute(self, opt, args):
-    projects = self.GetProjects(args)
+    projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
     out = BranchColoring(self.manifest.manifestProject.config)
     all_branches = {}
     project_cnt = len(projects)
@@ -147,26 +147,28 @@ is shown, then the branch appears in all projects.
       hdr('%c%c %-*s' % (current, published, width, name))
       out.write(' |')
 
+      _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
       if in_cnt < project_cnt:
         fmt = out.write
         paths = []
         non_cur_paths = []
-        if i.IsSplitCurrent or (in_cnt < project_cnt - in_cnt):
+        if i.IsSplitCurrent or (in_cnt <= project_cnt - in_cnt):
           in_type = 'in'
           for b in i.projects:
+            relpath = b.project.relpath
             if not i.IsSplitCurrent or b.current:
-              paths.append(b.project.relpath)
+              paths.append(_RelPath(b.project))
             else:
-              non_cur_paths.append(b.project.relpath)
+              non_cur_paths.append(_RelPath(b.project))
         else:
           fmt = out.notinproject
           in_type = 'not in'
           have = set()
           for b in i.projects:
-            have.add(b.project)
+            have.add(_RelPath(b.project))
           for p in projects:
-            if p not in have:
-              paths.append(p.relpath)
+            if _RelPath(p) not in have:
+              paths.append(_RelPath(p))
 
         s = ' %s %s' % (in_type, ', '.join(paths))
         if not i.IsSplitCurrent and (width + 7 + len(s) < 80):

subcmds/checkout.py

@@ -47,7 +47,7 @@ The command is equivalent to:
     nb = args[0]
     err = []
     success = []
-    all_projects = self.GetProjects(args[1:])
+    all_projects = self.GetProjects(args[1:], all_manifests=not opt.this_manifest_only)
 
     def _ProcessResults(_pool, pm, results):
       for status, project in results:

subcmds/diff.py

@@ -50,7 +50,7 @@ to the Unix 'patch' command.
     return (ret, buf.getvalue())
 
   def Execute(self, opt, args):
-    all_projects = self.GetProjects(args)
+    all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     def _ProcessResults(_pool, _output, results):
       ret = 0

subcmds/diffmanifests.py

@@ -179,6 +179,9 @@ synced and their revisions won't be found.
   def ValidateOptions(self, opt, args):
     if not args or len(args) > 2:
       self.OptionParser.error('missing manifests to diff')
+    if opt.this_manifest_only is False:
+      raise self.OptionParser.error(
+          '`diffmanifest` only supports the current tree')
 
   def Execute(self, opt, args):
     self.out = _Coloring(self.client.globalConfig)

subcmds/download.py

@@ -48,7 +48,7 @@ If no project is specified try to use current directory as a project.
                  dest='ffonly', action='store_true',
                  help="force fast-forward merge")
 
-  def _ParseChangeIds(self, args):
+  def _ParseChangeIds(self, opt, args):
     if not args:
       self.Usage()
 
@@ -77,7 +77,7 @@ If no project is specified try to use current directory as a project.
                 ps_id = max(int(match.group(1)), ps_id)
         to_get.append((project, chg_id, ps_id))
       else:
-        projects = self.GetProjects([a])
+        projects = self.GetProjects([a], all_manifests=not opt.this_manifest_only)
         if len(projects) > 1:
           # If the cwd is one of the projects, assume they want that.
           try:
@@ -88,8 +88,8 @@ If no project is specified try to use current directory as a project.
             print('error: %s matches too many projects; please re-run inside '
                   'the project checkout.' % (a,), file=sys.stderr)
             for project in projects:
-              print('  %s/ @ %s' % (project.relpath, project.revisionExpr),
-                    file=sys.stderr)
+              print('  %s/ @ %s' % (project.RelPath(local=opt.this_manifest_only),
+                                    project.revisionExpr), file=sys.stderr)
             sys.exit(1)
         else:
           project = projects[0]
@@ -105,7 +105,7 @@ If no project is specified try to use current directory as a project.
         self.OptionParser.error('-x and --ff are mutually exclusive options')
 
   def Execute(self, opt, args):
-    for project, change_id, ps_id in self._ParseChangeIds(args):
+    for project, change_id, ps_id in self._ParseChangeIds(opt, args):
       dl = project.DownloadPatchSet(change_id, ps_id)
       if not dl:
         print('[%s] change %d/%d not found'

subcmds/forall.py

@@ -84,6 +84,11 @@ REPO_PROJECT is set to the unique name of the project.
 
 REPO_PATH is the path relative the the root of the client.
 
+REPO_OUTERPATH is the path of the sub manifest's root relative to the root of
+the client.
+
+REPO_INNERPATH is the path relative to the root of the sub manifest.
+
 REPO_REMOTE is the name of the remote system from the manifest.
 
 REPO_LREV is the name of the revision from the manifest, translated
@@ -168,6 +173,7 @@ without iterating through the remaining projects.
 
   def Execute(self, opt, args):
     cmd = [opt.command[0]]
+    all_trees = not opt.this_manifest_only
 
     shell = True
     if re.compile(r'^[a-z0-9A-Z_/\.-]+$').match(cmd[0]):
@@ -213,11 +219,11 @@ without iterating through the remaining projects.
       self.manifest.Override(smart_sync_manifest_path)
 
     if opt.regex:
-      projects = self.FindProjects(args)
+      projects = self.FindProjects(args, all_manifests=all_trees)
     elif opt.inverse_regex:
-      projects = self.FindProjects(args, inverse=True)
+      projects = self.FindProjects(args, inverse=True, all_manifests=all_trees)
     else:
-      projects = self.GetProjects(args, groups=opt.groups)
+      projects = self.GetProjects(args, groups=opt.groups, all_manifests=all_trees)
 
     os.environ['REPO_COUNT'] = str(len(projects))
 
@@ -289,7 +295,9 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
     env[name] = val
 
   setenv('REPO_PROJECT', project.name)
-  setenv('REPO_PATH', project.relpath)
+  setenv('REPO_OUTERPATH', project.manifest.path_prefix)
+  setenv('REPO_INNERPATH', project.relpath)
+  setenv('REPO_PATH', project.RelPath(local=opt.this_manifest_only))
   setenv('REPO_REMOTE', project.remote.name)
   try:
     # If we aren't in a fully synced state and we don't have the ref the manifest
@@ -320,7 +328,7 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
     output = ''
     if ((opt.project_header and opt.verbose)
             or not opt.project_header):
-      output = 'skipping %s/' % project.relpath
+      output = 'skipping %s/' % project.RelPath(local=opt.this_manifest_only)
     return (1, output)
 
   if opt.verbose:
@@ -344,7 +352,7 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
       if mirror:
         project_header_path = project.name
       else:
-        project_header_path = project.relpath
+        project_header_path = project.RelPath(local=opt.this_manifest_only)
       out.project('project %s/' % project_header_path)
       out.nl()
       buf.write(output)

subcmds/gitc_init.py

@@ -24,6 +24,7 @@ import wrapper
 
 class GitcInit(init.Init, GitcAvailableCommand):
   COMMON = True
+  MULTI_MANIFEST_SUPPORT = False
   helpSummary = "Initialize a GITC Client."
   helpUsage = """
 %prog [options] [client name]

subcmds/grep.py

@@ -172,15 +172,16 @@ contain a line that matches both expressions:
     return (project, p.Wait(), p.stdout, p.stderr)
 
   @staticmethod
-  def _ProcessResults(full_name, have_rev, _pool, out, results):
+  def _ProcessResults(full_name, have_rev, opt, _pool, out, results):
     git_failed = False
     bad_rev = False
     have_match = False
+    _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
 
     for project, rc, stdout, stderr in results:
       if rc < 0:
         git_failed = True
-        out.project('--- project %s ---' % project.relpath)
+        out.project('--- project %s ---' % _RelPath(project))
         out.nl()
         out.fail('%s', stderr)
         out.nl()
@@ -192,7 +193,7 @@ contain a line that matches both expressions:
           if have_rev and 'fatal: ambiguous argument' in stderr:
             bad_rev = True
           else:
-            out.project('--- project %s ---' % project.relpath)
+            out.project('--- project %s ---' % _RelPath(project))
             out.nl()
             out.fail('%s', stderr.strip())
             out.nl()
@@ -208,13 +209,13 @@ contain a line that matches both expressions:
           rev, line = line.split(':', 1)
           out.write("%s", rev)
           out.write(':')
-          out.project(project.relpath)
+          out.project(_RelPath(project))
           out.write('/')
           out.write("%s", line)
           out.nl()
       elif full_name:
         for line in r:
-          out.project(project.relpath)
+          out.project(_RelPath(project))
           out.write('/')
           out.write("%s", line)
           out.nl()
@@ -239,7 +240,7 @@ contain a line that matches both expressions:
       cmd_argv.append(args[0])
       args = args[1:]
 
-    projects = self.GetProjects(args)
+    projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     full_name = False
     if len(projects) > 1:
@@ -259,7 +260,7 @@ contain a line that matches both expressions:
         opt.jobs,
         functools.partial(self._ExecuteOne, cmd_argv),
         projects,
-        callback=functools.partial(self._ProcessResults, full_name, have_rev),
+        callback=functools.partial(self._ProcessResults, full_name, have_rev, opt),
         output=out,
         ordered=True)
 

subcmds/help.py

@@ -53,7 +53,7 @@ Displays detailed usage information about a command.
     self.PrintAllCommandsBody()
 
   def PrintAllCommandsBody(self):
-    print('The complete list of recognized repo commands are:')
+    print('The complete list of recognized repo commands is:')
     commandNames = list(sorted(all_commands))
     self._PrintCommands(commandNames)
     print("See 'repo help <command>' for more information on a "

subcmds/info.py

@@ -61,10 +61,11 @@ class Info(PagedCommand):
 
     self.opt = opt
 
+    if not opt.this_manifest_only:
+      self.manifest = self.manifest.outer_client
     manifestConfig = self.manifest.manifestProject.config
     mergeBranch = manifestConfig.GetBranch("default").merge
-    manifestGroups = (manifestConfig.GetString('manifest.groups')
-                      or 'all,-notdefault')
+    manifestGroups = self.manifest.GetGroupsStr()
 
     self.heading("Manifest branch: ")
     if self.manifest.default.revisionExpr:
@@ -80,17 +81,17 @@ class Info(PagedCommand):
     self.printSeparator()
 
     if not opt.overview:
-      self.printDiffInfo(args)
+      self._printDiffInfo(opt, args)
     else:
-      self.printCommitOverview(args)
+      self._printCommitOverview(opt, args)
 
   def printSeparator(self):
     self.text("----------------------------")
     self.out.nl()
 
-  def printDiffInfo(self, args):
+  def _printDiffInfo(self, opt, args):
     # We let exceptions bubble up to main as they'll be well structured.
-    projs = self.GetProjects(args)
+    projs = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     for p in projs:
       self.heading("Project: ")
@@ -179,9 +180,9 @@ class Info(PagedCommand):
       self.text(" ".join(split[1:]))
       self.out.nl()
 
-  def printCommitOverview(self, args):
+  def _printCommitOverview(self, opt, args):
     all_branches = []
-    for project in self.GetProjects(args):
+    for project in self.GetProjects(args, all_manifests=not opt.this_manifest_only):
       br = [project.GetUploadableBranch(x)
             for x in project.GetBranches()]
       br = [x for x in br if x]
@@ -200,7 +201,7 @@ class Info(PagedCommand):
       if project != branch.project:
         project = branch.project
         self.out.nl()
-        self.headtext(project.relpath)
+        self.headtext(project.RelPath(local=opt.this_manifest_only))
         self.out.nl()
 
       commits = branch.commits

subcmds/init.py

@@ -15,7 +15,6 @@
 import os
 import platform
 import re
-import subprocess
 import sys
 import urllib.parse
 
@@ -25,14 +24,12 @@ from error import ManifestParseError
 from project import SyncBuffer
 from git_config import GitConfig
 from git_command import git_require, MIN_GIT_VERSION_SOFT, MIN_GIT_VERSION_HARD
-import fetch
-import git_superproject
-import platform_utils
 from wrapper import Wrapper
 
 
 class Init(InteractiveCommand, MirrorSafeCommand):
   COMMON = True
+  MULTI_MANIFEST_SUPPORT = True
   helpSummary = "Initialize a repo client checkout in the current directory"
   helpUsage = """
 %prog [options] [manifest url]
@@ -91,250 +88,51 @@ to update the working directory files.
 
   def _Options(self, p, gitc_init=False):
     Wrapper().InitParser(p, gitc_init=gitc_init)
+    m = p.add_option_group('Multi-manifest')
+    m.add_option('--outer-manifest', action='store_true', default=True,
+                 help='operate starting at the outermost manifest')
+    m.add_option('--no-outer-manifest', dest='outer_manifest',
+                 action='store_false', help='do not operate on outer manifests')
+    m.add_option('--this-manifest-only', action='store_true', default=None,
+                 help='only operate on this (sub)manifest')
+    m.add_option('--no-this-manifest-only', '--all-manifests',
+                 dest='this_manifest_only', action='store_false',
+                 help='operate on this manifest and its submanifests')
 
   def _RegisteredEnvironmentOptions(self):
     return {'REPO_MANIFEST_URL': 'manifest_url',
             'REPO_MIRROR_LOCATION': 'reference'}
 
-  def _CloneSuperproject(self, opt):
-    """Clone the superproject based on the superproject's url and branch.
+  def _SyncManifest(self, opt):
+    """Call manifestProject.Sync with arguments from opt.
 
     Args:
-      opt: Program options returned from optparse.  See _Options().
+      opt: options from optparse.
     """
-    superproject = git_superproject.Superproject(self.manifest,
-                                                 self.repodir,
-                                                 self.git_event_log,
-                                                 quiet=opt.quiet)
-    sync_result = superproject.Sync()
-    if not sync_result.success:
-      print('warning: git update of superproject failed, repo sync will not '
-            'use superproject to fetch source; while this error is not fatal, '
-            'and you can continue to run repo sync, please run repo init with '
-            'the --no-use-superproject option to stop seeing this warning',
-            file=sys.stderr)
-      if sync_result.fatal and opt.use_superproject is not None:
-        sys.exit(1)
-
-  def _SyncManifest(self, opt):
-    m = self.manifest.manifestProject
-    is_new = not m.Exists
-
-    # If repo has already been initialized, we take -u with the absence of
-    # --standalone-manifest to mean "transition to a standard repo set up",
-    # which necessitates starting fresh.
-    # If --standalone-manifest is set, we always tear everything down and start
-    # anew.
-    if not is_new:
-      was_standalone_manifest = m.config.GetString('manifest.standalone')
-      if opt.standalone_manifest or (
-          was_standalone_manifest and opt.manifest_url):
-        m.config.ClearCache()
-        if m.gitdir and os.path.exists(m.gitdir):
-          platform_utils.rmtree(m.gitdir)
-        if m.worktree and os.path.exists(m.worktree):
-          platform_utils.rmtree(m.worktree)
-
-    is_new = not m.Exists
-    if is_new:
-      if not opt.manifest_url:
-        print('fatal: manifest url is required.', file=sys.stderr)
-        sys.exit(1)
-
-      if not opt.quiet:
-        print('Downloading manifest from %s' %
-              (GitConfig.ForUser().UrlInsteadOf(opt.manifest_url),),
-              file=sys.stderr)
-
-      # The manifest project object doesn't keep track of the path on the
-      # server where this git is located, so let's save that here.
-      mirrored_manifest_git = None
-      if opt.reference:
-        manifest_git_path = urllib.parse.urlparse(opt.manifest_url).path[1:]
-        mirrored_manifest_git = os.path.join(opt.reference, manifest_git_path)
-        if not mirrored_manifest_git.endswith(".git"):
-          mirrored_manifest_git += ".git"
-        if not os.path.exists(mirrored_manifest_git):
-          mirrored_manifest_git = os.path.join(opt.reference,
-                                               '.repo/manifests.git')
-
-      m._InitGitDir(mirror_git=mirrored_manifest_git)
-
-    # If standalone_manifest is set, mark the project as "standalone" -- we'll
-    # still do much of the manifests.git set up, but will avoid actual syncs to
-    # a remote.
-    standalone_manifest = False
-    if opt.standalone_manifest:
-      standalone_manifest = True
-    elif not opt.manifest_url:
-      # If -u is set and --standalone-manifest is not, then we're not in
-      # standalone mode. Otherwise, use config to infer what we were in the last
-      # init.
-      standalone_manifest = bool(m.config.GetString('manifest.standalone'))
-    m.config.SetString('manifest.standalone', opt.manifest_url)
-
-    self._ConfigureDepth(opt)
-
-    # Set the remote URL before the remote branch as we might need it below.
-    if opt.manifest_url:
-      r = m.GetRemote(m.remote.name)
-      r.url = opt.manifest_url
-      r.ResetFetch()
-      r.Save()
-
-    if not standalone_manifest:
-      if opt.manifest_branch:
-        if opt.manifest_branch == 'HEAD':
-          opt.manifest_branch = m.ResolveRemoteHead()
-          if opt.manifest_branch is None:
-            print('fatal: unable to resolve HEAD', file=sys.stderr)
-            sys.exit(1)
-        m.revisionExpr = opt.manifest_branch
-      else:
-        if is_new:
-          default_branch = m.ResolveRemoteHead()
-          if default_branch is None:
-            # If the remote doesn't have HEAD configured, default to master.
-            default_branch = 'refs/heads/master'
-          m.revisionExpr = default_branch
-        else:
-          m.PreSync()
-
-    groups = re.split(r'[,\s]+', opt.groups)
-    all_platforms = ['linux', 'darwin', 'windows']
-    platformize = lambda x: 'platform-' + x
-    if opt.platform == 'auto':
-      if (not opt.mirror and
-              not m.config.GetString('repo.mirror') == 'true'):
-        groups.append(platformize(platform.system().lower()))
-    elif opt.platform == 'all':
-      groups.extend(map(platformize, all_platforms))
-    elif opt.platform in all_platforms:
-      groups.append(platformize(opt.platform))
-    elif opt.platform != 'none':
-      print('fatal: invalid platform flag', file=sys.stderr)
-      sys.exit(1)
-
-    groups = [x for x in groups if x]
-    groupstr = ','.join(groups)
-    if opt.platform == 'auto' and groupstr == self.manifest.GetDefaultGroupsStr():
-      groupstr = None
-    m.config.SetString('manifest.groups', groupstr)
-
-    if opt.reference:
-      m.config.SetString('repo.reference', opt.reference)
-
-    if opt.dissociate:
-      m.config.SetBoolean('repo.dissociate', opt.dissociate)
-
-    if opt.worktree:
-      if opt.mirror:
-        print('fatal: --mirror and --worktree are incompatible',
-              file=sys.stderr)
-        sys.exit(1)
-      if opt.submodules:
-        print('fatal: --submodules and --worktree are incompatible',
-              file=sys.stderr)
-        sys.exit(1)
-      m.config.SetBoolean('repo.worktree', opt.worktree)
-      if is_new:
-        m.use_git_worktrees = True
-      print('warning: --worktree is experimental!', file=sys.stderr)
-
-    if opt.archive:
-      if is_new:
-        m.config.SetBoolean('repo.archive', opt.archive)
-      else:
-        print('fatal: --archive is only supported when initializing a new '
-              'workspace.', file=sys.stderr)
-        print('Either delete the .repo folder in this workspace, or initialize '
-              'in another location.', file=sys.stderr)
-        sys.exit(1)
-
-    if opt.mirror:
-      if is_new:
-        m.config.SetBoolean('repo.mirror', opt.mirror)
-      else:
-        print('fatal: --mirror is only supported when initializing a new '
-              'workspace.', file=sys.stderr)
-        print('Either delete the .repo folder in this workspace, or initialize '
-              'in another location.', file=sys.stderr)
-        sys.exit(1)
-
-    if opt.partial_clone is not None:
-      if opt.mirror:
-        print('fatal: --mirror and --partial-clone are mutually exclusive',
-              file=sys.stderr)
-        sys.exit(1)
-      m.config.SetBoolean('repo.partialclone', opt.partial_clone)
-      if opt.clone_filter:
-        m.config.SetString('repo.clonefilter', opt.clone_filter)
-    elif m.config.GetBoolean('repo.partialclone'):
-      opt.clone_filter = m.config.GetString('repo.clonefilter')
-    else:
-      opt.clone_filter = None
-
-    if opt.partial_clone_exclude is not None:
-      m.config.SetString('repo.partialcloneexclude', opt.partial_clone_exclude)
-
-    if opt.clone_bundle is None:
-      opt.clone_bundle = False if opt.partial_clone else True
-    else:
-      m.config.SetBoolean('repo.clonebundle', opt.clone_bundle)
-
-    if opt.submodules:
-      m.config.SetBoolean('repo.submodules', opt.submodules)
-
-    if opt.use_superproject is not None:
-      m.config.SetBoolean('repo.superproject', opt.use_superproject)
-
-    if standalone_manifest:
-      if is_new:
-        manifest_name = 'default.xml'
-        manifest_data = fetch.fetch_file(opt.manifest_url)
-        dest = os.path.join(m.worktree, manifest_name)
-        os.makedirs(os.path.dirname(dest), exist_ok=True)
-        with open(dest, 'wb') as f:
-          f.write(manifest_data)
-      return
-
-    if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet, verbose=opt.verbose,
-                              clone_bundle=opt.clone_bundle,
-                              current_branch_only=opt.current_branch_only,
-                              tags=opt.tags, submodules=opt.submodules,
-                              clone_filter=opt.clone_filter,
-                              partial_clone_exclude=self.manifest.PartialCloneExclude):
-      r = m.GetRemote(m.remote.name)
-      print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
-
-      # Better delete the manifest git dir if we created it; otherwise next
-      # time (when user fixes problems) we won't go through the "is_new" logic.
-      if is_new:
-        platform_utils.rmtree(m.gitdir)
-      sys.exit(1)
-
-    if opt.manifest_branch:
-      m.MetaBranchSwitch(submodules=opt.submodules)
-
-    syncbuf = SyncBuffer(m.config)
-    m.Sync_LocalHalf(syncbuf, submodules=opt.submodules)
-    syncbuf.Finish()
-
-    if is_new or m.CurrentBranch is None:
-      if not m.StartBranch('default'):
-        print('fatal: cannot create default in manifest', file=sys.stderr)
-        sys.exit(1)
-
-  def _LinkManifest(self, name):
-    if not name:
-      print('fatal: manifest name (-m) is required.', file=sys.stderr)
-      sys.exit(1)
-
-    try:
-      self.manifest.Link(name)
-    except ManifestParseError as e:
-      print("fatal: manifest '%s' not available" % name, file=sys.stderr)
-      print('fatal: %s' % str(e), file=sys.stderr)
+    if not self.manifest.manifestProject.Sync(
+        manifest_url=opt.manifest_url,
+        manifest_branch=opt.manifest_branch,
+        standalone_manifest=opt.standalone_manifest,
+        groups=opt.groups,
+        platform=opt.platform,
+        mirror=opt.mirror,
+        dissociate=opt.dissociate,
+        reference=opt.reference,
+        worktree=opt.worktree,
+        submodules=opt.submodules,
+        archive=opt.archive,
+        partial_clone=opt.partial_clone,
+        clone_filter=opt.clone_filter,
+        partial_clone_exclude=opt.partial_clone_exclude,
+        clone_bundle=opt.clone_bundle,
+        git_lfs=opt.git_lfs,
+        use_superproject=opt.use_superproject,
+        verbose=opt.verbose,
+        current_branch_only=opt.current_branch_only,
+        tags=opt.tags,
+        depth=opt.depth,
+        git_event_log=self.git_event_log,
+        manifest_name=opt.manifest_name):
       sys.exit(1)
 
   def _Prompt(self, prompt, value):
@@ -428,25 +226,6 @@ to update the working directory files.
     if a in ('y', 'yes', 't', 'true', 'on'):
       gc.SetString('color.ui', 'auto')
 
-  def _ConfigureDepth(self, opt):
-    """Configure the depth we'll sync down.
-
-    Args:
-      opt: Options from optparse.  We care about opt.depth.
-    """
-    # Opt.depth will be non-None if user actually passed --depth to repo init.
-    if opt.depth is not None:
-      if opt.depth > 0:
-        # Positive values will set the depth.
-        depth = str(opt.depth)
-      else:
-        # Negative numbers will clear the depth; passing None to SetString
-        # will do that.
-        depth = None
-
-      # We store the depth in the main manifest project.
-      self.manifest.manifestProject.config.SetString('repo.depth', depth)
-
   def _DisplayResult(self, opt):
     if self.manifest.IsMirror:
       init_type = 'mirror '
@@ -471,11 +250,19 @@ to update the working directory files.
 
     # Check this here, else manifest will be tagged "not new" and init won't be
     # possible anymore without removing the .repo/manifests directory.
-    if opt.archive and opt.mirror:
-      self.OptionParser.error('--mirror and --archive cannot be used together.')
+    if opt.mirror:
+      if opt.archive:
+        self.OptionParser.error('--mirror and --archive cannot be used '
+                                'together.')
+      if opt.use_superproject is not None:
+        self.OptionParser.error('--mirror and --use-superproject cannot be '
+                                'used together.')
+    if opt.archive and opt.use_superproject is not None:
+      self.OptionParser.error('--archive and --use-superproject cannot be used '
+                              'together.')
 
-    if opt.standalone_manifest and (
-        opt.manifest_branch or opt.manifest_name != 'default.xml'):
+    if opt.standalone_manifest and (opt.manifest_branch or
+                                    opt.manifest_name != 'default.xml'):
       self.OptionParser.error('--manifest-branch and --manifest-name cannot'
                               ' be used with --standalone-manifest.')
 
@@ -509,8 +296,12 @@ to update the working directory files.
     # Handle new --repo-rev requests.
     if opt.repo_rev:
       wrapper = Wrapper()
-      remote_ref, rev = wrapper.check_repo_rev(
-          rp.gitdir, opt.repo_rev, repo_verify=opt.repo_verify, quiet=opt.quiet)
+      try:
+        remote_ref, rev = wrapper.check_repo_rev(
+            rp.gitdir, opt.repo_rev, repo_verify=opt.repo_verify, quiet=opt.quiet)
+      except wrapper.CloneFailure:
+        print('fatal: double check your --repo-rev setting.', file=sys.stderr)
+        sys.exit(1)
       branch = rp.GetBranch('default')
       branch.merge = remote_ref
       rp.work_git.reset('--hard', rev)
@@ -521,10 +312,6 @@ to update the working directory files.
       git_require((2, 15, 0), fail=True, msg='git gc worktree corruption')
 
     self._SyncManifest(opt)
-    self._LinkManifest(opt.manifest_name)
-
-    if self.manifest.manifestProject.config.GetBoolean('repo.superproject'):
-      self._CloneSuperproject(opt)
 
     if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
       if opt.config_name or self._ShouldConfigureUser(opt):

subcmds/list.py

@@ -77,16 +77,17 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
       args: Positional args.  Can be a list of projects to list, or empty.
     """
     if not opt.regex:
-      projects = self.GetProjects(args, groups=opt.groups, missing_ok=opt.all)
+      projects = self.GetProjects(args, groups=opt.groups, missing_ok=opt.all,
+                                  all_manifests=not opt.this_manifest_only)
     else:
-      projects = self.FindProjects(args)
+      projects = self.FindProjects(args, all_manifests=not opt.this_manifest_only)
 
     def _getpath(x):
       if opt.fullpath:
         return x.worktree
       if opt.relative_to:
         return os.path.relpath(x.worktree, opt.relative_to)
-      return x.relpath
+      return x.RelPath(local=opt.this_manifest_only)
 
     lines = []
     for project in projects:

subcmds/manifest.py

@@ -15,6 +15,7 @@
 import json
 import os
 import sys
+import optparse
 
 from command import PagedCommand
 
@@ -75,7 +76,7 @@ to indicate the remote ref to push changes to via 'repo upload'.
     p.add_option('-o', '--output-file',
                  dest='output_file',
                  default='-',
-                 help='file to save the manifest to',
+                 help='file to save the manifest to. (Filename prefix for multi-tree.)',
                  metavar='-|NAME.xml')
 
   def _Output(self, opt):
@@ -83,36 +84,45 @@ to indicate the remote ref to push changes to via 'repo upload'.
     if opt.manifest_name:
       self.manifest.Override(opt.manifest_name, False)
 
-    if opt.output_file == '-':
-      fd = sys.stdout
-    else:
-      fd = open(opt.output_file, 'w')
+    for manifest in self.ManifestList(opt):
+      output_file = opt.output_file
+      if output_file == '-':
+        fd = sys.stdout
+      else:
+        if manifest.path_prefix:
+          output_file = f'{opt.output_file}:{manifest.path_prefix.replace("/", "%2f")}'
+        fd = open(output_file, 'w')
 
-    self.manifest.SetUseLocalManifests(not opt.ignore_local_manifests)
+      manifest.SetUseLocalManifests(not opt.ignore_local_manifests)
 
-    if opt.json:
-      print('warning: --json is experimental!', file=sys.stderr)
-      doc = self.manifest.ToDict(peg_rev=opt.peg_rev,
-                                 peg_rev_upstream=opt.peg_rev_upstream,
-                                 peg_rev_dest_branch=opt.peg_rev_dest_branch)
+      if opt.json:
+        print('warning: --json is experimental!', file=sys.stderr)
+        doc = manifest.ToDict(peg_rev=opt.peg_rev,
+                                   peg_rev_upstream=opt.peg_rev_upstream,
+                                   peg_rev_dest_branch=opt.peg_rev_dest_branch)
+
+        json_settings = {
+            # JSON style guide says Uunicode characters are fully allowed.
+            'ensure_ascii': False,
+            # We use 2 space indent to match JSON style guide.
+            'indent': 2 if opt.pretty else None,
+            'separators': (',', ': ') if opt.pretty else (',', ':'),
+            'sort_keys': True,
+        }
+        fd.write(json.dumps(doc, **json_settings))
+      else:
+        manifest.Save(fd,
+                      peg_rev=opt.peg_rev,
+                      peg_rev_upstream=opt.peg_rev_upstream,
+                      peg_rev_dest_branch=opt.peg_rev_dest_branch)
+      if output_file != '-':
+        fd.close()
+        if manifest.path_prefix:
+          print(f'Saved {manifest.path_prefix} submanifest to {output_file}',
+                file=sys.stderr)
+        else:
+          print(f'Saved manifest to {output_file}', file=sys.stderr)
 
-      json_settings = {
-          # JSON style guide says Uunicode characters are fully allowed.
-          'ensure_ascii': False,
-          # We use 2 space indent to match JSON style guide.
-          'indent': 2 if opt.pretty else None,
-          'separators': (',', ': ') if opt.pretty else (',', ':'),
-          'sort_keys': True,
-      }
-      fd.write(json.dumps(doc, **json_settings))
-    else:
-      self.manifest.Save(fd,
-                         peg_rev=opt.peg_rev,
-                         peg_rev_upstream=opt.peg_rev_upstream,
-                         peg_rev_dest_branch=opt.peg_rev_dest_branch)
-    fd.close()
-    if opt.output_file != '-':
-      print('Saved manifest to %s' % opt.output_file, file=sys.stderr)
 
   def ValidateOptions(self, opt, args):
     if args:

subcmds/overview.py

@@ -47,7 +47,7 @@ are displayed.
 
   def Execute(self, opt, args):
     all_branches = []
-    for project in self.GetProjects(args):
+    for project in self.GetProjects(args, all_manifests=not opt.this_manifest_only):
       br = [project.GetUploadableBranch(x)
             for x in project.GetBranches()]
       br = [x for x in br if x]
@@ -76,7 +76,7 @@ are displayed.
       if project != branch.project:
         project = branch.project
         out.nl()
-        out.project('project %s/' % project.relpath)
+        out.project('project %s/' % project.RelPath(local=opt.this_manifest_only))
         out.nl()
 
       commits = branch.commits

subcmds/prune.py

@@ -31,7 +31,7 @@ class Prune(PagedCommand):
     return project.PruneHeads()
 
   def Execute(self, opt, args):
-    projects = self.GetProjects(args)
+    projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     # NB: Should be able to refactor this module to display summary as results
     # come back from children.
@@ -63,7 +63,7 @@ class Prune(PagedCommand):
       if project != branch.project:
         project = branch.project
         out.nl()
-        out.project('project %s/' % project.relpath)
+        out.project('project %s/' % project.RelPath(local=opt.this_manifest_only))
         out.nl()
 
       print('%s %-33s ' % (

subcmds/rebase.py

@@ -69,7 +69,7 @@ branch but need to incorporate new upstream changes "underneath" them.
                       'consistent if you previously synced to a manifest)')
 
   def Execute(self, opt, args):
-    all_projects = self.GetProjects(args)
+    all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
     one_project = len(all_projects) == 1
 
     if opt.interactive and not one_project:
@@ -98,6 +98,7 @@ branch but need to incorporate new upstream changes "underneath" them.
     config = self.manifest.manifestProject.config
     out = RebaseColoring(config)
     out.redirect(sys.stdout)
+    _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
 
     ret = 0
     for project in all_projects:
@@ -107,7 +108,7 @@ branch but need to incorporate new upstream changes "underneath" them.
       cb = project.CurrentBranch
       if not cb:
         if one_project:
-          print("error: project %s has a detached HEAD" % project.relpath,
+          print("error: project %s has a detached HEAD" % _RelPath(project),
                 file=sys.stderr)
           return 1
         # ignore branches with detatched HEADs
@@ -117,7 +118,7 @@ branch but need to incorporate new upstream changes "underneath" them.
       if not upbranch.LocalMerge:
         if one_project:
           print("error: project %s does not track any remote branches"
-                % project.relpath, file=sys.stderr)
+                % _RelPath(project), file=sys.stderr)
           return 1
         # ignore branches without remotes
         continue
@@ -130,7 +131,7 @@ branch but need to incorporate new upstream changes "underneath" them.
       args.append(upbranch.LocalMerge)
 
       out.project('project %s: rebasing %s -> %s',
-                  project.relpath, cb, upbranch.LocalMerge)
+                  _RelPath(project), cb, upbranch.LocalMerge)
       out.nl()
       out.flush()
 

subcmds/stage.py

@@ -50,7 +50,9 @@ The '%prog' command stages files to prepare the next commit.
       self.Usage()
 
   def _Interactive(self, opt, args):
-    all_projects = [p for p in self.GetProjects(args) if p.IsDirty()]
+    all_projects = [
+        p for p in self.GetProjects(args, all_manifests=not opt.this_manifest_only)
+        if p.IsDirty()]
     if not all_projects:
       print('no projects have uncommitted modifications', file=sys.stderr)
       return
@@ -62,7 +64,8 @@ The '%prog' command stages files to prepare the next commit.
 
       for i in range(len(all_projects)):
         project = all_projects[i]
-        out.write('%3d:    %s', i + 1, project.relpath + '/')
+        out.write('%3d:    %s', i + 1,
+                  project.RelPath(local=opt.this_manifest_only) + '/')
         out.nl()
       out.nl()
 
@@ -99,7 +102,9 @@ The '%prog' command stages files to prepare the next commit.
           _AddI(all_projects[a_index - 1])
           continue
 
-      projects = [p for p in all_projects if a in [p.name, p.relpath]]
+      projects = [
+          p for p in all_projects
+          if a in [p.name, p.RelPath(local=opt.this_manifest_only)]]
       if len(projects) == 1:
         _AddI(projects[0])
         continue

subcmds/start.py

@@ -84,7 +84,8 @@ revision specified in the manifest.
         projects = ['.']  # start it in the local project by default
 
     all_projects = self.GetProjects(projects,
-                                    missing_ok=bool(self.gitc_manifest))
+                                    missing_ok=bool(self.gitc_manifest),
+                                    all_manifests=not opt.this_manifest_only)
 
     # This must happen after we find all_projects, since GetProjects may need
     # the local directory, which will disappear once we save the GITC manifest.
@@ -137,6 +138,6 @@ revision specified in the manifest.
 
     if err:
       for p in err:
-        print("error: %s/: cannot start %s" % (p.relpath, nb),
+        print("error: %s/: cannot start %s" % (p.RelPath(local=opt.this_manifest_only), nb),
               file=sys.stderr)
       sys.exit(1)

subcmds/status.py

@@ -117,7 +117,7 @@ the following meanings:
       outstring.append(''.join([status_header, item, '/']))
 
   def Execute(self, opt, args):
-    all_projects = self.GetProjects(args)
+    all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     def _ProcessResults(_pool, _output, results):
       ret = 0
@@ -141,9 +141,10 @@ the following meanings:
     if opt.orphans:
       proj_dirs = set()
       proj_dirs_parents = set()
-      for project in self.GetProjects(None, missing_ok=True):
-        proj_dirs.add(project.relpath)
-        (head, _tail) = os.path.split(project.relpath)
+      for project in self.GetProjects(None, missing_ok=True, all_manifests=not opt.this_manifest_only):
+        relpath = project.RelPath(local=opt.this_manifest_only)
+        proj_dirs.add(relpath)
+        (head, _tail) = os.path.split(relpath)
         while head != "":
           proj_dirs_parents.add(head)
           (head, _tail) = os.path.split(head)

subcmds/sync.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import errno
+import collections
 import functools
 import http.cookiejar as cookielib
 import io
@@ -67,6 +67,7 @@ _ONE_DAY_S = 24 * 60 * 60
 class Sync(Command, MirrorSafeCommand):
   jobs = 1
   COMMON = True
+  MULTI_MANIFEST_SUPPORT = True
   helpSummary = "Update working tree to the latest revision"
   helpUsage = """
 %prog [<project>...]
@@ -170,9 +171,9 @@ later is required to fix a server side protocol bug.
   PARALLEL_JOBS = 1
 
   def _CommonOptions(self, p):
-    if self.manifest:
+    if self.outer_client and self.outer_client.manifest:
       try:
-        self.PARALLEL_JOBS = self.manifest.default.sync_j
+        self.PARALLEL_JOBS = self.outer_client.manifest.default.sync_j
       except ManifestParseError:
         pass
     super()._CommonOptions(p)
@@ -235,24 +236,25 @@ later is required to fix a server side protocol bug.
                  dest='fetch_submodules', action='store_true',
                  help='fetch submodules from server')
     p.add_option('--use-superproject', action='store_true',
-                 help='use the manifest superproject to sync projects')
+                 help='use the manifest superproject to sync projects; implies -c')
     p.add_option('--no-use-superproject', action='store_false',
                  dest='use_superproject',
                  help='disable use of manifest superprojects')
-    p.add_option('--tags',
-                 action='store_false',
+    p.add_option('--tags', action='store_true',
                  help='fetch tags')
     p.add_option('--no-tags',
                  dest='tags', action='store_false',
-                 help="don't fetch tags")
+                 help="don't fetch tags (default)")
     p.add_option('--optimized-fetch',
                  dest='optimized_fetch', action='store_true',
                  help='only fetch projects fixed to sha1 if revision does not exist locally')
     p.add_option('--retry-fetches',
                  default=0, action='store', type='int',
                  help='number of times to retry fetches on transient errors')
-    p.add_option('--prune', dest='prune', action='store_true',
-                 help='delete refs that no longer exist on the remote')
+    p.add_option('--prune', action='store_true',
+                 help='delete refs that no longer exist on the remote (default)')
+    p.add_option('--no-prune', dest='prune', action='store_false',
+                 help='do not delete refs that no longer exist on the remote')
     if show_smart:
       p.add_option('-s', '--smart-sync',
                    dest='smart_sync', action='store_true',
@@ -269,64 +271,117 @@ later is required to fix a server side protocol bug.
                  dest='repo_upgraded', action='store_true',
                  help=SUPPRESS_HELP)
 
-  def _GetBranch(self):
-    """Returns the branch name for getting the approved manifest."""
-    p = self.manifest.manifestProject
-    b = p.GetBranch(p.CurrentBranch)
+  def _GetBranch(self, manifest_project):
+    """Returns the branch name for getting the approved smartsync manifest.
+
+    Args:
+      manifest_project: the manifestProject to query.
+    """
+    b = manifest_project.GetBranch(manifest_project.CurrentBranch)
     branch = b.merge
     if branch.startswith(R_HEADS):
       branch = branch[len(R_HEADS):]
     return branch
 
-  def _GetCurrentBranchOnly(self, opt):
-    """Returns True if current-branch or use-superproject options are enabled."""
-    return opt.current_branch_only or git_superproject.UseSuperproject(opt, self.manifest)
+  def _GetCurrentBranchOnly(self, opt, manifest):
+    """Returns whether current-branch or use-superproject options are enabled.
 
-  def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests, superproject_logging_data):
-    """Update revisionId of every project with the SHA from superproject.
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      manifest: The manifest to use.
 
-    This function updates each project's revisionId with SHA from superproject.
-    It writes the updated manifest into a file and reloads the manifest from it.
+    Returns:
+      True if a superproject is requested, otherwise the value of the
+      current_branch option (True, False or None).
+    """
+    return git_superproject.UseSuperproject(opt.use_superproject, manifest) or opt.current_branch_only
+
+  def _UpdateProjectsRevisionId(self, opt, args, superproject_logging_data,
+                                manifest):
+    """Update revisionId of projects with the commit hash from the superproject.
+
+    This function updates each project's revisionId with the commit hash from
+    the superproject.  It writes the updated manifest into a file and reloads
+    the manifest from it.  When appropriate, sub manifests are also processed.
 
     Args:
       opt: Program options returned from optparse.  See _Options().
       args: Arguments to pass to GetProjects. See the GetProjects
           docstring for details.
-      load_local_manifests: Whether to load local manifests.
-      superproject_logging_data: A dictionary of superproject data that is to be logged.
-
-    Returns:
-      Returns path to the overriding manifest file instead of None.
+      superproject_logging_data: A dictionary of superproject data to log.
+      manifest: The manifest to use.
     """
-    print_messages = git_superproject.PrintMessages(opt, self.manifest)
-    superproject = git_superproject.Superproject(self.manifest,
-                                                 self.repodir,
-                                                 self.git_event_log,
-                                                 quiet=opt.quiet,
-                                                 print_messages=print_messages)
-    if opt.local_only:
-      manifest_path = superproject.manifest_path
+    have_superproject = manifest.superproject or any(
+        m.superproject for m in manifest.all_children)
+    if not have_superproject:
+      return
+
+    if opt.local_only and manifest.superproject:
+      manifest_path = manifest.superproject.manifest_path
       if manifest_path:
-        self._ReloadManifest(manifest_path, load_local_manifests)
-      return manifest_path
+        self._ReloadManifest(manifest_path, manifest)
+      return
 
     all_projects = self.GetProjects(args,
                                     missing_ok=True,
-                                    submodules_ok=opt.fetch_submodules)
-    update_result = superproject.UpdateProjectsRevisionId(all_projects)
-    manifest_path = update_result.manifest_path
-    superproject_logging_data['updatedrevisionid'] = bool(manifest_path)
-    if manifest_path:
-      self._ReloadManifest(manifest_path, load_local_manifests)
+                                    submodules_ok=opt.fetch_submodules,
+                                    manifest=manifest,
+                                    all_manifests=not opt.this_manifest_only)
+
+    per_manifest = collections.defaultdict(list)
+    manifest_paths = {}
+    if opt.this_manifest_only:
+      per_manifest[manifest.path_prefix] = all_projects
     else:
-      if print_messages:
-        print('warning: Update of revisionId from superproject has failed, '
-              'repo sync will not use superproject to fetch the source. ',
-              'Please resync with the --no-use-superproject option to avoid this repo warning.',
-              file=sys.stderr)
-      if update_result.fatal and opt.use_superproject is not None:
-        sys.exit(1)
-    return manifest_path
+      for p in all_projects:
+        per_manifest[p.manifest.path_prefix].append(p)
+
+    superproject_logging_data = {}
+    need_unload = False
+    for m in self.ManifestList(opt):
+      if not m.path_prefix in per_manifest:
+        continue
+      use_super = git_superproject.UseSuperproject(opt.use_superproject, m)
+      if superproject_logging_data:
+        superproject_logging_data['multimanifest'] = True
+      superproject_logging_data.update(
+          superproject=use_super,
+          haslocalmanifests=bool(m.HasLocalManifests),
+          hassuperprojecttag=bool(m.superproject),
+      )
+      if use_super and (m.IsMirror or m.IsArchive):
+        # Don't use superproject, because we have no working tree.
+        use_super = False
+        superproject_logging_data['superproject'] = False
+        superproject_logging_data['noworktree'] = True
+        if opt.use_superproject is not False:
+          print(f'{m.path_prefix}: not using superproject because there is no '
+                'working tree.')
+
+      if not use_super:
+        continue
+      m.superproject.SetQuiet(opt.quiet)
+      print_messages = git_superproject.PrintMessages(opt.use_superproject, m)
+      m.superproject.SetPrintMessages(print_messages)
+      update_result = m.superproject.UpdateProjectsRevisionId(
+          per_manifest[m.path_prefix], git_event_log=self.git_event_log)
+      manifest_path = update_result.manifest_path
+      superproject_logging_data['updatedrevisionid'] = bool(manifest_path)
+      if manifest_path:
+        m.SetManifestOverride(manifest_path)
+        need_unload = True
+      else:
+        if print_messages:
+          print(f'{m.path_prefix}: warning: Update of revisionId from '
+                'superproject has failed, repo sync will not use superproject '
+                'to fetch the source. ',
+                'Please resync with the --no-use-superproject option to avoid '
+                'this repo warning.',
+                file=sys.stderr)
+        if update_result.fatal and opt.use_superproject is not None:
+          sys.exit(1)
+    if need_unload:
+      m.outer_client.manifest.Unload()
 
   def _FetchProjectList(self, opt, projects):
     """Main function of the fetch worker.
@@ -360,16 +415,16 @@ later is required to fix a server side protocol bug.
           quiet=opt.quiet,
           verbose=opt.verbose,
           output_redir=buf,
-          current_branch_only=self._GetCurrentBranchOnly(opt),
+          current_branch_only=self._GetCurrentBranchOnly(opt, project.manifest),
           force_sync=opt.force_sync,
           clone_bundle=opt.clone_bundle,
-          tags=opt.tags, archive=self.manifest.IsArchive,
+          tags=opt.tags, archive=project.manifest.IsArchive,
           optimized_fetch=opt.optimized_fetch,
           retry_fetches=opt.retry_fetches,
           prune=opt.prune,
           ssh_proxy=self.ssh_proxy,
-          clone_filter=self.manifest.CloneFilter,
-          partial_clone_exclude=self.manifest.PartialCloneExclude)
+          clone_filter=project.manifest.CloneFilter,
+          partial_clone_exclude=project.manifest.PartialCloneExclude)
 
       output = buf.getvalue()
       if (opt.verbose or not success) and output:
@@ -448,8 +503,8 @@ later is required to fix a server side protocol bug.
       else:
         pm.update(inc=0, msg='warming up')
         chunksize = 4
-      with multiprocessing.Pool(
-          jobs, initializer=self._FetchInitChild, initargs=(ssh_proxy,)) as pool:
+      with multiprocessing.Pool(jobs, initializer=self._FetchInitChild,
+                                initargs=(ssh_proxy,)) as pool:
         results = pool.imap_unordered(
             functools.partial(self._FetchProjectList, opt),
             projects_list,
@@ -466,13 +521,13 @@ later is required to fix a server side protocol bug.
     pm.end()
     self._fetch_times.Save()
 
-    if not self.manifest.IsArchive:
+    if not self.outer_client.manifest.IsArchive:
       self._GCProjects(projects, opt, err_event)
 
     return (ret, fetched)
 
-  def _FetchMain(self, opt, args, all_projects, err_event, manifest_name,
-                 load_local_manifests, ssh_proxy):
+  def _FetchMain(self, opt, args, all_projects, err_event,
+                 ssh_proxy, manifest):
     """The main network fetch loop.
 
     Args:
@@ -480,14 +535,13 @@ later is required to fix a server side protocol bug.
       args: Command line args used to filter out projects.
       all_projects: List of all projects that should be fetched.
       err_event: Whether an error was hit while processing.
-      manifest_name: Manifest file to be reloaded.
-      load_local_manifests: Whether to load local manifests.
       ssh_proxy: SSH manager for clients & masters.
+      manifest: The manifest to use.
 
     Returns:
       List of all projects that should be checked out.
     """
-    rp = self.manifest.repoProject
+    rp = manifest.repoProject
 
     to_fetch = []
     now = time.time()
@@ -511,10 +565,12 @@ later is required to fix a server side protocol bug.
     # Iteratively fetch missing and/or nested unregistered submodules
     previously_missing_set = set()
     while True:
-      self._ReloadManifest(manifest_name, load_local_manifests)
+      self._ReloadManifest(None, manifest)
       all_projects = self.GetProjects(args,
                                       missing_ok=True,
-                                      submodules_ok=opt.fetch_submodules)
+                                      submodules_ok=opt.fetch_submodules,
+                                      manifest=manifest,
+                                      all_manifests=not opt.this_manifest_only)
       missing = []
       for project in all_projects:
         if project.gitdir not in fetched:
@@ -546,7 +602,7 @@ later is required to fix a server side protocol bug.
       Whether the fetch was successful.
     """
     start = time.time()
-    syncbuf = SyncBuffer(self.manifest.manifestProject.config,
+    syncbuf = SyncBuffer(project.manifest.manifestProject.config,
                          detach_head=detach_head)
     success = False
     try:
@@ -605,11 +661,11 @@ later is required to fix a server side protocol bug.
     pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet)
     pm.update(inc=0, msg='prescan')
 
-    gc_gitdirs = {}
+    tidy_dirs = {}
     for project in projects:
       # Make sure pruning never kicks in with shared projects.
       if (not project.use_git_worktrees and
-              len(project.manifest.GetProjectsWithName(project.name)) > 1):
+              len(project.manifest.GetProjectsWithName(project.name, all_manifests=True)) > 1):
         if not opt.quiet:
           print('\r%s: Shared project %s found, disabling pruning.' %
                 (project.relpath, project.name))
@@ -622,17 +678,29 @@ later is required to fix a server side protocol bug.
                 % (project.relpath,),
                 file=sys.stderr)
           project.config.SetString('gc.pruneExpire', 'never')
-      gc_gitdirs[project.gitdir] = project.bare_git
-
-    pm.update(inc=len(projects) - len(gc_gitdirs), msg='warming up')
+      project.config.SetString('gc.autoDetach', 'false')
+      # Only call git gc once per objdir, but call pack-refs for the remainder.
+      if project.objdir not in tidy_dirs:
+        tidy_dirs[project.objdir] = (
+            True,  # Run a full gc.
+            project.bare_git,
+        )
+      elif project.gitdir not in tidy_dirs:
+        tidy_dirs[project.gitdir] = (
+            False,  # Do not run a full gc; just run pack-refs.
+            project.bare_git,
+        )
 
     cpu_count = os.cpu_count()
     jobs = min(self.jobs, cpu_count)
 
     if jobs < 2:
-      for bare_git in gc_gitdirs.values():
+      for (run_gc, bare_git) in tidy_dirs.values():
         pm.update(msg=bare_git._project.name)
-        bare_git.gc('--auto')
+        if run_gc:
+          bare_git.gc('--auto')
+        else:
+          bare_git.pack_refs()
       pm.end()
       return
 
@@ -641,11 +709,14 @@ later is required to fix a server side protocol bug.
     threads = set()
     sem = _threading.Semaphore(jobs)
 
-    def GC(bare_git):
+    def tidy_up(run_gc, bare_git):
       pm.start(bare_git._project.name)
       try:
         try:
-          bare_git.gc('--auto', config=config)
+          if run_gc:
+            bare_git.gc('--auto', config=config)
+          else:
+            bare_git.pack_refs(config=config)
         except GitError:
           err_event.set()
         except Exception:
@@ -655,11 +726,11 @@ later is required to fix a server side protocol bug.
         pm.finish(bare_git._project.name)
         sem.release()
 
-    for bare_git in gc_gitdirs.values():
+    for (run_gc, bare_git) in tidy_dirs.values():
       if err_event.is_set() and opt.fail_fast:
         break
       sem.acquire()
-      t = _threading.Thread(target=GC, args=(bare_git,))
+      t = _threading.Thread(target=tidy_up, args=(run_gc, bare_git,))
       t.daemon = True
       threads.add(t)
       t.start()
@@ -668,28 +739,41 @@ later is required to fix a server side protocol bug.
       t.join()
     pm.end()
 
-  def _ReloadManifest(self, manifest_name=None, load_local_manifests=True):
+  def _ReloadManifest(self, manifest_name, manifest):
     """Reload the manfiest from the file specified by the |manifest_name|.
 
     It unloads the manifest if |manifest_name| is None.
 
     Args:
       manifest_name: Manifest file to be reloaded.
-      load_local_manifests: Whether to load local manifests.
+      manifest: The manifest to use.
     """
     if manifest_name:
-      # Override calls _Unload already
-      self.manifest.Override(manifest_name, load_local_manifests=load_local_manifests)
+      # Override calls Unload already
+      manifest.Override(manifest_name)
     else:
-      self.manifest._Unload()
+      manifest.Unload()
 
-  def UpdateProjectList(self, opt):
+  def UpdateProjectList(self, opt, manifest):
+    """Update the cached projects list for |manifest|
+
+    In a multi-manifest checkout, each manifest has its own project.list.
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      manifest: The manifest to use.
+
+    Returns:
+      0: success
+      1: failure
+    """
     new_project_paths = []
-    for project in self.GetProjects(None, missing_ok=True):
+    for project in self.GetProjects(None, missing_ok=True, manifest=manifest,
+                                    all_manifests=False):
       if project.relpath:
         new_project_paths.append(project.relpath)
     file_name = 'project.list'
-    file_path = os.path.join(self.repodir, file_name)
+    file_path = os.path.join(manifest.subdir, file_name)
     old_project_paths = []
 
     if os.path.exists(file_path):
@@ -701,16 +785,16 @@ later is required to fix a server side protocol bug.
           continue
         if path not in new_project_paths:
           # If the path has already been deleted, we don't need to do it
-          gitdir = os.path.join(self.manifest.topdir, path, '.git')
+          gitdir = os.path.join(manifest.topdir, path, '.git')
           if os.path.exists(gitdir):
             project = Project(
-                manifest=self.manifest,
+                manifest=manifest,
                 name=path,
                 remote=RemoteSpec('origin'),
                 gitdir=gitdir,
                 objdir=gitdir,
                 use_git_worktrees=os.path.isfile(gitdir),
-                worktree=os.path.join(self.manifest.topdir, path),
+                worktree=os.path.join(manifest.topdir, path),
                 relpath=path,
                 revisionExpr='HEAD',
                 revisionId=None,
@@ -726,7 +810,7 @@ later is required to fix a server side protocol bug.
       fd.write('\n')
     return 0
 
-  def UpdateCopyLinkfileList(self):
+  def UpdateCopyLinkfileList(self, manifest):
     """Save all dests of copyfile and linkfile, and update them if needed.
 
     Returns:
@@ -735,7 +819,8 @@ later is required to fix a server side protocol bug.
     new_paths = {}
     new_linkfile_paths = []
     new_copyfile_paths = []
-    for project in self.GetProjects(None, missing_ok=True):
+    for project in self.GetProjects(None, missing_ok=True,
+                                    manifest=manifest, all_manifests=False):
       new_linkfile_paths.extend(x.dest for x in project.linkfiles)
       new_copyfile_paths.extend(x.dest for x in project.copyfiles)
 
@@ -745,14 +830,14 @@ later is required to fix a server side protocol bug.
     }
 
     copylinkfile_name = 'copy-link-files.json'
-    copylinkfile_path = os.path.join(self.manifest.repodir, copylinkfile_name)
+    copylinkfile_path = os.path.join(manifest.subdir, copylinkfile_name)
     old_copylinkfile_paths = {}
 
     if os.path.exists(copylinkfile_path):
       with open(copylinkfile_path, 'rb') as fp:
         try:
           old_copylinkfile_paths = json.load(fp)
-        except:
+        except Exception:
           print('error: %s is not a json formatted file.' %
                 copylinkfile_path, file=sys.stderr)
           platform_utils.remove(copylinkfile_path)
@@ -776,13 +861,13 @@ later is required to fix a server side protocol bug.
       json.dump(new_paths, fp)
     return True
 
-  def _SmartSyncSetup(self, opt, smart_sync_manifest_path):
-    if not self.manifest.manifest_server:
+  def _SmartSyncSetup(self, opt, smart_sync_manifest_path, manifest):
+    if not manifest.manifest_server:
       print('error: cannot smart sync: no manifest server defined in '
             'manifest', file=sys.stderr)
       sys.exit(1)
 
-    manifest_server = self.manifest.manifest_server
+    manifest_server = manifest.manifest_server
     if not opt.quiet:
       print('Using manifest server %s' % manifest_server)
 
@@ -823,7 +908,7 @@ later is required to fix a server side protocol bug.
     try:
       server = xmlrpc.client.Server(manifest_server, transport=transport)
       if opt.smart_sync:
-        branch = self._GetBranch()
+        branch = self._GetBranch(manifest.manifestProject)
 
         if 'SYNC_TARGET' in os.environ:
           target = os.environ['SYNC_TARGET']
@@ -849,36 +934,68 @@ later is required to fix a server side protocol bug.
                 % (smart_sync_manifest_path, e),
                 file=sys.stderr)
           sys.exit(1)
-        self._ReloadManifest(manifest_name)
+        self._ReloadManifest(manifest_name, manifest)
       else:
         print('error: manifest server RPC call failed: %s' %
               manifest_str, file=sys.stderr)
         sys.exit(1)
     except (socket.error, IOError, xmlrpc.client.Fault) as e:
       print('error: cannot connect to manifest server %s:\n%s'
-            % (self.manifest.manifest_server, e), file=sys.stderr)
+            % (manifest.manifest_server, e), file=sys.stderr)
       sys.exit(1)
     except xmlrpc.client.ProtocolError as e:
       print('error: cannot connect to manifest server %s:\n%d %s'
-            % (self.manifest.manifest_server, e.errcode, e.errmsg),
+            % (manifest.manifest_server, e.errcode, e.errmsg),
             file=sys.stderr)
       sys.exit(1)
 
     return manifest_name
 
+  def _UpdateAllManifestProjects(self, opt, mp, manifest_name):
+    """Fetch & update the local manifest project.
+
+    After syncing the manifest project, if the manifest has any sub manifests,
+    those are recursively processed.
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      mp: the manifestProject to query.
+      manifest_name: Manifest file to be reloaded.
+    """
+    if not mp.standalone_manifest_url:
+      self._UpdateManifestProject(opt, mp, manifest_name)
+
+    if mp.manifest.submanifests:
+      for submanifest in mp.manifest.submanifests.values():
+        child = submanifest.repo_client.manifest
+        child.manifestProject.SyncWithPossibleInit(
+            submanifest,
+            current_branch_only=self._GetCurrentBranchOnly(opt, child),
+            verbose=opt.verbose,
+            tags=opt.tags,
+            git_event_log=self.git_event_log,
+        )
+        self._UpdateAllManifestProjects(opt, child.manifestProject, None)
+
   def _UpdateManifestProject(self, opt, mp, manifest_name):
-    """Fetch & update the local manifest project."""
+    """Fetch & update the local manifest project.
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      mp: the manifestProject to query.
+      manifest_name: Manifest file to be reloaded.
+    """
     if not opt.local_only:
       start = time.time()
       success = mp.Sync_NetworkHalf(quiet=opt.quiet, verbose=opt.verbose,
-                                    current_branch_only=self._GetCurrentBranchOnly(opt),
+                                    current_branch_only=self._GetCurrentBranchOnly(opt, mp.manifest),
                                     force_sync=opt.force_sync,
                                     tags=opt.tags,
                                     optimized_fetch=opt.optimized_fetch,
                                     retry_fetches=opt.retry_fetches,
-                                    submodules=self.manifest.HasSubmodules,
-                                    clone_filter=self.manifest.CloneFilter,
-                                    partial_clone_exclude=self.manifest.PartialCloneExclude)
+                                    submodules=mp.manifest.HasSubmodules,
+                                    clone_filter=mp.manifest.CloneFilter,
+                                    partial_clone_exclude=mp.manifest.PartialCloneExclude)
       finish = time.time()
       self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK,
                              start, finish, success)
@@ -886,15 +1003,16 @@ later is required to fix a server side protocol bug.
     if mp.HasChanges:
       syncbuf = SyncBuffer(mp.config)
       start = time.time()
-      mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules)
+      mp.Sync_LocalHalf(syncbuf, submodules=mp.manifest.HasSubmodules)
       clean = syncbuf.Finish()
       self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL,
                              start, time.time(), clean)
       if not clean:
         sys.exit(1)
-      self._ReloadManifest(manifest_name)
+      self._ReloadManifest(manifest_name, mp.manifest)
+
       if opt.jobs is None:
-        self.jobs = self.manifest.default.sync_j
+        self.jobs = mp.manifest.default.sync_j
 
   def ValidateOptions(self, opt, args):
     if opt.force_broken:
@@ -914,6 +1032,9 @@ later is required to fix a server side protocol bug.
       if None in [opt.manifest_server_username, opt.manifest_server_password]:
         self.OptionParser.error('both -u and -p must be given')
 
+    if opt.prune is None:
+      opt.prune = True
+
   def Execute(self, opt, args):
     if opt.jobs:
       self.jobs = opt.jobs
@@ -921,18 +1042,22 @@ later is required to fix a server side protocol bug.
       soft_limit, _ = _rlimit_nofile()
       self.jobs = min(self.jobs, (soft_limit - 5) // 3)
 
+    manifest = self.outer_manifest
+    if not opt.outer_manifest:
+      manifest = self.manifest
+
     if opt.manifest_name:
-      self.manifest.Override(opt.manifest_name)
+      manifest.Override(opt.manifest_name)
 
     manifest_name = opt.manifest_name
     smart_sync_manifest_path = os.path.join(
-        self.manifest.manifestProject.worktree, 'smart_sync_override.xml')
+        manifest.manifestProject.worktree, 'smart_sync_override.xml')
 
     if opt.clone_bundle is None:
-      opt.clone_bundle = self.manifest.CloneBundle
+      opt.clone_bundle = manifest.CloneBundle
 
     if opt.smart_sync or opt.smart_tag:
-      manifest_name = self._SmartSyncSetup(opt, smart_sync_manifest_path)
+      manifest_name = self._SmartSyncSetup(opt, smart_sync_manifest_path, manifest)
     else:
       if os.path.isfile(smart_sync_manifest_path):
         try:
@@ -943,7 +1068,7 @@ later is required to fix a server side protocol bug.
 
     err_event = multiprocessing.Event()
 
-    rp = self.manifest.repoProject
+    rp = manifest.repoProject
     rp.PreSync()
     cb = rp.CurrentBranch
     if cb:
@@ -953,31 +1078,26 @@ later is required to fix a server side protocol bug.
               'receive updates; run `repo init --repo-rev=stable` to fix.',
               file=sys.stderr)
 
-    mp = self.manifest.manifestProject
-    mp.PreSync()
+    for m in self.ManifestList(opt):
+      mp = m.manifestProject
+      is_standalone_manifest = bool(mp.standalone_manifest_url)
+      if not is_standalone_manifest:
+        mp.PreSync()
 
-    if opt.repo_upgraded:
-      _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
+      if opt.repo_upgraded:
+        _PostRepoUpgrade(m, quiet=opt.quiet)
 
-    if not opt.mp_update:
-      print('Skipping update of local manifest project.')
+    if opt.mp_update:
+      self._UpdateAllManifestProjects(opt, mp, manifest_name)
     else:
-      self._UpdateManifestProject(opt, mp, manifest_name)
+      print('Skipping update of local manifest project.')
 
-    load_local_manifests = not self.manifest.HasLocalManifests
-    use_superproject = git_superproject.UseSuperproject(opt, self.manifest)
-    superproject_logging_data = {
-        'superproject': use_superproject,
-        'haslocalmanifests': bool(self.manifest.HasLocalManifests),
-        'hassuperprojecttag': bool(self.manifest.superproject),
-    }
-    if use_superproject:
-      manifest_name = self._UpdateProjectsRevisionId(
-          opt, args, load_local_manifests, superproject_logging_data) or opt.manifest_name
+    superproject_logging_data = {}
+    self._UpdateProjectsRevisionId(opt, args, superproject_logging_data,
+                                   manifest)
 
     if self.gitc_manifest:
-      gitc_manifest_projects = self.GetProjects(args,
-                                                missing_ok=True)
+      gitc_manifest_projects = self.GetProjects(args, missing_ok=True)
       gitc_projects = []
       opened_projects = []
       for project in gitc_manifest_projects:
@@ -996,7 +1116,7 @@ later is required to fix a server side protocol bug.
         if manifest_name:
           manifest.Override(manifest_name)
         else:
-          manifest.Override(self.manifest.manifestFile)
+          manifest.Override(manifest.manifestFile)
         gitc_utils.generate_gitc_manifest(self.gitc_manifest,
                                           manifest,
                                           gitc_projects)
@@ -1006,26 +1126,28 @@ later is required to fix a server side protocol bug.
       # generate a new args list to represent the opened projects.
       # TODO: make this more reliable -- if there's a project name/path overlap,
       # this may choose the wrong project.
-      args = [os.path.relpath(self.manifest.paths[path].worktree, os.getcwd())
+      args = [os.path.relpath(manifest.paths[path].worktree, os.getcwd())
               for path in opened_projects]
       if not args:
         return
+
     all_projects = self.GetProjects(args,
                                     missing_ok=True,
-                                    submodules_ok=opt.fetch_submodules)
+                                    submodules_ok=opt.fetch_submodules,
+                                    manifest=manifest,
+                                    all_manifests=not opt.this_manifest_only)
 
     err_network_sync = False
     err_update_projects = False
 
-    self._fetch_times = _FetchTimes(self.manifest)
+    self._fetch_times = _FetchTimes(manifest)
     if not opt.local_only:
       with multiprocessing.Manager() as manager:
         with ssh.ProxyManager(manager) as ssh_proxy:
           # Initialize the socket dir once in the parent.
           ssh_proxy.sock()
           all_projects = self._FetchMain(opt, args, all_projects, err_event,
-                                         manifest_name, load_local_manifests,
-                                         ssh_proxy)
+                                         ssh_proxy, manifest)
 
       if opt.network_only:
         return
@@ -1041,23 +1163,24 @@ later is required to fix a server side protocol bug.
                 file=sys.stderr)
           sys.exit(1)
 
-    if self.manifest.IsMirror or self.manifest.IsArchive:
-      # bail out now, we have no working tree
-      return
+    for m in self.ManifestList(opt):
+      if m.IsMirror or m.IsArchive:
+        # bail out now, we have no working tree
+        continue
 
-    if self.UpdateProjectList(opt):
-      err_event.set()
-      err_update_projects = True
-      if opt.fail_fast:
-        print('\nerror: Local checkouts *not* updated.', file=sys.stderr)
-        sys.exit(1)
+      if self.UpdateProjectList(opt, m):
+        err_event.set()
+        err_update_projects = True
+        if opt.fail_fast:
+          print('\nerror: Local checkouts *not* updated.', file=sys.stderr)
+          sys.exit(1)
 
-    err_update_linkfiles = not self.UpdateCopyLinkfileList()
-    if err_update_linkfiles:
-      err_event.set()
-      if opt.fail_fast:
-        print('\nerror: Local update copyfile or linkfile failed.', file=sys.stderr)
-        sys.exit(1)
+      err_update_linkfiles = not self.UpdateCopyLinkfileList(m)
+      if err_update_linkfiles:
+        err_event.set()
+        if opt.fail_fast:
+          print('\nerror: Local update copyfile or linkfile failed.', file=sys.stderr)
+          sys.exit(1)
 
     err_results = []
     # NB: We don't exit here because this is the last step.
@@ -1065,10 +1188,14 @@ later is required to fix a server side protocol bug.
     if err_checkout:
       err_event.set()
 
-    # If there's a notice that's supposed to print at the end of the sync, print
-    # it now...
-    if self.manifest.notice:
-      print(self.manifest.notice)
+    printed_notices = set()
+    # If there's a notice that's supposed to print at the end of the sync,
+    # print it now...  But avoid printing duplicate messages, and preserve
+    # order.
+    for m in sorted(self.ManifestList(opt), key=lambda x: x.path_prefix):
+      if m.notice and m.notice not in printed_notices:
+        print(m.notice)
+        printed_notices.add(m.notice)
 
     # If we saw an error, exit with code 1 so that other scripts can check.
     if err_event.is_set():
@@ -1101,6 +1228,15 @@ later is required to fix a server side protocol bug.
 
 
 def _PostRepoUpgrade(manifest, quiet=False):
+  # Link the docs for the internal .repo/ layout for people
+  link = os.path.join(manifest.repodir, 'internal-fs-layout.md')
+  if not platform_utils.islink(link):
+    target = os.path.join('repo', 'docs', 'internal-fs-layout.md')
+    try:
+      platform_utils.symlink(target, link)
+    except Exception:
+      pass
+
   wrapper = Wrapper()
   if wrapper.NeedSetupGnuPG():
     wrapper.SetupGnuPG(quiet)

subcmds/upload.py

@@ -226,7 +226,8 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
 
       destination = opt.dest_branch or project.dest_branch or project.revisionExpr
       print('Upload project %s/ to remote branch %s%s:' %
-            (project.relpath, destination, ' (private)' if opt.private else ''))
+            (project.RelPath(local=opt.this_manifest_only), destination,
+             ' (private)' if opt.private else ''))
       print('  branch %s (%2d commit%s, %s):' % (
           name,
           len(commit_list),
@@ -262,7 +263,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     script.append('# Uncomment the branches to upload:')
     for project, avail in pending:
       script.append('#')
-      script.append('# project %s/:' % project.relpath)
+      script.append('# project %s/:' % project.RelPath(local=opt.this_manifest_only))
 
       b = {}
       for branch in avail:
@@ -285,7 +286,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
           script.append('#         %s' % commit)
         b[name] = branch
 
-      projects[project.relpath] = project
+      projects[project.RelPath(local=opt.this_manifest_only)] = project
       branches[project.name] = b
     script.append('')
 
@@ -313,7 +314,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
           _die('project for branch %s not in script', name)
         branch = branches[project.name].get(name)
         if not branch:
-          _die('branch %s not in %s', name, project.relpath)
+          _die('branch %s not in %s', name, project.RelPath(local=opt.this_manifest_only))
         todo.append(branch)
     if not todo:
       _die("nothing uncommented for upload")
@@ -420,12 +421,6 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
         labels = set(_ExpandCommaList(branch.project.config.GetString(key)))
         for label in opt.labels:
           labels.update(_ExpandCommaList(label))
-        # Basic sanity check on label syntax.
-        for label in labels:
-          if not re.match(r'^.+[+-][0-9]+$', label):
-            print('repo: error: invalid label syntax "%s": labels use forms '
-                  'like CodeReview+1 or Verified-1' % (label,), file=sys.stderr)
-            sys.exit(1)
 
         # Handle e-mail notifications.
         if opt.notify is False:
@@ -481,7 +476,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
           else:
             fmt = '\n       (%s)'
           print(('[FAILED] %-15s %-15s' + fmt) % (
-              branch.project.relpath + '/',
+              branch.project.RelPath(local=opt.this_manifest_only) + '/',
               branch.name,
               str(branch.error)),
               file=sys.stderr)
@@ -490,7 +485,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     for branch in todo:
       if branch.uploaded:
         print('[OK    ] %-15s %s' % (
-            branch.project.relpath + '/',
+            branch.project.RelPath(local=opt.this_manifest_only) + '/',
             branch.name),
             file=sys.stderr)
 
@@ -524,7 +519,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     return (project, avail)
 
   def Execute(self, opt, args):
-    projects = self.GetProjects(args)
+    projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     def _ProcessResults(_pool, _out, results):
       pending = []
@@ -534,7 +529,8 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
           print('repo: error: %s: Unable to upload branch "%s". '
                 'You might be able to fix the branch by running:\n'
                 '  git branch --set-upstream-to m/%s' %
-                (project.relpath, project.CurrentBranch, self.manifest.branch),
+                (project.RelPath(local=opt.this_manifest_only), project.CurrentBranch,
+                 project.manifest.branch),
                 file=sys.stderr)
         elif avail:
           pending.append(result)
@@ -554,15 +550,23 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
               (opt.branch,), file=sys.stderr)
       return 1
 
-    pending_proj_names = [project.name for (project, available) in pending]
-    pending_worktrees = [project.worktree for (project, available) in pending]
-    hook = RepoHook.FromSubcmd(
-        hook_type='pre-upload', manifest=self.manifest,
-        opt=opt, abort_if_user_denies=True)
-    if not hook.Run(
-        project_list=pending_proj_names,
-        worktree_list=pending_worktrees):
-      return 1
+    manifests = {project.manifest.topdir: project.manifest
+                 for (project, available) in pending}
+    ret = 0
+    for manifest in manifests.values():
+      pending_proj_names = [project.name for (project, available) in pending
+                            if project.manifest.topdir == manifest.topdir]
+      pending_worktrees = [project.worktree for (project, available) in pending
+                           if project.manifest.topdir == manifest.topdir]
+      hook = RepoHook.FromSubcmd(
+          hook_type='pre-upload', manifest=manifest,
+          opt=opt, abort_if_user_denies=True)
+      if not hook.Run(
+          project_list=pending_proj_names,
+          worktree_list=pending_worktrees):
+        ret = 1
+    if ret:
+      return ret
 
     reviewers = _SplitEmails(opt.reviewers) if opt.reviewers else []
     cc = _SplitEmails(opt.cc) if opt.cc else []

tests/test_git_superproject.py

@@ -24,7 +24,6 @@ from unittest import mock
 import git_superproject
 import git_trace2_event_log
 import manifest_xml
-import platform_utils
 from test_manifest_xml import sort_attributes
 
 
@@ -38,7 +37,8 @@ class SuperprojectTestCase(unittest.TestCase):
 
   def setUp(self):
     """Set up superproject every time."""
-    self.tempdir = tempfile.mkdtemp(prefix='repo_tests')
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    self.tempdir = self.tempdirobj.name
     self.repodir = os.path.join(self.tempdir, '.repo')
     self.manifest_file = os.path.join(
         self.repodir, manifest_xml.MANIFEST_FILE_NAME)
@@ -68,12 +68,14 @@ class SuperprojectTestCase(unittest.TestCase):
   <project path="art" name="platform/art" groups="notdefault,platform-""" + self.platform + """
   " /></manifest>
 """)
-    self._superproject = git_superproject.Superproject(manifest, self.repodir,
-                                                       self.git_event_log)
+    self._superproject = git_superproject.Superproject(
+        manifest, name='superproject',
+        remote=manifest.remotes.get('default-remote').ToRemoteSpec('superproject'),
+        revision='refs/heads/main')
 
   def tearDown(self):
     """Tear down superproject every time."""
-    platform_utils.rmtree(self.tempdir)
+    self.tempdirobj.cleanup()
 
   def getXmlManifest(self, data):
     """Helper to initialize a manifest for testing."""
@@ -125,12 +127,7 @@ class SuperprojectTestCase(unittest.TestCase):
 <manifest>
 </manifest>
 """)
-    superproject = git_superproject.Superproject(manifest, self.repodir, self.git_event_log)
-    # Test that exit condition is false when there is no superproject tag.
-    sync_result = superproject.Sync()
-    self.assertFalse(sync_result.success)
-    self.assertFalse(sync_result.fatal)
-    self.verifyErrorEvent()
+    self.assertIsNone(manifest.superproject)
 
   def test_superproject_get_superproject_invalid_url(self):
     """Test with an invalid url."""
@@ -141,8 +138,11 @@ class SuperprojectTestCase(unittest.TestCase):
   <superproject name="superproject"/>
 </manifest>
 """)
-    superproject = git_superproject.Superproject(manifest, self.repodir, self.git_event_log)
-    sync_result = superproject.Sync()
+    superproject = git_superproject.Superproject(
+        manifest, name='superproject',
+        remote=manifest.remotes.get('test-remote').ToRemoteSpec('superproject'),
+        revision='refs/heads/main')
+    sync_result = superproject.Sync(self.git_event_log)
     self.assertFalse(sync_result.success)
     self.assertTrue(sync_result.fatal)
 
@@ -155,17 +155,19 @@ class SuperprojectTestCase(unittest.TestCase):
   <superproject name="superproject"/>
 </manifest>
 """)
-    self._superproject = git_superproject.Superproject(manifest, self.repodir,
-                                                       self.git_event_log)
+    self._superproject = git_superproject.Superproject(
+        manifest, name='superproject',
+        remote=manifest.remotes.get('test-remote').ToRemoteSpec('superproject'),
+        revision='refs/heads/main')
     with mock.patch.object(self._superproject, '_branch', 'junk'):
-      sync_result = self._superproject.Sync()
+      sync_result = self._superproject.Sync(self.git_event_log)
       self.assertFalse(sync_result.success)
       self.assertTrue(sync_result.fatal)
 
   def test_superproject_get_superproject_mock_init(self):
     """Test with _Init failing."""
     with mock.patch.object(self._superproject, '_Init', return_value=False):
-      sync_result = self._superproject.Sync()
+      sync_result = self._superproject.Sync(self.git_event_log)
       self.assertFalse(sync_result.success)
       self.assertTrue(sync_result.fatal)
 
@@ -174,7 +176,7 @@ class SuperprojectTestCase(unittest.TestCase):
     with mock.patch.object(self._superproject, '_Init', return_value=True):
       os.mkdir(self._superproject._superproject_path)
       with mock.patch.object(self._superproject, '_Fetch', return_value=False):
-        sync_result = self._superproject.Sync()
+        sync_result = self._superproject.Sync(self.git_event_log)
         self.assertFalse(sync_result.success)
         self.assertTrue(sync_result.fatal)
 
@@ -230,7 +232,7 @@ class SuperprojectTestCase(unittest.TestCase):
                                return_value=data):
           # Create temporary directory so that it can write the file.
           os.mkdir(self._superproject._superproject_path)
-          update_result = self._superproject.UpdateProjectsRevisionId(projects)
+          update_result = self._superproject.UpdateProjectsRevisionId(projects, self.git_event_log)
           self.assertIsNotNone(update_result.manifest_path)
           self.assertFalse(update_result.fatal)
           with open(update_result.manifest_path, 'r') as fp:
@@ -256,22 +258,13 @@ class SuperprojectTestCase(unittest.TestCase):
 </manifest>
 """)
     self.maxDiff = None
-    self._superproject = git_superproject.Superproject(manifest, self.repodir,
-                                                       self.git_event_log)
-    self.assertEqual(len(self._superproject._manifest.projects), 1)
-    projects = self._superproject._manifest.projects
-    project = projects[0]
-    project.SetRevisionId('ABCDEF')
-    update_result = self._superproject.UpdateProjectsRevisionId(projects)
-    self.assertIsNone(update_result.manifest_path)
-    self.assertFalse(update_result.fatal)
-    self.verifyErrorEvent()
+    self.assertIsNone(manifest.superproject)
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
         '<remote fetch="http://localhost" name="default-remote"/>'
         '<default remote="default-remote" revision="refs/heads/main"/>'
-        '<project name="test-name" revision="ABCDEF" upstream="refs/heads/main"/>'
+        '<project name="test-name"/>'
         '</manifest>')
 
   def test_superproject_update_project_revision_id_from_local_manifest_group(self):
@@ -290,8 +283,10 @@ class SuperprojectTestCase(unittest.TestCase):
   " /></manifest>
 """)
     self.maxDiff = None
-    self._superproject = git_superproject.Superproject(manifest, self.repodir,
-                                                       self.git_event_log)
+    self._superproject = git_superproject.Superproject(
+        manifest, name='superproject',
+        remote=manifest.remotes.get('default-remote').ToRemoteSpec('superproject'),
+        revision='refs/heads/main')
     self.assertEqual(len(self._superproject._manifest.projects), 2)
     projects = self._superproject._manifest.projects
     data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00')
@@ -302,7 +297,7 @@ class SuperprojectTestCase(unittest.TestCase):
                                return_value=data):
           # Create temporary directory so that it can write the file.
           os.mkdir(self._superproject._superproject_path)
-          update_result = self._superproject.UpdateProjectsRevisionId(projects)
+          update_result = self._superproject.UpdateProjectsRevisionId(projects, self.git_event_log)
           self.assertIsNotNone(update_result.manifest_path)
           self.assertFalse(update_result.fatal)
           with open(update_result.manifest_path, 'r') as fp:
@@ -337,8 +332,10 @@ class SuperprojectTestCase(unittest.TestCase):
   " /></manifest>
 """)
     self.maxDiff = None
-    self._superproject = git_superproject.Superproject(manifest, self.repodir,
-                                                       self.git_event_log)
+    self._superproject = git_superproject.Superproject(
+        manifest, name='superproject',
+        remote=manifest.remotes.get('default-remote').ToRemoteSpec('superproject'),
+        revision='refs/heads/main')
     self.assertEqual(len(self._superproject._manifest.projects), 3)
     projects = self._superproject._manifest.projects
     data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00'
@@ -350,7 +347,7 @@ class SuperprojectTestCase(unittest.TestCase):
                                return_value=data):
           # Create temporary directory so that it can write the file.
           os.mkdir(self._superproject._superproject_path)
-          update_result = self._superproject.UpdateProjectsRevisionId(projects)
+          update_result = self._superproject.UpdateProjectsRevisionId(projects, self.git_event_log)
           self.assertIsNotNone(update_result.manifest_path)
           self.assertFalse(update_result.fatal)
           with open(update_result.manifest_path, 'r') as fp:

tests/test_git_trace2_event_log.py

@@ -16,11 +16,42 @@
 
 import json
 import os
+import socket
 import tempfile
+import threading
 import unittest
 from unittest import mock
 
 import git_trace2_event_log
+import platform_utils
+
+
+def serverLoggingThread(socket_path, server_ready, received_traces):
+  """Helper function to receive logs over a Unix domain socket.
+
+  Appends received messages on the provided socket and appends to received_traces.
+
+  Args:
+    socket_path: path to a Unix domain socket on which to listen for traces
+    server_ready: a threading.Condition used to signal to the caller that this thread is ready to
+        accept connections
+    received_traces: a list to which received traces will be appended (after decoding to a utf-8
+        string).
+  """
+  platform_utils.remove(socket_path, missing_ok=True)
+  data = b''
+  with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
+    sock.bind(socket_path)
+    sock.listen(0)
+    with server_ready:
+      server_ready.notify()
+    with sock.accept()[0] as conn:
+      while True:
+        recved = conn.recv(4096)
+        if not recved:
+          break
+        data += recved
+  received_traces.extend(data.decode('utf-8').splitlines())
 
 
 class EventLogTestCase(unittest.TestCase):
@@ -324,6 +355,37 @@ class EventLogTestCase(unittest.TestCase):
     with self.assertRaises(TypeError):
       self._event_log_module.Write(path=1234)
 
+  def test_write_socket(self):
+    """Test Write() with Unix domain socket for |path| and validate received traces."""
+    received_traces = []
+    with tempfile.TemporaryDirectory(prefix='test_server_sockets') as tempdir:
+      socket_path = os.path.join(tempdir, "server.sock")
+      server_ready = threading.Condition()
+      # Start "server" listening on Unix domain socket at socket_path.
+      try:
+        server_thread = threading.Thread(
+            target=serverLoggingThread,
+            args=(socket_path, server_ready, received_traces))
+        server_thread.start()
+
+        with server_ready:
+          server_ready.wait()
+
+        self._event_log_module.StartEvent()
+        path = self._event_log_module.Write(path=f'af_unix:{socket_path}')
+      finally:
+        server_thread.join(timeout=5)
+
+    self.assertEqual(path, f'af_unix:stream:{socket_path}')
+    self.assertEqual(len(received_traces), 2)
+    version_event = json.loads(received_traces[0])
+    start_event = json.loads(received_traces[1])
+    self.verifyCommonKeys(version_event, expected_event_name='version')
+    self.verifyCommonKeys(start_event, expected_event_name='start')
+    # Check for 'start' event specific fields.
+    self.assertIn('argv', start_event)
+    self.assertIsInstance(start_event['argv'], list)
+
 
 if __name__ == '__main__':
   unittest.main()

tests/test_manifest_xml.py

@@ -17,7 +17,6 @@
 import os
 import platform
 import re
-import shutil
 import tempfile
 import unittest
 import xml.dom.minidom
@@ -92,7 +91,8 @@ class ManifestParseTestCase(unittest.TestCase):
   """TestCase for parsing manifests."""
 
   def setUp(self):
-    self.tempdir = tempfile.mkdtemp(prefix='repo_tests')
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    self.tempdir = self.tempdirobj.name
     self.repodir = os.path.join(self.tempdir, '.repo')
     self.manifest_dir = os.path.join(self.repodir, 'manifests')
     self.manifest_file = os.path.join(
@@ -111,7 +111,7 @@ class ManifestParseTestCase(unittest.TestCase):
 """)
 
   def tearDown(self):
-    shutil.rmtree(self.tempdir, ignore_errors=True)
+    self.tempdirobj.cleanup()
 
   def getXmlManifest(self, data):
     """Helper to initialize a manifest for testing."""
@@ -289,8 +289,8 @@ class XmlManifestTests(ManifestParseTestCase):
   <x-custom-tag>X tags are always ignored</x-custom-tag>
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
+    self.assertEqual(manifest.superproject.name, 'superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'test-remote')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -569,10 +569,10 @@ class SuperProjectElementTests(ManifestParseTestCase):
   <superproject name="superproject"/>
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
-    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
-    self.assertEqual(manifest.superproject['revision'], 'refs/heads/main')
+    self.assertEqual(manifest.superproject.name, 'superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'test-remote')
+    self.assertEqual(manifest.superproject.remote.url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject.revision, 'refs/heads/main')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -591,10 +591,10 @@ class SuperProjectElementTests(ManifestParseTestCase):
   <superproject name="superproject" revision="refs/heads/stable" />
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
-    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
-    self.assertEqual(manifest.superproject['revision'], 'refs/heads/stable')
+    self.assertEqual(manifest.superproject.name, 'superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'test-remote')
+    self.assertEqual(manifest.superproject.remote.url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject.revision, 'refs/heads/stable')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -613,10 +613,10 @@ class SuperProjectElementTests(ManifestParseTestCase):
   <superproject name="superproject" revision="refs/heads/stable" />
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
-    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
-    self.assertEqual(manifest.superproject['revision'], 'refs/heads/stable')
+    self.assertEqual(manifest.superproject.name, 'superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'test-remote')
+    self.assertEqual(manifest.superproject.remote.url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject.revision, 'refs/heads/stable')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -635,10 +635,10 @@ class SuperProjectElementTests(ManifestParseTestCase):
   <superproject name="superproject" revision="refs/heads/stable" />
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
-    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
-    self.assertEqual(manifest.superproject['revision'], 'refs/heads/stable')
+    self.assertEqual(manifest.superproject.name, 'superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'test-remote')
+    self.assertEqual(manifest.superproject.remote.url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject.revision, 'refs/heads/stable')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -657,10 +657,10 @@ class SuperProjectElementTests(ManifestParseTestCase):
   <superproject name="platform/superproject" remote="superproject-remote"/>
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'platform/superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'superproject-remote')
-    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/platform/superproject')
-    self.assertEqual(manifest.superproject['revision'], 'refs/heads/main')
+    self.assertEqual(manifest.superproject.name, 'platform/superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'superproject-remote')
+    self.assertEqual(manifest.superproject.remote.url, 'http://localhost/platform/superproject')
+    self.assertEqual(manifest.superproject.revision, 'refs/heads/main')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -679,9 +679,9 @@ class SuperProjectElementTests(ManifestParseTestCase):
   <superproject name="superproject" remote="default-remote"/>
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'default-remote')
-    self.assertEqual(manifest.superproject['revision'], 'refs/heads/main')
+    self.assertEqual(manifest.superproject.name, 'superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'default-remote')
+    self.assertEqual(manifest.superproject.revision, 'refs/heads/main')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'

tests/test_project.py

@@ -16,7 +16,7 @@
 
 import contextlib
 import os
-import shutil
+from pathlib import Path
 import subprocess
 import tempfile
 import unittest
@@ -31,11 +31,7 @@ import project
 @contextlib.contextmanager
 def TempGitTree():
   """Create a new empty git checkout for testing."""
-  # TODO(vapier): Convert this to tempfile.TemporaryDirectory once we drop
-  # Python 2 support entirely.
-  try:
-    tempdir = tempfile.mkdtemp(prefix='repo-tests')
-
+  with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
     # Tests need to assume, that main is default branch at init,
     # which is not supported in config until 2.28.
     cmd = ['git', 'init']
@@ -49,8 +45,6 @@ def TempGitTree():
       cmd += ['--template', templatedir]
     subprocess.check_call(cmd, cwd=tempdir)
     yield tempdir
-  finally:
-    platform_utils.rmtree(tempdir)
 
 
 class FakeProject(object):
@@ -123,14 +117,15 @@ class CopyLinkTestCase(unittest.TestCase):
   """
 
   def setUp(self):
-    self.tempdir = tempfile.mkdtemp(prefix='repo_tests')
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    self.tempdir = self.tempdirobj.name
     self.topdir = os.path.join(self.tempdir, 'checkout')
     self.worktree = os.path.join(self.topdir, 'git-project')
     os.makedirs(self.topdir)
     os.makedirs(self.worktree)
 
   def tearDown(self):
-    shutil.rmtree(self.tempdir, ignore_errors=True)
+    self.tempdirobj.cleanup()
 
   @staticmethod
   def touch(path):
@@ -335,3 +330,76 @@ class LinkFile(CopyLinkTestCase):
     platform_utils.symlink(self.tempdir, dest)
     lf._Link()
     self.assertEqual(os.path.join('git-project', 'foo.txt'), os.readlink(dest))
+
+
+class MigrateWorkTreeTests(unittest.TestCase):
+  """Check _MigrateOldWorkTreeGitDir handling."""
+
+  _SYMLINKS = {
+      'config', 'description', 'hooks', 'info', 'logs', 'objects',
+      'packed-refs', 'refs', 'rr-cache', 'shallow', 'svn',
+  }
+  _FILES = {
+      'COMMIT_EDITMSG', 'FETCH_HEAD', 'HEAD', 'index', 'ORIG_HEAD',
+      'unknown-file-should-be-migrated',
+  }
+  _CLEAN_FILES = {
+      'a-vim-temp-file~', '#an-emacs-temp-file#',
+  }
+
+  @classmethod
+  @contextlib.contextmanager
+  def _simple_layout(cls):
+    """Create a simple repo client checkout to test against."""
+    with tempfile.TemporaryDirectory() as tempdir:
+      tempdir = Path(tempdir)
+
+      gitdir = tempdir / '.repo/projects/src/test.git'
+      gitdir.mkdir(parents=True)
+      cmd = ['git', 'init', '--bare', str(gitdir)]
+      subprocess.check_call(cmd)
+
+      dotgit = tempdir / 'src/test/.git'
+      dotgit.mkdir(parents=True)
+      for name in cls._SYMLINKS:
+        (dotgit / name).symlink_to(f'../../../.repo/projects/src/test.git/{name}')
+      for name in cls._FILES | cls._CLEAN_FILES:
+        (dotgit / name).write_text(name)
+
+      yield tempdir
+
+  def test_standard(self):
+    """Migrate a standard checkout that we expect."""
+    with self._simple_layout() as tempdir:
+      dotgit = tempdir / 'src/test/.git'
+      project.Project._MigrateOldWorkTreeGitDir(str(dotgit))
+
+      # Make sure the dir was transformed into a symlink.
+      self.assertTrue(dotgit.is_symlink())
+      self.assertEqual(os.readlink(dotgit), '../../.repo/projects/src/test.git')
+
+      # Make sure files were moved over.
+      gitdir = tempdir / '.repo/projects/src/test.git'
+      for name in self._FILES:
+        self.assertEqual(name, (gitdir / name).read_text())
+      # Make sure files were removed.
+      for name in self._CLEAN_FILES:
+        self.assertFalse((gitdir / name).exists())
+
+  def test_unknown(self):
+    """A checkout with unknown files should abort."""
+    with self._simple_layout() as tempdir:
+      dotgit = tempdir / 'src/test/.git'
+      (tempdir / '.repo/projects/src/test.git/random-file').write_text('one')
+      (dotgit / 'random-file').write_text('two')
+      with self.assertRaises(error.GitError):
+        project.Project._MigrateOldWorkTreeGitDir(str(dotgit))
+
+      # Make sure no content was actually changed.
+      self.assertTrue(dotgit.is_dir())
+      for name in self._FILES:
+        self.assertTrue((dotgit / name).is_file())
+      for name in self._CLEAN_FILES:
+        self.assertTrue((dotgit / name).is_file())
+      for name in self._SYMLINKS:
+        self.assertTrue((dotgit / name).is_symlink())

tests/test_subcmds_sync.py

@@ -0,0 +1,45 @@
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for the subcmds/sync.py module."""
+
+from unittest import mock
+
+import pytest
+
+from subcmds import sync
+
+
+@pytest.mark.parametrize(
+  'use_superproject, cli_args, result',
+  [
+    (True, ['--current-branch'], True),
+    (True, ['--no-current-branch'], True),
+    (True, [], True),
+    (False, ['--current-branch'], True),
+    (False, ['--no-current-branch'], False),
+    (False, [], None),
+  ]
+)
+def test_get_current_branch_only(use_superproject, cli_args, result):
+  """Test Sync._GetCurrentBranchOnly logic.
+
+  Sync._GetCurrentBranchOnly should return True if a superproject is requested,
+  and otherwise the value of the current_branch_only option.
+  """
+  cmd = sync.Sync()
+  opts, _ = cmd.OptionParser.parse_args(cli_args)
+
+  with mock.patch('git_superproject.UseSuperproject', return_value=use_superproject):
+    assert cmd._GetCurrentBranchOnly(opts, cmd.manifest) == result

tests/test_wrapper.py

@@ -14,11 +14,9 @@
 
 """Unittests for the wrapper.py module."""
 
-import contextlib
 from io import StringIO
 import os
 import re
-import shutil
 import sys
 import tempfile
 import unittest
@@ -26,22 +24,9 @@ from unittest import mock
 
 import git_command
 import main
-import platform_utils
 import wrapper
 
 
-@contextlib.contextmanager
-def TemporaryDirectory():
-  """Create a new empty git checkout for testing."""
-  # TODO(vapier): Convert this to tempfile.TemporaryDirectory once we drop
-  # Python 2 support entirely.
-  try:
-    tempdir = tempfile.mkdtemp(prefix='repo-tests')
-    yield tempdir
-  finally:
-    platform_utils.rmtree(tempdir)
-
-
 def fixture(*paths):
   """Return a path relative to tests/fixtures.
   """
@@ -336,19 +321,19 @@ class NeedSetupGnuPG(RepoWrapperTestCase):
 
   def test_missing_dir(self):
     """The ~/.repoconfig tree doesn't exist yet."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = os.path.join(tempdir, 'foo')
       self.assertTrue(self.wrapper.NeedSetupGnuPG())
 
   def test_missing_keyring(self):
     """The keyring-version file doesn't exist yet."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       self.assertTrue(self.wrapper.NeedSetupGnuPG())
 
   def test_empty_keyring(self):
     """The keyring-version file exists, but is empty."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       with open(os.path.join(tempdir, 'keyring-version'), 'w'):
         pass
@@ -356,7 +341,7 @@ class NeedSetupGnuPG(RepoWrapperTestCase):
 
   def test_old_keyring(self):
     """The keyring-version file exists, but it's old."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       with open(os.path.join(tempdir, 'keyring-version'), 'w') as fp:
         fp.write('1.0\n')
@@ -364,7 +349,7 @@ class NeedSetupGnuPG(RepoWrapperTestCase):
 
   def test_new_keyring(self):
     """The keyring-version file exists, and is up-to-date."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       with open(os.path.join(tempdir, 'keyring-version'), 'w') as fp:
         fp.write('1000.0\n')
@@ -376,7 +361,7 @@ class SetupGnuPG(RepoWrapperTestCase):
 
   def test_full(self):
     """Make sure it works completely."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       self.wrapper.gpg_dir = os.path.join(self.wrapper.home_dot_repo, 'gnupg')
       self.assertTrue(self.wrapper.SetupGnuPG(True))
@@ -426,7 +411,8 @@ class GitCheckoutTestCase(RepoWrapperTestCase):
   @classmethod
   def setUpClass(cls):
     # Create a repo to operate on, but do it once per-class.
-    cls.GIT_DIR = tempfile.mkdtemp(prefix='repo-rev-tests')
+    cls.tempdirobj = tempfile.TemporaryDirectory(prefix='repo-rev-tests')
+    cls.GIT_DIR = cls.tempdirobj.name
     run_git = wrapper.Wrapper().run_git
 
     remote = os.path.join(cls.GIT_DIR, 'remote')
@@ -455,10 +441,10 @@ class GitCheckoutTestCase(RepoWrapperTestCase):
 
   @classmethod
   def tearDownClass(cls):
-    if not cls.GIT_DIR:
+    if not cls.tempdirobj:
       return
 
-    shutil.rmtree(cls.GIT_DIR)
+    cls.tempdirobj.cleanup()
 
 
 class ResolveRepoRev(GitCheckoutTestCase):