sync: add multi-manifest support

With this change, partial syncs (sync with a project list) are again
supported.

If the updated manifest includes new sub manifests, download them
inheriting options from the parent manifestProject.

Change-Id: Id952f85df2e26d34e38b251973be26434443ff56
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/334819
Reviewed-by: Mike Frysinger <vapier@google.com>
Tested-by: LaMont Jones <lamontjones@google.com>

command.py

@@ -144,11 +144,10 @@ class Command(object):
           help=f'number of jobs to run in parallel (default: {default})')
 
     m = p.add_option_group('Multi-manifest options')
-    m.add_option('--outer-manifest', action='store_true',
+    m.add_option('--outer-manifest', action='store_true', default=None,
                  help='operate starting at the outermost manifest')
     m.add_option('--no-outer-manifest', dest='outer_manifest',
-                 action='store_false', default=None,
-                 help='do not operate on outer manifests')
+                 action='store_false', help='do not operate on outer manifests')
     m.add_option('--this-manifest-only', action='store_true', default=None,
                  help='only operate on this (sub)manifest')
     m.add_option('--no-this-manifest-only', '--all-manifests',
@@ -186,6 +185,10 @@ class Command(object):
     """Validate common options."""
     opt.quiet = opt.output_mode is False
     opt.verbose = opt.output_mode is True
+    if opt.outer_manifest is None:
+      # By default, treat multi-manifest instances as a single manifest from
+      # the user's perspective.
+      opt.outer_manifest = True
 
   def ValidateOptions(self, opt, args):
     """Validate the user options & arguments before executing.
@@ -385,7 +388,7 @@ class Command(object):
       opt: The command options.
     """
     top = self.outer_manifest
-    if opt.outer_manifest is False or opt.this_manifest_only:
+    if not opt.outer_manifest or opt.this_manifest_only:
       top = self.manifest
     yield top
     if not opt.this_manifest_only:

docs/manifest-format.md

@@ -66,6 +66,7 @@ following DTD:
   <!ATTLIST submanifest revision       CDATA #IMPLIED>
   <!ATTLIST submanifest path           CDATA #IMPLIED>
   <!ATTLIST submanifest groups         CDATA #IMPLIED>
+  <!ATTLIST submanifest default-groups CDATA #IMPLIED>
 
   <!ELEMENT project (annotation*,
                      project*,
@@ -302,6 +303,9 @@ in the included submanifest belong. This appends and recurses, meaning
 all projects in submanifests carry all parent submanifest groups.
 Same syntax as the corresponding element of `project`.
 
+Attribute `default-groups`: The list of manifest groups to sync if no
+`--groups=` parameter was specified at init.  When that list is empty, use this
+list instead of "default" as the list of groups to sync.
 
 ### Element project
 

git_superproject.py

@@ -238,8 +238,8 @@ class Superproject(object):
                        f'{self._manifest.manifestFile}')
       return SyncResult(False, False)
 
-    print('NOTICE: --use-superproject is in beta; report any issues to the '
-          'address described in `repo version`', file=sys.stderr)
+    _PrintBetaNotice()
+
     should_exit = True
     if not self._remote_url:
       self._LogWarning(f'superproject URL is not defined in manifest: '
@@ -364,6 +364,13 @@ class Superproject(object):
     return UpdateProjectsResult(manifest_path, False)
 
 
+@functools.lru_cache(maxsize=10)
+def _PrintBetaNotice():
+  """Print the notice of beta status."""
+  print('NOTICE: --use-superproject is in beta; report any issues to the '
+        'address described in `repo version`', file=sys.stderr)
+
+
 @functools.lru_cache(maxsize=None)
 def _UseSuperprojectFromConfiguration():
   """Returns the user choice of whether to use superproject."""
@@ -408,16 +415,26 @@ def _UseSuperprojectFromConfiguration():
   return False
 
 
-def PrintMessages(opt, manifest):
-  """Returns a boolean if error/warning messages are to be printed."""
-  return opt.use_superproject is not None or bool(manifest.superproject)
+def PrintMessages(use_superproject, manifest):
+  """Returns a boolean if error/warning messages are to be printed.
+
+  Args:
+    use_superproject: option value from optparse.
+    manifest: manifest to use.
+  """
+  return use_superproject is not None or bool(manifest.superproject)
 
 
-def UseSuperproject(opt, manifest):
-  """Returns a boolean if use-superproject option is enabled."""
+def UseSuperproject(use_superproject, manifest):
+  """Returns a boolean if use-superproject option is enabled.
 
-  if opt.use_superproject is not None:
-    return opt.use_superproject
+  Args:
+    use_superproject: option value from optparse.
+    manifest: manifest to use.
+  """
+
+  if use_superproject is not None:
+    return use_superproject
   else:
     client_value = manifest.manifestProject.use_superproject
     if client_value is not None:

main.py

@@ -294,8 +294,7 @@ class _Repo(object):
       cmd.ValidateOptions(copts, cargs)
 
       this_manifest_only = copts.this_manifest_only
-      # If not specified, default to using the outer manifest.
-      outer_manifest = copts.outer_manifest is not False
+      outer_manifest = copts.outer_manifest
       if cmd.MULTI_MANIFEST_SUPPORT or this_manifest_only:
         result = cmd.Execute(copts, cargs)
       elif outer_manifest and repo_client.manifest.is_submanifest:
@@ -310,7 +309,7 @@ class _Repo(object):
         # (sub)manifest, and then any child submanifests.
         result = cmd.Execute(copts, cargs)
         for submanifest in repo_client.manifest.submanifests.values():
-          spec = submanifest.ToSubmanifestSpec(root=repo_client.outer_client)
+          spec = submanifest.ToSubmanifestSpec()
           gopts.submanifest_path = submanifest.repo_client.path_prefix
           child_argv = argv[:]
           child_argv.append('--no-outer-manifest')

manifest_xml.py

@@ -214,9 +214,11 @@ class _XmlSubmanifest:
     revision: a string, the commitish.
     manifestName: a string, the submanifest file name.
     groups: a list of strings, the groups to add to all projects in the submanifest.
+    default_groups: a list of strings, the default groups to sync.
     path: a string, the relative path for the submanifest checkout.
+    parent: an XmlManifest, the parent manifest.
     annotations: (derived) a list of annotations.
-    present: (derived) a boolean, whether the submanifest's manifest file is present.
+    present: (derived) a boolean, whether the sub manifest file is present.
   """
   def __init__(self,
                name,
@@ -225,6 +227,7 @@ class _XmlSubmanifest:
                revision=None,
                manifestName=None,
                groups=None,
+               default_groups=None,
                path=None,
                parent=None):
     self.name = name
@@ -233,7 +236,9 @@ class _XmlSubmanifest:
     self.revision = revision
     self.manifestName = manifestName
     self.groups = groups
+    self.default_groups = default_groups
     self.path = path
+    self.parent = parent
     self.annotations = []
     outer_client = parent._outer_client or parent
     if self.remote and not self.project:
@@ -248,7 +253,8 @@ class _XmlSubmanifest:
         os.path.join(parent.path_prefix, self.relpath), MANIFEST_FILE_NAME)
     rc = self.repo_client = RepoClient(
         parent.repodir, linkFile, parent_groups=','.join(groups) or '',
-        submanifest_path=self.relpath, outer_client=outer_client)
+        submanifest_path=self.relpath, outer_client=outer_client,
+        default_groups=default_groups)
 
     self.present = os.path.exists(manifestFile)
 
@@ -262,16 +268,17 @@ class _XmlSubmanifest:
         self.revision == other.revision and
         self.manifestName == other.manifestName and
         self.groups == other.groups and
+        self.default_groups == other.default_groups and
         self.path == other.path and
         sorted(self.annotations) == sorted(other.annotations))
 
   def __ne__(self, other):
     return not self.__eq__(other)
 
-  def ToSubmanifestSpec(self, root):
+  def ToSubmanifestSpec(self):
     """Return a SubmanifestSpec object, populating attributes"""
-    mp = root.manifestProject
-    remote = root.remotes[self.remote or root.default.remote.name]
+    mp = self.parent.manifestProject
+    remote = self.parent.remotes[self.remote or self.parent.default.remote.name]
     # If a project was given, generate the url from the remote and project.
     # If not, use this manifestProject's url.
     if self.project:
@@ -282,6 +289,7 @@ class _XmlSubmanifest:
     revision = self.revision or self.name
     path = self.path or revision.split('/')[-1]
     groups = self.groups or []
+    default_groups = self.default_groups or []
 
     return SubmanifestSpec(self.name, manifestUrl, manifestName, revision, path,
                            groups)
@@ -298,6 +306,10 @@ class _XmlSubmanifest:
       return ','.join(self.groups)
     return ''
 
+  def GetDefaultGroupsStr(self):
+    """Returns the `default-groups` given for this submanifest."""
+    return ','.join(self.default_groups or [])
+
   def AddAnnotation(self, name, value, keep):
     """Add annotations to the submanifest."""
     self.annotations.append(Annotation(name, value, keep))
@@ -325,7 +337,8 @@ class XmlManifest(object):
   """manages the repo configuration file"""
 
   def __init__(self, repodir, manifest_file, local_manifests=None,
-               outer_client=None, parent_groups='', submanifest_path=''):
+               outer_client=None, parent_groups='', submanifest_path='',
+               default_groups=None):
     """Initialize.
 
     Args:
@@ -338,6 +351,7 @@ class XmlManifest(object):
       outer_client: RepoClient of the outertree.
       parent_groups: a string, the groups to apply to this projects.
       submanifest_path: The submanifest root relative to the repo root.
+      default_groups: a string, the default manifest groups to use.
     """
     # TODO(vapier): Move this out of this class.
     self.globalConfig = GitConfig.ForUser()
@@ -348,9 +362,15 @@ class XmlManifest(object):
     if manifest_file != os.path.abspath(manifest_file):
       raise ManifestParseError('manifest_file must be abspath')
     self.manifestFile = manifest_file
+    if not outer_client or outer_client == self:
+      # manifestFileOverrides only exists in the outer_client's manifest, since
+      # that is the only instance left when Unload() is called on the outer
+      # manifest.
+      self.manifestFileOverrides = {}
     self.local_manifests = local_manifests
     self._load_local_manifests = True
     self.parent_groups = parent_groups
+    self.default_groups = default_groups
 
     if outer_client and self.isGitcClient:
       raise ManifestParseError('Multi-manifest is incompatible with `gitc-init`')
@@ -396,14 +416,10 @@ class XmlManifest(object):
       if not os.path.isfile(path):
         raise ManifestParseError('manifest %s not found' % name)
 
-    old = self.manifestFile
-    try:
     self._load_local_manifests = load_local_manifests
-      self.manifestFile = path
+    self._outer_client.manifestFileOverrides[self.path_prefix] = path
     self.Unload()
     self._Load()
-    finally:
-      self.manifestFile = old
 
   def Link(self, name):
     """Update the repo metadata to use a different manifest.
@@ -469,6 +485,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
       e.setAttribute('path', r.path)
     if r.groups:
       e.setAttribute('groups', r.GetGroupsStr())
+    if r.default_groups:
+      e.setAttribute('default-groups', r.GetDefaultGroupsStr())
 
     for a in r.annotations:
       if a.keep == 'true':
@@ -750,8 +768,11 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
   @property
   def is_multimanifest(self):
-    """Whether this is a multimanifest checkout"""
-    return bool(self.outer_client.submanifests)
+    """Whether this is a multimanifest checkout.
+
+    This is safe to use as long as the outermost manifest XML has been parsed.
+    """
+    return bool(self._outer_client._submanifests)
 
   @property
   def is_submanifest(self):
@@ -880,6 +901,10 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     exclude = self.manifest.manifestProject.partial_clone_exclude or ''
     return set(x.strip() for x in exclude.split(','))
 
+  def SetManifestOverride(self, path):
+    """Override manifestFile.  The caller must call Unload()"""
+    self._outer_client.manifest.manifestFileOverrides[self.path_prefix] = path
+
   @property
   def UseLocalManifests(self):
     return self._load_local_manifests
@@ -960,16 +985,21 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                          worktree=os.path.join(subdir, 'manifests'))
     return mp
 
-  def GetDefaultGroupsStr(self):
-    """Returns the default group string for the platform."""
-    return 'default,platform-' + platform.system().lower()
+  def GetDefaultGroupsStr(self, with_platform=True):
+    """Returns the default group string to use.
+
+    Args:
+      with_platform: a boolean, whether to include the group for the
+                     underlying platform.
+    """
+    groups = ','.join(self.default_groups or ['default'])
+    if with_platform:
+      groups += f',platform-{platform.system().lower()}'
+    return groups
 
   def GetGroupsStr(self):
     """Returns the manifest group string that should be synced."""
-    groups = self.manifestProject.manifest_groups
-    if not groups:
-      groups = self.GetDefaultGroupsStr()
-    return groups
+    return self.manifestProject.manifest_groups or self.GetDefaultGroupsStr()
 
   def Unload(self):
     """Unload the manifest.
@@ -1005,6 +1035,12 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         # This will load all clients.
         self._outer_client._Load(initial_client=self)
 
+      savedManifestFile = self.manifestFile
+      override = self._outer_client.manifestFileOverrides.get(self.path_prefix)
+      if override:
+        self.manifestFile = override
+
+      try:
         m = self.manifestProject
         b = m.GetBranch(m.CurrentBranch).merge
         if b is not None and b.startswith(R_HEADS):
@@ -1050,12 +1086,15 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
           self._AddMetaProjectMirror(self.manifestProject)
 
         self._loaded = True
+      finally:
+        if override:
+          self.manifestFile = savedManifestFile
 
       # Now that we have loaded this manifest, load any submanifest  manifests
       # as well.  We need to do this after self._loaded is set to avoid looping.
       for name in self._submanifests:
         tree = self._submanifests[name]
-        spec = tree.ToSubmanifestSpec(self)
+        spec = tree.ToSubmanifestSpec()
         present = os.path.exists(os.path.join(self.subdir, MANIFEST_FILE_NAME))
         if present and tree.present and not tree.repo_client:
           if initial_client and initial_client.topdir == self.topdir:
@@ -1475,6 +1514,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     if node.hasAttribute('groups'):
       groups = node.getAttribute('groups')
     groups = self._ParseList(groups)
+    default_groups = self._ParseList(node.getAttribute('default-groups'))
     path = node.getAttribute('path')
     if path == '':
       path = None
@@ -1495,7 +1535,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
             '<submanifest> invalid "path": %s: %s' % (path, msg))
 
     submanifest = _XmlSubmanifest(name, remote, project, revision, manifestName,
-                                  groups, path, self)
+                                  groups, default_groups, path, self)
 
     for n in node.childNodes:
       if n.nodeName == 'annotation':
@@ -1642,7 +1682,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
       # We allow people to mix git worktrees & non-git worktrees for now.
       # This allows for in situ migration of repo clients.
       if os.path.exists(gitdir) or not self.UseGitWorktrees:
-        objdir = os.path.join(self.subdir, 'project-objects', namepath)
+        objdir = os.path.join(self.repodir, 'project-objects', namepath)
       else:
         use_git_worktrees = True
         gitdir = os.path.join(self.repodir, 'worktrees', namepath)

progress.py

@@ -24,6 +24,11 @@ _NOT_TTY = not os.isatty(2)
 # column 0.
 CSI_ERASE_LINE = '\x1b[2K'
 
+# This will erase all content in the current line after the cursor.  This is
+# useful for partial updates & progress messages as the terminal can display
+# it better.
+CSI_ERASE_LINE_AFTER = '\x1b[K'
+
 
 def duration_str(total):
   """A less noisy timedelta.__str__.
@@ -85,10 +90,10 @@ class Progress(object):
         return
 
     if self._total <= 0:
-      sys.stderr.write('%s\r%s: %d,' % (
-          CSI_ERASE_LINE,
+      sys.stderr.write('\r%s: %d,%s' % (
           self._title,
-          self._done))
+          self._done,
+          CSI_ERASE_LINE_AFTER))
       sys.stderr.flush()
     else:
       p = (100 * self._done) / self._total
@@ -96,14 +101,14 @@ class Progress(object):
         jobs = '[%d job%s] ' % (self._active, 's' if self._active > 1 else '')
       else:
         jobs = ''
-      sys.stderr.write('%s\r%s: %2d%% %s(%d%s/%d%s)%s%s%s' % (
-          CSI_ERASE_LINE,
+      sys.stderr.write('\r%s: %2d%% %s(%d%s/%d%s)%s%s%s%s' % (
           self._title,
           p,
           jobs,
           self._done, self._units,
           self._total, self._units,
           ' ' if msg else '', msg,
+          CSI_ERASE_LINE_AFTER,
           '\n' if self._print_newline else ''))
       sys.stderr.flush()
 
@@ -113,19 +118,19 @@ class Progress(object):
 
     duration = duration_str(time() - self._start)
     if self._total <= 0:
-      sys.stderr.write('%s\r%s: %d, done in %s\n' % (
-          CSI_ERASE_LINE,
+      sys.stderr.write('\r%s: %d, done in %s%s\n' % (
           self._title,
           self._done,
-          duration))
+          duration,
+          CSI_ERASE_LINE_AFTER))
       sys.stderr.flush()
     else:
       p = (100 * self._done) / self._total
-      sys.stderr.write('%s\r%s: %3d%% (%d%s/%d%s), done in %s\n' % (
-          CSI_ERASE_LINE,
+      sys.stderr.write('\r%s: %3d%% (%d%s/%d%s), done in %s%s\n' % (
           self._title,
           p,
           self._done, self._units,
           self._total, self._units,
-          duration))
+          duration,
+          CSI_ERASE_LINE_AFTER))
       sys.stderr.flush()

project.py

@@ -29,6 +29,7 @@ import time
 import urllib.parse
 
 from color import Coloring
+import fetch
 from git_command import GitCommand, git_require
 from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \
     ID_RE
@@ -48,6 +49,9 @@ MAXIMUM_RETRY_SLEEP_SEC = 3600.0
 # +-10% random jitter is added to each Fetches retry sleep duration.
 RETRY_JITTER_PERCENT = 0.1
 
+# Whether to use alternates.
+# TODO(vapier): Remove knob once behavior is verified.
+_ALTERNATES = os.environ.get('REPO_USE_ALTERNATES') == '1'
 
 def _lwrite(path, content):
   lock = '%s.lock' % path
@@ -459,7 +463,7 @@ class RemoteSpec(object):
 
 class Project(object):
   # These objects can be shared between several working trees.
-  shareable_dirs = ['hooks', 'objects', 'rr-cache']
+  shareable_dirs = ['hooks', 'rr-cache']
 
   def __init__(self,
                manifest,
@@ -714,7 +718,8 @@ class Project(object):
        The special manifest group "default" will match any project that
        does not have the special project group "notdefault"
     """
-    expanded_manifest_groups = manifest_groups or ['default']
+    default_groups = self.manifest.default_groups or ['default']
+    expanded_manifest_groups = manifest_groups or default_groups
     expanded_project_groups = ['all'] + (self.groups or [])
     if 'notdefault' not in expanded_project_groups:
       expanded_project_groups += ['default']
@@ -994,6 +999,13 @@ class Project(object):
     if not branch.remote.review:
       raise GitError('remote %s has no review url' % branch.remote.name)
 
+    # Basic validity check on label syntax.
+    for label in labels:
+      if not re.match(r'^.+[+-][0-9]+$', label):
+        raise UploadError(
+            f'invalid label syntax "{label}": labels use forms like '
+            'CodeReview+1 or Verified-1')
+
     if dest_branch is None:
       dest_branch = self.dest_branch
     if dest_branch is None:
@@ -1029,6 +1041,7 @@ class Project(object):
     if auto_topic:
       opts += ['topic=' + branch.name]
     opts += ['t=%s' % p for p in hashtags]
+    # NB: No need to encode labels as they've been validated above.
     opts += ['l=%s' % p for p in labels]
 
     opts += ['r=%s' % p for p in people[0]]
@@ -1133,6 +1146,17 @@ class Project(object):
       self._UpdateHooks(quiet=quiet)
     self._InitRemote()
 
+    if _ALTERNATES or self.manifest.is_multimanifest:
+      # If gitdir/objects is a symlink, migrate it from the old layout.
+      gitdir_objects = os.path.join(self.gitdir, 'objects')
+      if platform_utils.islink(gitdir_objects):
+        platform_utils.remove(gitdir_objects, missing_ok=True)
+      gitdir_alt = os.path.join(self.gitdir, 'objects/info/alternates')
+      if not os.path.exists(gitdir_alt):
+        os.makedirs(os.path.dirname(gitdir_alt), exist_ok=True)
+        _lwrite(gitdir_alt, os.path.join(
+            os.path.relpath(self.objdir, gitdir_objects), 'objects') + '\n')
+
     if is_new:
       alt = os.path.join(self.objdir, 'objects/info/alternates')
       try:
@@ -3371,73 +3395,139 @@ class ManifestProject(MetaProject):
   @property
   def reference(self):
     """The --reference for this manifest."""
-    self.config.GetString('repo.reference')
+    return self.config.GetString('repo.reference')
 
   @property
   def dissociate(self):
     """Whether to dissociate."""
-    self.config.GetBoolean('repo.dissociate')
+    return self.config.GetBoolean('repo.dissociate')
 
   @property
   def archive(self):
     """Whether we use archive."""
-    self.config.GetBoolean('repo.archive')
+    return self.config.GetBoolean('repo.archive')
 
   @property
   def mirror(self):
     """Whether we use mirror."""
-    self.config.GetBoolean('repo.mirror')
+    return self.config.GetBoolean('repo.mirror')
 
   @property
   def use_worktree(self):
     """Whether we use worktree."""
-    self.config.GetBoolean('repo.worktree')
+    return self.config.GetBoolean('repo.worktree')
 
   @property
   def clone_bundle(self):
     """Whether we use clone_bundle."""
-    self.config.GetBoolean('repo.clonebundle')
+    return self.config.GetBoolean('repo.clonebundle')
 
   @property
   def submodules(self):
     """Whether we use submodules."""
-    self.config.GetBoolean('repo.submodules')
+    return self.config.GetBoolean('repo.submodules')
 
   @property
   def git_lfs(self):
     """Whether we use git_lfs."""
-    self.config.GetBoolean('repo.git-lfs')
+    return self.config.GetBoolean('repo.git-lfs')
 
   @property
   def use_superproject(self):
     """Whether we use superproject."""
-    self.config.GetBoolean('repo.superproject')
+    return self.config.GetBoolean('repo.superproject')
 
   @property
   def partial_clone(self):
     """Whether this is a partial clone."""
-    self.config.GetBoolean('repo.partialclone')
+    return self.config.GetBoolean('repo.partialclone')
 
   @property
   def depth(self):
     """Partial clone depth."""
-    self.config.GetString('repo.depth')
+    return self.config.GetString('repo.depth')
 
   @property
   def clone_filter(self):
     """The clone filter."""
-    self.config.GetString('repo.clonefilter')
+    return self.config.GetString('repo.clonefilter')
 
   @property
   def partial_clone_exclude(self):
     """Partial clone exclude string"""
-    self.config.GetBoolean('repo.partialcloneexclude')
+    return self.config.GetBoolean('repo.partialcloneexclude')
+
+  @property
+  def manifest_platform(self):
+    """The --platform argument from `repo init`."""
+    return self.config.GetString('manifest.platform')
 
   @property
   def _platform_name(self):
     """Return the name of the platform."""
     return platform.system().lower()
 
+  def SyncWithPossibleInit(self, submanifest, verbose=False,
+                           current_branch_only=False, tags='', git_event_log=None):
+    """Sync a manifestProject, possibly for the first time.
+
+    Call Sync() with arguments from the most recent `repo init`.  If this is a
+    new sub manifest, then inherit options from the parent's manifestProject.
+
+    This is used by subcmds.Sync() to do an initial download of new sub
+    manifests.
+
+    Args:
+      submanifest: an XmlSubmanifest, the submanifest to re-sync.
+      verbose: a boolean, whether to show all output, rather than only errors.
+      current_branch_only: a boolean, whether to only fetch the current manifest
+          branch from the server.
+      tags: a boolean, whether to fetch tags.
+      git_event_log: an EventLog, for git tracing.
+    """
+    # TODO(lamontjones): when refactoring sync (and init?) consider how to
+    # better get the init options that we should use when syncing uncovers a new
+    # submanifest.
+    git_event_log = git_event_log or EventLog()
+    spec = submanifest.ToSubmanifestSpec()
+    # Use the init options from the existing manifestProject, or the parent if
+    # it doesn't exist.
+    #
+    # Today, we only support changing manifest_groups on the sub-manifest, with
+    # no supported-for-the-user way to change the other arguments from those
+    # specified by the outermost manifest.
+    #
+    # TODO(lamontjones): determine which of these should come from the outermost
+    # manifest and which should come from the parent manifest.
+    mp = self if self.Exists else submanifest.parent.manifestProject
+    return self.Sync(
+        manifest_url=spec.manifestUrl,
+        manifest_branch=spec.revision,
+        standalone_manifest=mp.standalone_manifest_url,
+        groups=mp.manifest_groups,
+        platform=mp.manifest_platform,
+        mirror=mp.mirror,
+        dissociate=mp.dissociate,
+        reference=mp.reference,
+        worktree=mp.use_worktree,
+        submodules=mp.submodules,
+        archive=mp.archive,
+        partial_clone=mp.partial_clone,
+        clone_filter=mp.clone_filter,
+        partial_clone_exclude=mp.partial_clone_exclude,
+        clone_bundle=mp.clone_bundle,
+        git_lfs=mp.git_lfs,
+        use_superproject=mp.use_superproject,
+        verbose=verbose,
+        current_branch_only=current_branch_only,
+        tags=tags,
+        depth=mp.depth,
+        git_event_log=git_event_log,
+        manifest_name=spec.manifestName,
+        this_manifest_only=True,
+        outer_manifest=False,
+    )
+
   def Sync(self, _kwargs_only=(), manifest_url='', manifest_branch=None,
            standalone_manifest=False, groups='', mirror=False, reference='',
            dissociate=False, worktree=False, submodules=False, archive=False,
@@ -3479,7 +3569,7 @@ class ManifestProject(MetaProject):
       platform: a string, restrict the checkout to projects with the specified
           platform group.
       git_event_log: an EventLog, for git tracing.
-      tags: a boolean, whether to fetch tags.,
+      tags: a boolean, whether to fetch tags.
       manifest_name: a string, the name of the manifest file to use.
       this_manifest_only: a boolean, whether to only operate on the current sub
           manifest.
@@ -3490,7 +3580,7 @@ class ManifestProject(MetaProject):
     """
     assert _kwargs_only == (), 'Sync only accepts keyword arguments.'
 
-    groups = groups or 'default'
+    groups = groups or self.manifest.GetDefaultGroupsStr(with_platform=False)
     platform = platform or 'auto'
     git_event_log = git_event_log or EventLog()
     if outer_manifest and self.manifest.is_submanifest:
@@ -3620,6 +3710,7 @@ class ManifestProject(MetaProject):
     elif platform != 'none':
       print('fatal: invalid platform flag', file=sys.stderr)
       return False
+    self.config.SetString('manifest.platform', platform)
 
     groups = [x for x in groups if x]
     groupstr = ','.join(groups)
@@ -3703,21 +3794,11 @@ class ManifestProject(MetaProject):
     if use_superproject is not None:
       self.config.SetBoolean('repo.superproject', use_superproject)
 
-    if standalone_manifest:
-      if is_new:
-        manifest_name = 'default.xml'
-        manifest_data = fetch.fetch_file(manifest_url, verbose=verbose)
-        dest = os.path.join(self.worktree, manifest_name)
-        os.makedirs(os.path.dirname(dest), exist_ok=True)
-        with open(dest, 'wb') as f:
-          f.write(manifest_data)
-      return
-
-    if not self.Sync_NetworkHalf(is_new=is_new, quiet=not verbose, verbose=verbose,
-                              clone_bundle=clone_bundle,
-                              current_branch_only=current_branch_only,
-                              tags=tags, submodules=submodules,
-                              clone_filter=clone_filter,
+    if not standalone_manifest:
+      if not self.Sync_NetworkHalf(
+          is_new=is_new, quiet=not verbose, verbose=verbose,
+          clone_bundle=clone_bundle, current_branch_only=current_branch_only,
+          tags=tags, submodules=submodules, clone_filter=clone_filter,
           partial_clone_exclude=self.manifest.PartialCloneExclude):
         r = self.GetRemote(self.remote.name)
         print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
@@ -3744,6 +3825,15 @@ class ManifestProject(MetaProject):
         print('fatal: manifest name (-m) is required.', file=sys.stderr)
         return False
 
+    elif is_new:
+      # This is a new standalone manifest.
+      manifest_name = 'default.xml'
+      manifest_data = fetch.fetch_file(manifest_url, verbose=verbose)
+      dest = os.path.join(self.worktree, manifest_name)
+      os.makedirs(os.path.dirname(dest), exist_ok=True)
+      with open(dest, 'wb') as f:
+        f.write(manifest_data)
+
     try:
       self.manifest.Link(manifest_name)
     except ManifestParseError as e:
@@ -3754,7 +3844,7 @@ class ManifestProject(MetaProject):
 
     if not this_manifest_only:
       for submanifest in self.manifest.submanifests.values():
-        spec = submanifest.ToSubmanifestSpec(root=self.manifest.outer_client)
+        spec = submanifest.ToSubmanifestSpec()
         submanifest.repo_client.manifestProject.Sync(
             manifest_url=spec.manifestUrl,
             manifest_branch=spec.revision,
@@ -3783,10 +3873,10 @@ class ManifestProject(MetaProject):
             outer_manifest=False,
         )
 
-    # Lastly, clone the superproject(s).
-    if self.manifest.manifestProject.use_superproject:
-      sync_result = Superproject(
-          self.manifest, self.manifest.repodir, git_event_log, quiet=not verbose).Sync()
+    # Lastly, if the manifest has a <superproject> then have the superproject
+    # sync it if it will be used.
+    if self.manifest.superproject:
+      sync_result = self.manifest.superproject.Sync(git_event_log)
       if not sync_result.success:
         print('warning: git update of superproject for '
               f'{self.manifest.path_prefix} failed, repo sync will not use '

subcmds/forall.py

@@ -84,6 +84,11 @@ REPO_PROJECT is set to the unique name of the project.
 
 REPO_PATH is the path relative the the root of the client.
 
+REPO_OUTERPATH is the path of the sub manifest's root relative to the root of
+the client.
+
+REPO_INNERPATH is the path relative to the root of the sub manifest.
+
 REPO_REMOTE is the name of the remote system from the manifest.
 
 REPO_LREV is the name of the revision from the manifest, translated
@@ -290,8 +295,9 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
     env[name] = val
 
   setenv('REPO_PROJECT', project.name)
-  setenv('REPO_PATH', project.relpath)
-  setenv('REPO_OUTERPATH', project.RelPath(local=opt.this_manifest_only))
+  setenv('REPO_OUTERPATH', project.manifest.path_prefix)
+  setenv('REPO_INNERPATH', project.relpath)
+  setenv('REPO_PATH', project.RelPath(local=opt.this_manifest_only))
   setenv('REPO_REMOTE', project.remote.name)
   try:
     # If we aren't in a fully synced state and we don't have the ref the manifest

subcmds/info.py

@@ -65,8 +65,7 @@ class Info(PagedCommand):
       self.manifest = self.manifest.outer_client
     manifestConfig = self.manifest.manifestProject.config
     mergeBranch = manifestConfig.GetBranch("default").merge
-    manifestGroups = (manifestConfig.GetString('manifest.groups')
-                      or 'all,-notdefault')
+    manifestGroups = self.manifest.GetGroupsStr()
 
     self.heading("Manifest branch: ")
     if self.manifest.default.revisionExpr:

subcmds/init.py

@@ -24,8 +24,6 @@ from error import ManifestParseError
 from project import SyncBuffer
 from git_config import GitConfig
 from git_command import git_require, MIN_GIT_VERSION_SOFT, MIN_GIT_VERSION_HARD
-import fetch
-import platform_utils
 from wrapper import Wrapper
 
 
@@ -91,11 +89,10 @@ to update the working directory files.
   def _Options(self, p, gitc_init=False):
     Wrapper().InitParser(p, gitc_init=gitc_init)
     m = p.add_option_group('Multi-manifest')
-    m.add_option('--outer-manifest', action='store_true',
+    m.add_option('--outer-manifest', action='store_true', default=True,
                  help='operate starting at the outermost manifest')
     m.add_option('--no-outer-manifest', dest='outer_manifest',
-                 action='store_false', default=None,
-                 help='do not operate on outer manifests')
+                 action='store_false', help='do not operate on outer manifests')
     m.add_option('--this-manifest-only', action='store_true', default=None,
                  help='only operate on this (sub)manifest')
     m.add_option('--no-this-manifest-only', '--all-manifests',
@@ -260,6 +257,9 @@ to update the working directory files.
       if opt.use_superproject is not None:
         self.OptionParser.error('--mirror and --use-superproject cannot be '
                                 'used together.')
+    if opt.archive and opt.use_superproject is not None:
+      self.OptionParser.error('--archive and --use-superproject cannot be used '
+                              'together.')
 
     if opt.standalone_manifest and (opt.manifest_branch or
                                     opt.manifest_name != 'default.xml'):

subcmds/sync.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
 import functools
 import http.cookiejar as cookielib
 import io
@@ -66,7 +67,7 @@ _ONE_DAY_S = 24 * 60 * 60
 class Sync(Command, MirrorSafeCommand):
   jobs = 1
   COMMON = True
-  MULTI_MANIFEST_SUPPORT = False
+  MULTI_MANIFEST_SUPPORT = True
   helpSummary = "Update working tree to the latest revision"
   helpUsage = """
 %prog [<project>...]
@@ -170,9 +171,9 @@ later is required to fix a server side protocol bug.
   PARALLEL_JOBS = 1
 
   def _CommonOptions(self, p):
-    if self.manifest:
+    if self.outer_client and self.outer_client.manifest:
       try:
-        self.PARALLEL_JOBS = self.manifest.default.sync_j
+        self.PARALLEL_JOBS = self.outer_client.manifest.default.sync_j
       except ManifestParseError:
         pass
     super()._CommonOptions(p)
@@ -270,68 +271,117 @@ later is required to fix a server side protocol bug.
                  dest='repo_upgraded', action='store_true',
                  help=SUPPRESS_HELP)
 
-  def _GetBranch(self):
-    """Returns the branch name for getting the approved manifest."""
-    p = self.manifest.manifestProject
-    b = p.GetBranch(p.CurrentBranch)
+  def _GetBranch(self, manifest_project):
+    """Returns the branch name for getting the approved smartsync manifest.
+
+    Args:
+      manifest_project: the manifestProject to query.
+    """
+    b = manifest_project.GetBranch(manifest_project.CurrentBranch)
     branch = b.merge
     if branch.startswith(R_HEADS):
       branch = branch[len(R_HEADS):]
     return branch
 
-  def _GetCurrentBranchOnly(self, opt):
+  def _GetCurrentBranchOnly(self, opt, manifest):
     """Returns whether current-branch or use-superproject options are enabled.
 
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      manifest: The manifest to use.
+
     Returns:
       True if a superproject is requested, otherwise the value of the
       current_branch option (True, False or None).
     """
-    return git_superproject.UseSuperproject(opt, self.manifest) or opt.current_branch_only
+    return git_superproject.UseSuperproject(opt.use_superproject, manifest) or opt.current_branch_only
 
-  def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests, superproject_logging_data):
-    """Update revisionId of every project with the SHA from superproject.
+  def _UpdateProjectsRevisionId(self, opt, args, superproject_logging_data,
+                                manifest):
+    """Update revisionId of projects with the commit hash from the superproject.
 
-    This function updates each project's revisionId with SHA from superproject.
-    It writes the updated manifest into a file and reloads the manifest from it.
+    This function updates each project's revisionId with the commit hash from
+    the superproject.  It writes the updated manifest into a file and reloads
+    the manifest from it.  When appropriate, sub manifests are also processed.
 
     Args:
       opt: Program options returned from optparse.  See _Options().
       args: Arguments to pass to GetProjects. See the GetProjects
           docstring for details.
-      load_local_manifests: Whether to load local manifests.
-      superproject_logging_data: A dictionary of superproject data that is to be logged.
-
-    Returns:
-      Returns path to the overriding manifest file instead of None.
+      superproject_logging_data: A dictionary of superproject data to log.
+      manifest: The manifest to use.
     """
-    superproject = self.manifest.superproject
-    superproject.SetQuiet(opt.quiet)
-    print_messages = git_superproject.PrintMessages(opt, self.manifest)
-    superproject.SetPrintMessages(print_messages)
+    have_superproject = manifest.superproject or any(
+        m.superproject for m in manifest.all_children)
+    if not have_superproject:
+      return
+
     if opt.local_only:
-      manifest_path = superproject.manifest_path
+      manifest_path = manifest.superproject.manifest_path
       if manifest_path:
-        self._ReloadManifest(manifest_path, load_local_manifests)
-      return manifest_path
+        self._ReloadManifest(manifest_path, manifest)
+      return
 
     all_projects = self.GetProjects(args,
                                     missing_ok=True,
-                                    submodules_ok=opt.fetch_submodules)
-    update_result = superproject.UpdateProjectsRevisionId(
-        all_projects, git_event_log=self.git_event_log)
+                                    submodules_ok=opt.fetch_submodules,
+                                    manifest=manifest,
+                                    all_manifests=not opt.this_manifest_only)
+
+    per_manifest = collections.defaultdict(list)
+    manifest_paths = {}
+    if opt.this_manifest_only:
+      per_manifest[manifest.path_prefix] = all_projects
+    else:
+      for p in all_projects:
+        per_manifest[p.manifest.path_prefix].append(p)
+
+    superproject_logging_data = {}
+    need_unload = False
+    for m in self.ManifestList(opt):
+      if not m.path_prefix in per_manifest:
+        continue
+      use_super = git_superproject.UseSuperproject(opt.use_superproject, m)
+      if superproject_logging_data:
+        superproject_logging_data['multimanifest'] = True
+      superproject_logging_data.update(
+          superproject=use_super,
+          haslocalmanifests=bool(m.HasLocalManifests),
+          hassuperprojecttag=bool(m.superproject),
+      )
+      if use_super and (m.IsMirror or m.IsArchive):
+        # Don't use superproject, because we have no working tree.
+        use_super = False
+        superproject_logging_data['superproject'] = False
+        superproject_logging_data['noworktree'] = True
+        if opt.use_superproject is not False:
+          print(f'{m.path_prefix}: not using superproject because there is no '
+                'working tree.')
+
+      if not use_super:
+        continue
+      m.superproject.SetQuiet(opt.quiet)
+      print_messages = git_superproject.PrintMessages(opt.use_superproject, m)
+      m.superproject.SetPrintMessages(print_messages)
+      update_result = m.superproject.UpdateProjectsRevisionId(
+          per_manifest[m.path_prefix], git_event_log=self.git_event_log)
       manifest_path = update_result.manifest_path
       superproject_logging_data['updatedrevisionid'] = bool(manifest_path)
       if manifest_path:
-      self._ReloadManifest(manifest_path, load_local_manifests)
+        m.SetManifestOverride(manifest_path)
+        need_unload = True
       else:
         if print_messages:
-        print('warning: Update of revisionId from superproject has failed, '
-              'repo sync will not use superproject to fetch the source. ',
-              'Please resync with the --no-use-superproject option to avoid this repo warning.',
+          print(f'{m.path_prefix}: warning: Update of revisionId from '
+                'superproject has failed, repo sync will not use superproject '
+                'to fetch the source. ',
+                'Please resync with the --no-use-superproject option to avoid '
+                'this repo warning.',
                 file=sys.stderr)
         if update_result.fatal and opt.use_superproject is not None:
           sys.exit(1)
-    return manifest_path
+    if need_unload:
+      m.outer_client.manifest.Unload()
 
   def _FetchProjectList(self, opt, projects):
     """Main function of the fetch worker.
@@ -365,16 +415,16 @@ later is required to fix a server side protocol bug.
           quiet=opt.quiet,
           verbose=opt.verbose,
           output_redir=buf,
-          current_branch_only=self._GetCurrentBranchOnly(opt),
+          current_branch_only=self._GetCurrentBranchOnly(opt, project.manifest),
           force_sync=opt.force_sync,
           clone_bundle=opt.clone_bundle,
-          tags=opt.tags, archive=self.manifest.IsArchive,
+          tags=opt.tags, archive=project.manifest.IsArchive,
           optimized_fetch=opt.optimized_fetch,
           retry_fetches=opt.retry_fetches,
           prune=opt.prune,
           ssh_proxy=self.ssh_proxy,
-          clone_filter=self.manifest.CloneFilter,
-          partial_clone_exclude=self.manifest.PartialCloneExclude)
+          clone_filter=project.manifest.CloneFilter,
+          partial_clone_exclude=project.manifest.PartialCloneExclude)
 
       output = buf.getvalue()
       if (opt.verbose or not success) and output:
@@ -471,13 +521,13 @@ later is required to fix a server side protocol bug.
     pm.end()
     self._fetch_times.Save()
 
-    if not self.manifest.IsArchive:
+    if not self.outer_client.manifest.IsArchive:
       self._GCProjects(projects, opt, err_event)
 
     return (ret, fetched)
 
-  def _FetchMain(self, opt, args, all_projects, err_event, manifest_name,
-                 load_local_manifests, ssh_proxy):
+  def _FetchMain(self, opt, args, all_projects, err_event,
+                 ssh_proxy, manifest):
     """The main network fetch loop.
 
     Args:
@@ -485,14 +535,13 @@ later is required to fix a server side protocol bug.
       args: Command line args used to filter out projects.
       all_projects: List of all projects that should be fetched.
       err_event: Whether an error was hit while processing.
-      manifest_name: Manifest file to be reloaded.
-      load_local_manifests: Whether to load local manifests.
       ssh_proxy: SSH manager for clients & masters.
+      manifest: The manifest to use.
 
     Returns:
       List of all projects that should be checked out.
     """
-    rp = self.manifest.repoProject
+    rp = manifest.repoProject
 
     to_fetch = []
     now = time.time()
@@ -516,10 +565,12 @@ later is required to fix a server side protocol bug.
     # Iteratively fetch missing and/or nested unregistered submodules
     previously_missing_set = set()
     while True:
-      self._ReloadManifest(manifest_name, load_local_manifests)
+      self._ReloadManifest(None, manifest)
       all_projects = self.GetProjects(args,
                                       missing_ok=True,
-                                      submodules_ok=opt.fetch_submodules)
+                                      submodules_ok=opt.fetch_submodules,
+                                      manifest=manifest,
+                                      all_manifests=not opt.this_manifest_only)
       missing = []
       for project in all_projects:
         if project.gitdir not in fetched:
@@ -551,7 +602,7 @@ later is required to fix a server side protocol bug.
       Whether the fetch was successful.
     """
     start = time.time()
-    syncbuf = SyncBuffer(self.manifest.manifestProject.config,
+    syncbuf = SyncBuffer(project.manifest.manifestProject.config,
                          detach_head=detach_head)
     success = False
     try:
@@ -614,7 +665,7 @@ later is required to fix a server side protocol bug.
     for project in projects:
       # Make sure pruning never kicks in with shared projects.
       if (not project.use_git_worktrees and
-              len(project.manifest.GetProjectsWithName(project.name)) > 1):
+              len(project.manifest.GetProjectsWithName(project.name, all_manifests=True)) > 1):
         if not opt.quiet:
           print('\r%s: Shared project %s found, disabling pruning.' %
                 (project.relpath, project.name))
@@ -688,28 +739,41 @@ later is required to fix a server side protocol bug.
       t.join()
     pm.end()
 
-  def _ReloadManifest(self, manifest_name=None, load_local_manifests=True):
+  def _ReloadManifest(self, manifest_name, manifest):
     """Reload the manfiest from the file specified by the |manifest_name|.
 
     It unloads the manifest if |manifest_name| is None.
 
     Args:
       manifest_name: Manifest file to be reloaded.
-      load_local_manifests: Whether to load local manifests.
+      manifest: The manifest to use.
     """
     if manifest_name:
       # Override calls Unload already
-      self.manifest.Override(manifest_name, load_local_manifests=load_local_manifests)
+      manifest.Override(manifest_name)
     else:
-      self.manifest.Unload()
+      manifest.Unload()
 
-  def UpdateProjectList(self, opt):
+  def UpdateProjectList(self, opt, manifest):
+    """Update the cached projects list for |manifest|
+
+    In a multi-manifest checkout, each manifest has its own project.list.
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      manifest: The manifest to use.
+
+    Returns:
+      0: success
+      1: failure
+    """
     new_project_paths = []
-    for project in self.GetProjects(None, missing_ok=True):
+    for project in self.GetProjects(None, missing_ok=True, manifest=manifest,
+                                    all_manifests=False):
       if project.relpath:
         new_project_paths.append(project.relpath)
     file_name = 'project.list'
-    file_path = os.path.join(self.manifest.subdir, file_name)
+    file_path = os.path.join(manifest.subdir, file_name)
     old_project_paths = []
 
     if os.path.exists(file_path):
@@ -721,16 +785,16 @@ later is required to fix a server side protocol bug.
           continue
         if path not in new_project_paths:
           # If the path has already been deleted, we don't need to do it
-          gitdir = os.path.join(self.manifest.topdir, path, '.git')
+          gitdir = os.path.join(manifest.topdir, path, '.git')
           if os.path.exists(gitdir):
             project = Project(
-                manifest=self.manifest,
+                manifest=manifest,
                 name=path,
                 remote=RemoteSpec('origin'),
                 gitdir=gitdir,
                 objdir=gitdir,
                 use_git_worktrees=os.path.isfile(gitdir),
-                worktree=os.path.join(self.manifest.topdir, path),
+                worktree=os.path.join(manifest.topdir, path),
                 relpath=path,
                 revisionExpr='HEAD',
                 revisionId=None,
@@ -746,7 +810,7 @@ later is required to fix a server side protocol bug.
       fd.write('\n')
     return 0
 
-  def UpdateCopyLinkfileList(self):
+  def UpdateCopyLinkfileList(self, manifest):
     """Save all dests of copyfile and linkfile, and update them if needed.
 
     Returns:
@@ -755,7 +819,8 @@ later is required to fix a server side protocol bug.
     new_paths = {}
     new_linkfile_paths = []
     new_copyfile_paths = []
-    for project in self.GetProjects(None, missing_ok=True):
+    for project in self.GetProjects(None, missing_ok=True,
+                                    manifest=manifest, all_manifests=False):
       new_linkfile_paths.extend(x.dest for x in project.linkfiles)
       new_copyfile_paths.extend(x.dest for x in project.copyfiles)
 
@@ -765,7 +830,7 @@ later is required to fix a server side protocol bug.
     }
 
     copylinkfile_name = 'copy-link-files.json'
-    copylinkfile_path = os.path.join(self.manifest.subdir, copylinkfile_name)
+    copylinkfile_path = os.path.join(manifest.subdir, copylinkfile_name)
     old_copylinkfile_paths = {}
 
     if os.path.exists(copylinkfile_path):
@@ -796,13 +861,13 @@ later is required to fix a server side protocol bug.
       json.dump(new_paths, fp)
     return True
 
-  def _SmartSyncSetup(self, opt, smart_sync_manifest_path):
-    if not self.manifest.manifest_server:
+  def _SmartSyncSetup(self, opt, smart_sync_manifest_path, manifest):
+    if not manifest.manifest_server:
       print('error: cannot smart sync: no manifest server defined in '
             'manifest', file=sys.stderr)
       sys.exit(1)
 
-    manifest_server = self.manifest.manifest_server
+    manifest_server = manifest.manifest_server
     if not opt.quiet:
       print('Using manifest server %s' % manifest_server)
 
@@ -843,7 +908,7 @@ later is required to fix a server side protocol bug.
     try:
       server = xmlrpc.client.Server(manifest_server, transport=transport)
       if opt.smart_sync:
-        branch = self._GetBranch()
+        branch = self._GetBranch(manifest.manifestProject)
 
         if 'SYNC_TARGET' in os.environ:
           target = os.environ['SYNC_TARGET']
@@ -869,36 +934,68 @@ later is required to fix a server side protocol bug.
                 % (smart_sync_manifest_path, e),
                 file=sys.stderr)
           sys.exit(1)
-        self._ReloadManifest(manifest_name)
+        self._ReloadManifest(manifest_name, manifest)
       else:
         print('error: manifest server RPC call failed: %s' %
               manifest_str, file=sys.stderr)
         sys.exit(1)
     except (socket.error, IOError, xmlrpc.client.Fault) as e:
       print('error: cannot connect to manifest server %s:\n%s'
-            % (self.manifest.manifest_server, e), file=sys.stderr)
+            % (manifest.manifest_server, e), file=sys.stderr)
       sys.exit(1)
     except xmlrpc.client.ProtocolError as e:
       print('error: cannot connect to manifest server %s:\n%d %s'
-            % (self.manifest.manifest_server, e.errcode, e.errmsg),
+            % (manifest.manifest_server, e.errcode, e.errmsg),
             file=sys.stderr)
       sys.exit(1)
 
     return manifest_name
 
+  def _UpdateAllManifestProjects(self, opt, mp, manifest_name):
+    """Fetch & update the local manifest project.
+
+    After syncing the manifest project, if the manifest has any sub manifests,
+    those are recursively processed.
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      mp: the manifestProject to query.
+      manifest_name: Manifest file to be reloaded.
+    """
+    if not mp.standalone_manifest_url:
+      self._UpdateManifestProject(opt, mp, manifest_name)
+
+    if mp.manifest.submanifests:
+      for submanifest in mp.manifest.submanifests.values():
+        child = submanifest.repo_client.manifest
+        child.manifestProject.SyncWithPossibleInit(
+            submanifest,
+            current_branch_only=self._GetCurrentBranchOnly(opt, child),
+            verbose=opt.verbose,
+            tags=opt.tags,
+            git_event_log=self.git_event_log,
+        )
+        self._UpdateAllManifestProjects(opt, child.manifestProject, None)
+
   def _UpdateManifestProject(self, opt, mp, manifest_name):
-    """Fetch & update the local manifest project."""
+    """Fetch & update the local manifest project.
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      mp: the manifestProject to query.
+      manifest_name: Manifest file to be reloaded.
+    """
     if not opt.local_only:
       start = time.time()
       success = mp.Sync_NetworkHalf(quiet=opt.quiet, verbose=opt.verbose,
-                                    current_branch_only=self._GetCurrentBranchOnly(opt),
+                                    current_branch_only=self._GetCurrentBranchOnly(opt, mp.manifest),
                                     force_sync=opt.force_sync,
                                     tags=opt.tags,
                                     optimized_fetch=opt.optimized_fetch,
                                     retry_fetches=opt.retry_fetches,
-                                    submodules=self.manifest.HasSubmodules,
-                                    clone_filter=self.manifest.CloneFilter,
-                                    partial_clone_exclude=self.manifest.PartialCloneExclude)
+                                    submodules=mp.manifest.HasSubmodules,
+                                    clone_filter=mp.manifest.CloneFilter,
+                                    partial_clone_exclude=mp.manifest.PartialCloneExclude)
       finish = time.time()
       self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK,
                              start, finish, success)
@@ -906,15 +1003,16 @@ later is required to fix a server side protocol bug.
     if mp.HasChanges:
       syncbuf = SyncBuffer(mp.config)
       start = time.time()
-      mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules)
+      mp.Sync_LocalHalf(syncbuf, submodules=mp.manifest.HasSubmodules)
       clean = syncbuf.Finish()
       self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL,
                              start, time.time(), clean)
       if not clean:
         sys.exit(1)
-      self._ReloadManifest(manifest_name)
+      self._ReloadManifest(manifest_name, mp.manifest)
+
       if opt.jobs is None:
-        self.jobs = self.manifest.default.sync_j
+        self.jobs = mp.manifest.default.sync_j
 
   def ValidateOptions(self, opt, args):
     if opt.force_broken:
@@ -937,9 +1035,6 @@ later is required to fix a server side protocol bug.
     if opt.prune is None:
       opt.prune = True
 
-    if self.manifest.is_multimanifest and not opt.this_manifest_only and args:
-      self.OptionParser.error('partial syncs must use --this-manifest-only')
-
   def Execute(self, opt, args):
     if opt.jobs:
       self.jobs = opt.jobs
@@ -947,18 +1042,22 @@ later is required to fix a server side protocol bug.
       soft_limit, _ = _rlimit_nofile()
       self.jobs = min(self.jobs, (soft_limit - 5) // 3)
 
+    manifest = self.outer_manifest
+    if not opt.outer_manifest:
+      manifest = self.manifest
+
     if opt.manifest_name:
-      self.manifest.Override(opt.manifest_name)
+      manifest.Override(opt.manifest_name)
 
     manifest_name = opt.manifest_name
     smart_sync_manifest_path = os.path.join(
-        self.manifest.manifestProject.worktree, 'smart_sync_override.xml')
+        manifest.manifestProject.worktree, 'smart_sync_override.xml')
 
     if opt.clone_bundle is None:
-      opt.clone_bundle = self.manifest.CloneBundle
+      opt.clone_bundle = manifest.CloneBundle
 
     if opt.smart_sync or opt.smart_tag:
-      manifest_name = self._SmartSyncSetup(opt, smart_sync_manifest_path)
+      manifest_name = self._SmartSyncSetup(opt, smart_sync_manifest_path, manifest)
     else:
       if os.path.isfile(smart_sync_manifest_path):
         try:
@@ -969,7 +1068,7 @@ later is required to fix a server side protocol bug.
 
     err_event = multiprocessing.Event()
 
-    rp = self.manifest.repoProject
+    rp = manifest.repoProject
     rp.PreSync()
     cb = rp.CurrentBranch
     if cb:
@@ -979,38 +1078,26 @@ later is required to fix a server side protocol bug.
               'receive updates; run `repo init --repo-rev=stable` to fix.',
               file=sys.stderr)
 
-    mp = self.manifest.manifestProject
+    for m in self.ManifestList(opt):
+      mp = m.manifestProject
       is_standalone_manifest = bool(mp.standalone_manifest_url)
       if not is_standalone_manifest:
         mp.PreSync()
 
       if opt.repo_upgraded:
-      _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
+        _PostRepoUpgrade(m, quiet=opt.quiet)
 
-    if not opt.mp_update:
+    if opt.mp_update:
+      self._UpdateAllManifestProjects(opt, mp, manifest_name)
+    else:
       print('Skipping update of local manifest project.')
-    elif not is_standalone_manifest:
-      self._UpdateManifestProject(opt, mp, manifest_name)
 
-    load_local_manifests = not self.manifest.HasLocalManifests
-    use_superproject = git_superproject.UseSuperproject(opt, self.manifest)
-    if use_superproject and (self.manifest.IsMirror or self.manifest.IsArchive):
-      # Don't use superproject, because we have no working tree.
-      use_superproject = False
-      if opt.use_superproject is not None:
-        print('Defaulting to no-use-superproject because there is no working tree.')
-    superproject_logging_data = {
-        'superproject': use_superproject,
-        'haslocalmanifests': bool(self.manifest.HasLocalManifests),
-        'hassuperprojecttag': bool(self.manifest.superproject),
-    }
-    if use_superproject:
-      manifest_name = self._UpdateProjectsRevisionId(
-          opt, args, load_local_manifests, superproject_logging_data) or opt.manifest_name
+    superproject_logging_data = {}
+    self._UpdateProjectsRevisionId(opt, args, superproject_logging_data,
+                                   manifest)
 
     if self.gitc_manifest:
-      gitc_manifest_projects = self.GetProjects(args,
-                                                missing_ok=True)
+      gitc_manifest_projects = self.GetProjects(args, missing_ok=True)
       gitc_projects = []
       opened_projects = []
       for project in gitc_manifest_projects:
@@ -1029,7 +1116,7 @@ later is required to fix a server side protocol bug.
         if manifest_name:
           manifest.Override(manifest_name)
         else:
-          manifest.Override(self.manifest.manifestFile)
+          manifest.Override(manifest.manifestFile)
         gitc_utils.generate_gitc_manifest(self.gitc_manifest,
                                           manifest,
                                           gitc_projects)
@@ -1039,26 +1126,28 @@ later is required to fix a server side protocol bug.
       # generate a new args list to represent the opened projects.
       # TODO: make this more reliable -- if there's a project name/path overlap,
       # this may choose the wrong project.
-      args = [os.path.relpath(self.manifest.paths[path].worktree, os.getcwd())
+      args = [os.path.relpath(manifest.paths[path].worktree, os.getcwd())
               for path in opened_projects]
       if not args:
         return
+
     all_projects = self.GetProjects(args,
                                     missing_ok=True,
-                                    submodules_ok=opt.fetch_submodules)
+                                    submodules_ok=opt.fetch_submodules,
+                                    manifest=manifest,
+                                    all_manifests=not opt.this_manifest_only)
 
     err_network_sync = False
     err_update_projects = False
 
-    self._fetch_times = _FetchTimes(self.manifest)
+    self._fetch_times = _FetchTimes(manifest)
     if not opt.local_only:
       with multiprocessing.Manager() as manager:
         with ssh.ProxyManager(manager) as ssh_proxy:
           # Initialize the socket dir once in the parent.
           ssh_proxy.sock()
           all_projects = self._FetchMain(opt, args, all_projects, err_event,
-                                         manifest_name, load_local_manifests,
-                                         ssh_proxy)
+                                         ssh_proxy, manifest)
 
       if opt.network_only:
         return
@@ -1074,18 +1163,19 @@ later is required to fix a server side protocol bug.
                 file=sys.stderr)
           sys.exit(1)
 
-    if self.manifest.IsMirror or self.manifest.IsArchive:
+    for m in self.ManifestList(opt):
+      if m.IsMirror or m.IsArchive:
         # bail out now, we have no working tree
-      return
+        continue
 
-    if self.UpdateProjectList(opt):
+      if self.UpdateProjectList(opt, m):
         err_event.set()
         err_update_projects = True
         if opt.fail_fast:
           print('\nerror: Local checkouts *not* updated.', file=sys.stderr)
           sys.exit(1)
 
-    err_update_linkfiles = not self.UpdateCopyLinkfileList()
+      err_update_linkfiles = not self.UpdateCopyLinkfileList(m)
       if err_update_linkfiles:
         err_event.set()
         if opt.fail_fast:
@@ -1098,10 +1188,14 @@ later is required to fix a server side protocol bug.
     if err_checkout:
       err_event.set()
 
-    # If there's a notice that's supposed to print at the end of the sync, print
-    # it now...
-    if self.manifest.notice:
-      print(self.manifest.notice)
+    printed_notices = set()
+    # If there's a notice that's supposed to print at the end of the sync,
+    # print it now...  But avoid printing duplicate messages, and preserve
+    # order.
+    for m in sorted(self.ManifestList(opt), key=lambda x: x.path_prefix):
+      if m.notice and m.notice not in printed_notices:
+        print(m.notice)
+        printed_notices.add(m.notice)
 
     # If we saw an error, exit with code 1 so that other scripts can check.
     if err_event.is_set():

subcmds/upload.py

@@ -421,12 +421,6 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
         labels = set(_ExpandCommaList(branch.project.config.GetString(key)))
         for label in opt.labels:
           labels.update(_ExpandCommaList(label))
-        # Basic sanity check on label syntax.
-        for label in labels:
-          if not re.match(r'^.+[+-][0-9]+$', label):
-            print('repo: error: invalid label syntax "%s": labels use forms '
-                  'like CodeReview+1 or Verified-1' % (label,), file=sys.stderr)
-            sys.exit(1)
 
         # Handle e-mail notifications.
         if opt.notify is False:

tests/test_git_superproject.py

@@ -24,7 +24,6 @@ from unittest import mock
 import git_superproject
 import git_trace2_event_log
 import manifest_xml
-import platform_utils
 from test_manifest_xml import sort_attributes
 
 
@@ -38,7 +37,8 @@ class SuperprojectTestCase(unittest.TestCase):
 
   def setUp(self):
     """Set up superproject every time."""
-    self.tempdir = tempfile.mkdtemp(prefix='repo_tests')
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    self.tempdir = self.tempdirobj.name
     self.repodir = os.path.join(self.tempdir, '.repo')
     self.manifest_file = os.path.join(
         self.repodir, manifest_xml.MANIFEST_FILE_NAME)
@@ -75,7 +75,7 @@ class SuperprojectTestCase(unittest.TestCase):
 
   def tearDown(self):
     """Tear down superproject every time."""
-    platform_utils.rmtree(self.tempdir)
+    self.tempdirobj.cleanup()
 
   def getXmlManifest(self, data):
     """Helper to initialize a manifest for testing."""

tests/test_manifest_xml.py

@@ -17,7 +17,6 @@
 import os
 import platform
 import re
-import shutil
 import tempfile
 import unittest
 import xml.dom.minidom
@@ -92,7 +91,8 @@ class ManifestParseTestCase(unittest.TestCase):
   """TestCase for parsing manifests."""
 
   def setUp(self):
-    self.tempdir = tempfile.mkdtemp(prefix='repo_tests')
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    self.tempdir = self.tempdirobj.name
     self.repodir = os.path.join(self.tempdir, '.repo')
     self.manifest_dir = os.path.join(self.repodir, 'manifests')
     self.manifest_file = os.path.join(
@@ -111,7 +111,7 @@ class ManifestParseTestCase(unittest.TestCase):
 """)
 
   def tearDown(self):
-    shutil.rmtree(self.tempdir, ignore_errors=True)
+    self.tempdirobj.cleanup()
 
   def getXmlManifest(self, data):
     """Helper to initialize a manifest for testing."""

tests/test_project.py

@@ -17,7 +17,6 @@
 import contextlib
 import os
 from pathlib import Path
-import shutil
 import subprocess
 import tempfile
 import unittest
@@ -32,11 +31,7 @@ import project
 @contextlib.contextmanager
 def TempGitTree():
   """Create a new empty git checkout for testing."""
-  # TODO(vapier): Convert this to tempfile.TemporaryDirectory once we drop
-  # Python 2 support entirely.
-  try:
-    tempdir = tempfile.mkdtemp(prefix='repo-tests')
-
+  with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
     # Tests need to assume, that main is default branch at init,
     # which is not supported in config until 2.28.
     cmd = ['git', 'init']
@@ -50,8 +45,6 @@ def TempGitTree():
       cmd += ['--template', templatedir]
     subprocess.check_call(cmd, cwd=tempdir)
     yield tempdir
-  finally:
-    platform_utils.rmtree(tempdir)
 
 
 class FakeProject(object):
@@ -124,14 +117,15 @@ class CopyLinkTestCase(unittest.TestCase):
   """
 
   def setUp(self):
-    self.tempdir = tempfile.mkdtemp(prefix='repo_tests')
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    self.tempdir = self.tempdirobj.name
     self.topdir = os.path.join(self.tempdir, 'checkout')
     self.worktree = os.path.join(self.topdir, 'git-project')
     os.makedirs(self.topdir)
     os.makedirs(self.worktree)
 
   def tearDown(self):
-    shutil.rmtree(self.tempdir, ignore_errors=True)
+    self.tempdirobj.cleanup()
 
   @staticmethod
   def touch(path):

tests/test_subcmds_sync.py

@@ -42,4 +42,4 @@ def test_get_current_branch_only(use_superproject, cli_args, result):
   opts, _ = cmd.OptionParser.parse_args(cli_args)
 
   with mock.patch('git_superproject.UseSuperproject', return_value=use_superproject):
-    assert cmd._GetCurrentBranchOnly(opts) == result
+    assert cmd._GetCurrentBranchOnly(opts, cmd.manifest) == result

tests/test_wrapper.py

@@ -14,11 +14,9 @@
 
 """Unittests for the wrapper.py module."""
 
-import contextlib
 from io import StringIO
 import os
 import re
-import shutil
 import sys
 import tempfile
 import unittest
@@ -26,22 +24,9 @@ from unittest import mock
 
 import git_command
 import main
-import platform_utils
 import wrapper
 
 
-@contextlib.contextmanager
-def TemporaryDirectory():
-  """Create a new empty git checkout for testing."""
-  # TODO(vapier): Convert this to tempfile.TemporaryDirectory once we drop
-  # Python 2 support entirely.
-  try:
-    tempdir = tempfile.mkdtemp(prefix='repo-tests')
-    yield tempdir
-  finally:
-    platform_utils.rmtree(tempdir)
-
-
 def fixture(*paths):
   """Return a path relative to tests/fixtures.
   """
@@ -336,19 +321,19 @@ class NeedSetupGnuPG(RepoWrapperTestCase):
 
   def test_missing_dir(self):
     """The ~/.repoconfig tree doesn't exist yet."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = os.path.join(tempdir, 'foo')
       self.assertTrue(self.wrapper.NeedSetupGnuPG())
 
   def test_missing_keyring(self):
     """The keyring-version file doesn't exist yet."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       self.assertTrue(self.wrapper.NeedSetupGnuPG())
 
   def test_empty_keyring(self):
     """The keyring-version file exists, but is empty."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       with open(os.path.join(tempdir, 'keyring-version'), 'w'):
         pass
@@ -356,7 +341,7 @@ class NeedSetupGnuPG(RepoWrapperTestCase):
 
   def test_old_keyring(self):
     """The keyring-version file exists, but it's old."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       with open(os.path.join(tempdir, 'keyring-version'), 'w') as fp:
         fp.write('1.0\n')
@@ -364,7 +349,7 @@ class NeedSetupGnuPG(RepoWrapperTestCase):
 
   def test_new_keyring(self):
     """The keyring-version file exists, and is up-to-date."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       with open(os.path.join(tempdir, 'keyring-version'), 'w') as fp:
         fp.write('1000.0\n')
@@ -376,7 +361,7 @@ class SetupGnuPG(RepoWrapperTestCase):
 
   def test_full(self):
     """Make sure it works completely."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       self.wrapper.gpg_dir = os.path.join(self.wrapper.home_dot_repo, 'gnupg')
       self.assertTrue(self.wrapper.SetupGnuPG(True))
@@ -426,7 +411,8 @@ class GitCheckoutTestCase(RepoWrapperTestCase):
   @classmethod
   def setUpClass(cls):
     # Create a repo to operate on, but do it once per-class.
-    cls.GIT_DIR = tempfile.mkdtemp(prefix='repo-rev-tests')
+    cls.tempdirobj = tempfile.TemporaryDirectory(prefix='repo-rev-tests')
+    cls.GIT_DIR = cls.tempdirobj.name
     run_git = wrapper.Wrapper().run_git
 
     remote = os.path.join(cls.GIT_DIR, 'remote')
@@ -455,10 +441,10 @@ class GitCheckoutTestCase(RepoWrapperTestCase):
 
   @classmethod
   def tearDownClass(cls):
-    if not cls.GIT_DIR:
+    if not cls.tempdirobj:
       return
 
-    shutil.rmtree(cls.GIT_DIR)
+    cls.tempdirobj.cleanup()
 
 
 class ResolveRepoRev(GitCheckoutTestCase):