sync: add multi-manifest support

With this change, partial syncs (sync with a project list) are again
supported.

If the updated manifest includes new sub manifests, download them
inheriting options from the parent manifestProject.

Change-Id: Id952f85df2e26d34e38b251973be26434443ff56
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/334819
Reviewed-by: Mike Frysinger <vapier@google.com>
Tested-by: LaMont Jones <lamontjones@google.com>

command.py

@@ -144,11 +144,10 @@ class Command(object):
           help=f'number of jobs to run in parallel (default: {default})')
 
     m = p.add_option_group('Multi-manifest options')
-    m.add_option('--outer-manifest', action='store_true',
+    m.add_option('--outer-manifest', action='store_true', default=None,
                  help='operate starting at the outermost manifest')
     m.add_option('--no-outer-manifest', dest='outer_manifest',
-                 action='store_false', default=None,
-                 help='do not operate on outer manifests')
+                 action='store_false', help='do not operate on outer manifests')
     m.add_option('--this-manifest-only', action='store_true', default=None,
                  help='only operate on this (sub)manifest')
     m.add_option('--no-this-manifest-only', '--all-manifests',
@@ -186,6 +185,10 @@ class Command(object):
     """Validate common options."""
     opt.quiet = opt.output_mode is False
     opt.verbose = opt.output_mode is True
+    if opt.outer_manifest is None:
+      # By default, treat multi-manifest instances as a single manifest from
+      # the user's perspective.
+      opt.outer_manifest = True
 
   def ValidateOptions(self, opt, args):
     """Validate the user options & arguments before executing.
@@ -385,7 +388,7 @@ class Command(object):
       opt: The command options.
     """
     top = self.outer_manifest
-    if opt.outer_manifest is False or opt.this_manifest_only:
+    if not opt.outer_manifest or opt.this_manifest_only:
       top = self.manifest
     yield top
     if not opt.this_manifest_only:

main.py

@@ -294,8 +294,7 @@ class _Repo(object):
       cmd.ValidateOptions(copts, cargs)
 
       this_manifest_only = copts.this_manifest_only
-      # If not specified, default to using the outer manifest.
-      outer_manifest = copts.outer_manifest is not False
+      outer_manifest = copts.outer_manifest
       if cmd.MULTI_MANIFEST_SUPPORT or this_manifest_only:
         result = cmd.Execute(copts, cargs)
       elif outer_manifest and repo_client.manifest.is_submanifest:

project.py

@@ -3467,6 +3467,67 @@ class ManifestProject(MetaProject):
     """Return the name of the platform."""
     return platform.system().lower()
 
+  def SyncWithPossibleInit(self, submanifest, verbose=False,
+                           current_branch_only=False, tags='', git_event_log=None):
+    """Sync a manifestProject, possibly for the first time.
+
+    Call Sync() with arguments from the most recent `repo init`.  If this is a
+    new sub manifest, then inherit options from the parent's manifestProject.
+
+    This is used by subcmds.Sync() to do an initial download of new sub
+    manifests.
+
+    Args:
+      submanifest: an XmlSubmanifest, the submanifest to re-sync.
+      verbose: a boolean, whether to show all output, rather than only errors.
+      current_branch_only: a boolean, whether to only fetch the current manifest
+          branch from the server.
+      tags: a boolean, whether to fetch tags.
+      git_event_log: an EventLog, for git tracing.
+    """
+    # TODO(lamontjones): when refactoring sync (and init?) consider how to
+    # better get the init options that we should use when syncing uncovers a new
+    # submanifest.
+    git_event_log = git_event_log or EventLog()
+    spec = submanifest.ToSubmanifestSpec()
+    # Use the init options from the existing manifestProject, or the parent if
+    # it doesn't exist.
+    #
+    # Today, we only support changing manifest_groups on the sub-manifest, with
+    # no supported-for-the-user way to change the other arguments from those
+    # specified by the outermost manifest.
+    #
+    # TODO(lamontjones): determine which of these should come from the outermost
+    # manifest and which should come from the parent manifest.
+    mp = self if self.Exists else submanifest.parent.manifestProject
+    return self.Sync(
+        manifest_url=spec.manifestUrl,
+        manifest_branch=spec.revision,
+        standalone_manifest=mp.standalone_manifest_url,
+        groups=mp.manifest_groups,
+        platform=mp.manifest_platform,
+        mirror=mp.mirror,
+        dissociate=mp.dissociate,
+        reference=mp.reference,
+        worktree=mp.use_worktree,
+        submodules=mp.submodules,
+        archive=mp.archive,
+        partial_clone=mp.partial_clone,
+        clone_filter=mp.clone_filter,
+        partial_clone_exclude=mp.partial_clone_exclude,
+        clone_bundle=mp.clone_bundle,
+        git_lfs=mp.git_lfs,
+        use_superproject=mp.use_superproject,
+        verbose=verbose,
+        current_branch_only=current_branch_only,
+        tags=tags,
+        depth=mp.depth,
+        git_event_log=git_event_log,
+        manifest_name=spec.manifestName,
+        this_manifest_only=True,
+        outer_manifest=False,
+    )
+
   def Sync(self, _kwargs_only=(), manifest_url='', manifest_branch=None,
            standalone_manifest=False, groups='', mirror=False, reference='',
            dissociate=False, worktree=False, submodules=False, archive=False,

subcmds/init.py

@@ -89,11 +89,10 @@ to update the working directory files.
   def _Options(self, p, gitc_init=False):
     Wrapper().InitParser(p, gitc_init=gitc_init)
     m = p.add_option_group('Multi-manifest')
-    m.add_option('--outer-manifest', action='store_true',
+    m.add_option('--outer-manifest', action='store_true', default=True,
                  help='operate starting at the outermost manifest')
     m.add_option('--no-outer-manifest', dest='outer_manifest',
-                 action='store_false', default=None,
-                 help='do not operate on outer manifests')
+                 action='store_false', help='do not operate on outer manifests')
     m.add_option('--this-manifest-only', action='store_true', default=None,
                  help='only operate on this (sub)manifest')
     m.add_option('--no-this-manifest-only', '--all-manifests',

subcmds/sync.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
 import functools
 import http.cookiejar as cookielib
 import io
@@ -66,7 +67,7 @@ _ONE_DAY_S = 24 * 60 * 60
 class Sync(Command, MirrorSafeCommand):
   jobs = 1
   COMMON = True
-  MULTI_MANIFEST_SUPPORT = False
+  MULTI_MANIFEST_SUPPORT = True
   helpSummary = "Update working tree to the latest revision"
   helpUsage = """
 %prog [<project>...]
@@ -295,52 +296,92 @@ later is required to fix a server side protocol bug.
     """
     return git_superproject.UseSuperproject(opt.use_superproject, manifest) or opt.current_branch_only
 
-  def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests, superproject_logging_data, manifest):
-    """Update revisionId of every project with the SHA from superproject.
+  def _UpdateProjectsRevisionId(self, opt, args, superproject_logging_data,
+                                manifest):
+    """Update revisionId of projects with the commit hash from the superproject.
 
-    This function updates each project's revisionId with SHA from superproject.
-    It writes the updated manifest into a file and reloads the manifest from it.
+    This function updates each project's revisionId with the commit hash from
+    the superproject.  It writes the updated manifest into a file and reloads
+    the manifest from it.  When appropriate, sub manifests are also processed.
 
     Args:
       opt: Program options returned from optparse.  See _Options().
       args: Arguments to pass to GetProjects. See the GetProjects
           docstring for details.
-      load_local_manifests: Whether to load local manifests.
-      superproject_logging_data: A dictionary of superproject data that is to be logged.
+      superproject_logging_data: A dictionary of superproject data to log.
       manifest: The manifest to use.
-
-    Returns:
-      Returns path to the overriding manifest file instead of None.
     """
-    superproject = self.manifest.superproject
-    superproject.SetQuiet(opt.quiet)
-    print_messages = git_superproject.PrintMessages(opt.use_superproject,
-                                                    self.manifest)
-    superproject.SetPrintMessages(print_messages)
+    have_superproject = manifest.superproject or any(
+        m.superproject for m in manifest.all_children)
+    if not have_superproject:
+      return
+
     if opt.local_only:
-      manifest_path = superproject.manifest_path
+      manifest_path = manifest.superproject.manifest_path
       if manifest_path:
-        self._ReloadManifest(manifest_path, manifest, load_local_manifests)
-      return manifest_path
+        self._ReloadManifest(manifest_path, manifest)
+      return
 
     all_projects = self.GetProjects(args,
                                     missing_ok=True,
-                                    submodules_ok=opt.fetch_submodules)
-    update_result = superproject.UpdateProjectsRevisionId(
-        all_projects, git_event_log=self.git_event_log)
-    manifest_path = update_result.manifest_path
-    superproject_logging_data['updatedrevisionid'] = bool(manifest_path)
-    if manifest_path:
-      self._ReloadManifest(manifest_path, manifest, load_local_manifests)
+                                    submodules_ok=opt.fetch_submodules,
+                                    manifest=manifest,
+                                    all_manifests=not opt.this_manifest_only)
+
+    per_manifest = collections.defaultdict(list)
+    manifest_paths = {}
+    if opt.this_manifest_only:
+      per_manifest[manifest.path_prefix] = all_projects
     else:
-      if print_messages:
-        print('warning: Update of revisionId from superproject has failed, '
-              'repo sync will not use superproject to fetch the source. ',
-              'Please resync with the --no-use-superproject option to avoid this repo warning.',
-              file=sys.stderr)
-      if update_result.fatal and opt.use_superproject is not None:
-        sys.exit(1)
-    return manifest_path
+      for p in all_projects:
+        per_manifest[p.manifest.path_prefix].append(p)
+
+    superproject_logging_data = {}
+    need_unload = False
+    for m in self.ManifestList(opt):
+      if not m.path_prefix in per_manifest:
+        continue
+      use_super = git_superproject.UseSuperproject(opt.use_superproject, m)
+      if superproject_logging_data:
+        superproject_logging_data['multimanifest'] = True
+      superproject_logging_data.update(
+          superproject=use_super,
+          haslocalmanifests=bool(m.HasLocalManifests),
+          hassuperprojecttag=bool(m.superproject),
+      )
+      if use_super and (m.IsMirror or m.IsArchive):
+        # Don't use superproject, because we have no working tree.
+        use_super = False
+        superproject_logging_data['superproject'] = False
+        superproject_logging_data['noworktree'] = True
+        if opt.use_superproject is not False:
+          print(f'{m.path_prefix}: not using superproject because there is no '
+                'working tree.')
+
+      if not use_super:
+        continue
+      m.superproject.SetQuiet(opt.quiet)
+      print_messages = git_superproject.PrintMessages(opt.use_superproject, m)
+      m.superproject.SetPrintMessages(print_messages)
+      update_result = m.superproject.UpdateProjectsRevisionId(
+          per_manifest[m.path_prefix], git_event_log=self.git_event_log)
+      manifest_path = update_result.manifest_path
+      superproject_logging_data['updatedrevisionid'] = bool(manifest_path)
+      if manifest_path:
+        m.SetManifestOverride(manifest_path)
+        need_unload = True
+      else:
+        if print_messages:
+          print(f'{m.path_prefix}: warning: Update of revisionId from '
+                'superproject has failed, repo sync will not use superproject '
+                'to fetch the source. ',
+                'Please resync with the --no-use-superproject option to avoid '
+                'this repo warning.',
+                file=sys.stderr)
+        if update_result.fatal and opt.use_superproject is not None:
+          sys.exit(1)
+    if need_unload:
+      m.outer_client.manifest.Unload()
 
   def _FetchProjectList(self, opt, projects):
     """Main function of the fetch worker.
@@ -485,8 +526,8 @@ later is required to fix a server side protocol bug.
 
     return (ret, fetched)
 
-  def _FetchMain(self, opt, args, all_projects, err_event, manifest_name,
-                 load_local_manifests, ssh_proxy, manifest):
+  def _FetchMain(self, opt, args, all_projects, err_event,
+                 ssh_proxy, manifest):
     """The main network fetch loop.
 
     Args:
@@ -494,8 +535,6 @@ later is required to fix a server side protocol bug.
       args: Command line args used to filter out projects.
       all_projects: List of all projects that should be fetched.
       err_event: Whether an error was hit while processing.
-      manifest_name: Manifest file to be reloaded.
-      load_local_manifests: Whether to load local manifests.
       ssh_proxy: SSH manager for clients & masters.
       manifest: The manifest to use.
 
@@ -526,10 +565,12 @@ later is required to fix a server side protocol bug.
     # Iteratively fetch missing and/or nested unregistered submodules
     previously_missing_set = set()
     while True:
-      self._ReloadManifest(manifest_name, self.manifest, load_local_manifests)
+      self._ReloadManifest(None, manifest)
       all_projects = self.GetProjects(args,
                                       missing_ok=True,
-                                      submodules_ok=opt.fetch_submodules)
+                                      submodules_ok=opt.fetch_submodules,
+                                      manifest=manifest,
+                                      all_manifests=not opt.this_manifest_only)
       missing = []
       for project in all_projects:
         if project.gitdir not in fetched:
@@ -624,7 +665,7 @@ later is required to fix a server side protocol bug.
     for project in projects:
       # Make sure pruning never kicks in with shared projects.
       if (not project.use_git_worktrees and
-              len(project.manifest.GetProjectsWithName(project.name)) > 1):
+              len(project.manifest.GetProjectsWithName(project.name, all_manifests=True)) > 1):
         if not opt.quiet:
           print('\r%s: Shared project %s found, disabling pruning.' %
                 (project.relpath, project.name))
@@ -698,7 +739,7 @@ later is required to fix a server side protocol bug.
       t.join()
     pm.end()
 
-  def _ReloadManifest(self, manifest_name, manifest, load_local_manifests=True):
+  def _ReloadManifest(self, manifest_name, manifest):
     """Reload the manfiest from the file specified by the |manifest_name|.
 
     It unloads the manifest if |manifest_name| is None.
@@ -706,17 +747,29 @@ later is required to fix a server side protocol bug.
     Args:
       manifest_name: Manifest file to be reloaded.
       manifest: The manifest to use.
-      load_local_manifests: Whether to load local manifests.
     """
     if manifest_name:
       # Override calls Unload already
-      manifest.Override(manifest_name, load_local_manifests=load_local_manifests)
+      manifest.Override(manifest_name)
     else:
       manifest.Unload()
 
   def UpdateProjectList(self, opt, manifest):
+    """Update the cached projects list for |manifest|
+
+    In a multi-manifest checkout, each manifest has its own project.list.
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      manifest: The manifest to use.
+
+    Returns:
+      0: success
+      1: failure
+    """
     new_project_paths = []
-    for project in self.GetProjects(None, missing_ok=True):
+    for project in self.GetProjects(None, missing_ok=True, manifest=manifest,
+                                    all_manifests=False):
       if project.relpath:
         new_project_paths.append(project.relpath)
     file_name = 'project.list'
@@ -766,7 +819,8 @@ later is required to fix a server side protocol bug.
     new_paths = {}
     new_linkfile_paths = []
     new_copyfile_paths = []
-    for project in self.GetProjects(None, missing_ok=True):
+    for project in self.GetProjects(None, missing_ok=True,
+                                    manifest=manifest, all_manifests=False):
       new_linkfile_paths.extend(x.dest for x in project.linkfiles)
       new_copyfile_paths.extend(x.dest for x in project.copyfiles)
 
@@ -897,8 +951,40 @@ later is required to fix a server side protocol bug.
 
     return manifest_name
 
+  def _UpdateAllManifestProjects(self, opt, mp, manifest_name):
+    """Fetch & update the local manifest project.
+
+    After syncing the manifest project, if the manifest has any sub manifests,
+    those are recursively processed.
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      mp: the manifestProject to query.
+      manifest_name: Manifest file to be reloaded.
+    """
+    if not mp.standalone_manifest_url:
+      self._UpdateManifestProject(opt, mp, manifest_name)
+
+    if mp.manifest.submanifests:
+      for submanifest in mp.manifest.submanifests.values():
+        child = submanifest.repo_client.manifest
+        child.manifestProject.SyncWithPossibleInit(
+            submanifest,
+            current_branch_only=self._GetCurrentBranchOnly(opt, child),
+            verbose=opt.verbose,
+            tags=opt.tags,
+            git_event_log=self.git_event_log,
+        )
+        self._UpdateAllManifestProjects(opt, child.manifestProject, None)
+
   def _UpdateManifestProject(self, opt, mp, manifest_name):
-    """Fetch & update the local manifest project."""
+    """Fetch & update the local manifest project.
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      mp: the manifestProject to query.
+      manifest_name: Manifest file to be reloaded.
+    """
     if not opt.local_only:
       start = time.time()
       success = mp.Sync_NetworkHalf(quiet=opt.quiet, verbose=opt.verbose,
@@ -924,6 +1010,7 @@ later is required to fix a server side protocol bug.
       if not clean:
         sys.exit(1)
       self._ReloadManifest(manifest_name, mp.manifest)
+
       if opt.jobs is None:
         self.jobs = mp.manifest.default.sync_j
 
@@ -948,9 +1035,6 @@ later is required to fix a server side protocol bug.
     if opt.prune is None:
       opt.prune = True
 
-    if self.outer_client.manifest.is_multimanifest and not opt.this_manifest_only and args:
-      self.OptionParser.error('partial syncs must use --this-manifest-only')
-
   def Execute(self, opt, args):
     if opt.jobs:
       self.jobs = opt.jobs
@@ -959,7 +1043,7 @@ later is required to fix a server side protocol bug.
       self.jobs = min(self.jobs, (soft_limit - 5) // 3)
 
     manifest = self.outer_manifest
-    if opt.this_manifest_only or not opt.outer_manifest:
+    if not opt.outer_manifest:
       manifest = self.manifest
 
     if opt.manifest_name:
@@ -994,39 +1078,26 @@ later is required to fix a server side protocol bug.
               'receive updates; run `repo init --repo-rev=stable` to fix.',
               file=sys.stderr)
 
-    mp = manifest.manifestProject
-    is_standalone_manifest = bool(mp.standalone_manifest_url)
-    if not is_standalone_manifest:
-      mp.PreSync()
+    for m in self.ManifestList(opt):
+      mp = m.manifestProject
+      is_standalone_manifest = bool(mp.standalone_manifest_url)
+      if not is_standalone_manifest:
+        mp.PreSync()
 
-    if opt.repo_upgraded:
-      _PostRepoUpgrade(manifest, quiet=opt.quiet)
+      if opt.repo_upgraded:
+        _PostRepoUpgrade(m, quiet=opt.quiet)
 
-    if not opt.mp_update:
+    if opt.mp_update:
+      self._UpdateAllManifestProjects(opt, mp, manifest_name)
+    else:
       print('Skipping update of local manifest project.')
-    elif not is_standalone_manifest:
-      self._UpdateManifestProject(opt, mp, manifest_name)
 
-    load_local_manifests = not manifest.HasLocalManifests
-    use_superproject = git_superproject.UseSuperproject(opt.use_superproject, manifest)
-    if use_superproject and (manifest.IsMirror or manifest.IsArchive):
-      # Don't use superproject, because we have no working tree.
-      use_superproject = False
-      if opt.use_superproject is not None:
-        print('Defaulting to no-use-superproject because there is no working tree.')
-    superproject_logging_data = {
-        'superproject': use_superproject,
-        'haslocalmanifests': bool(manifest.HasLocalManifests),
-        'hassuperprojecttag': bool(manifest.superproject),
-    }
-    if use_superproject:
-      manifest_name = self._UpdateProjectsRevisionId(
-          opt, args, load_local_manifests, superproject_logging_data,
-          manifest) or opt.manifest_name
+    superproject_logging_data = {}
+    self._UpdateProjectsRevisionId(opt, args, superproject_logging_data,
+                                   manifest)
 
     if self.gitc_manifest:
-      gitc_manifest_projects = self.GetProjects(args,
-                                                missing_ok=True)
+      gitc_manifest_projects = self.GetProjects(args, missing_ok=True)
       gitc_projects = []
       opened_projects = []
       for project in gitc_manifest_projects:
@@ -1059,9 +1130,12 @@ later is required to fix a server side protocol bug.
               for path in opened_projects]
       if not args:
         return
+
     all_projects = self.GetProjects(args,
                                     missing_ok=True,
-                                    submodules_ok=opt.fetch_submodules)
+                                    submodules_ok=opt.fetch_submodules,
+                                    manifest=manifest,
+                                    all_manifests=not opt.this_manifest_only)
 
     err_network_sync = False
     err_update_projects = False
@@ -1073,7 +1147,6 @@ later is required to fix a server side protocol bug.
           # Initialize the socket dir once in the parent.
           ssh_proxy.sock()
           all_projects = self._FetchMain(opt, args, all_projects, err_event,
-                                         manifest_name, load_local_manifests,
                                          ssh_proxy, manifest)
 
       if opt.network_only:
@@ -1090,23 +1163,24 @@ later is required to fix a server side protocol bug.
                 file=sys.stderr)
           sys.exit(1)
 
-    if manifest.IsMirror or manifest.IsArchive:
-      # bail out now, we have no working tree
-      return
+    for m in self.ManifestList(opt):
+      if m.IsMirror or m.IsArchive:
+        # bail out now, we have no working tree
+        continue
 
-    if self.UpdateProjectList(opt, manifest):
-      err_event.set()
-      err_update_projects = True
-      if opt.fail_fast:
-        print('\nerror: Local checkouts *not* updated.', file=sys.stderr)
-        sys.exit(1)
+      if self.UpdateProjectList(opt, m):
+        err_event.set()
+        err_update_projects = True
+        if opt.fail_fast:
+          print('\nerror: Local checkouts *not* updated.', file=sys.stderr)
+          sys.exit(1)
 
-    err_update_linkfiles = not self.UpdateCopyLinkfileList(manifest)
-    if err_update_linkfiles:
-      err_event.set()
-      if opt.fail_fast:
-        print('\nerror: Local update copyfile or linkfile failed.', file=sys.stderr)
-        sys.exit(1)
+      err_update_linkfiles = not self.UpdateCopyLinkfileList(m)
+      if err_update_linkfiles:
+        err_event.set()
+        if opt.fail_fast:
+          print('\nerror: Local update copyfile or linkfile failed.', file=sys.stderr)
+          sys.exit(1)
 
     err_results = []
     # NB: We don't exit here because this is the last step.
@@ -1114,10 +1188,14 @@ later is required to fix a server side protocol bug.
     if err_checkout:
       err_event.set()
 
-    # If there's a notice that's supposed to print at the end of the sync, print
-    # it now...
-    if manifest.notice:
-      print(manifest.notice)
+    printed_notices = set()
+    # If there's a notice that's supposed to print at the end of the sync,
+    # print it now...  But avoid printing duplicate messages, and preserve
+    # order.
+    for m in sorted(self.ManifestList(opt), key=lambda x: x.path_prefix):
+      if m.notice and m.notice not in printed_notices:
+        print(m.notice)
+        printed_notices.add(m.notice)
 
     # If we saw an error, exit with code 1 so that other scripts can check.
     if err_event.is_set():