From bdcba7dc36f1c8e6041681eb5b3b5229c93c7c5c Mon Sep 17 00:00:00 2001 From: LaMont Jones Date: Mon, 11 Apr 2022 22:50:11 +0000 Subject: [PATCH] sync: add multi-manifest support With this change, partial syncs (sync with a project list) are again supported. If the updated manifest includes new sub manifests, download them inheriting options from the parent manifestProject. Change-Id: Id952f85df2e26d34e38b251973be26434443ff56 Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/334819 Reviewed-by: Mike Frysinger Tested-by: LaMont Jones --- command.py | 11 +- main.py | 3 +- project.py | 61 +++++++++++ subcmds/init.py | 5 +- subcmds/sync.py | 274 +++++++++++++++++++++++++++++++----------------- 5 files changed, 247 insertions(+), 107 deletions(-) diff --git a/command.py b/command.py index 12fe4172..bd6d0817 100644 --- a/command.py +++ b/command.py @@ -144,11 +144,10 @@ class Command(object): help=f'number of jobs to run in parallel (default: {default})') m = p.add_option_group('Multi-manifest options') - m.add_option('--outer-manifest', action='store_true', + m.add_option('--outer-manifest', action='store_true', default=None, help='operate starting at the outermost manifest') m.add_option('--no-outer-manifest', dest='outer_manifest', - action='store_false', default=None, - help='do not operate on outer manifests') + action='store_false', help='do not operate on outer manifests') m.add_option('--this-manifest-only', action='store_true', default=None, help='only operate on this (sub)manifest') m.add_option('--no-this-manifest-only', '--all-manifests', @@ -186,6 +185,10 @@ class Command(object): """Validate common options.""" opt.quiet = opt.output_mode is False opt.verbose = opt.output_mode is True + if opt.outer_manifest is None: + # By default, treat multi-manifest instances as a single manifest from + # the user's perspective. + opt.outer_manifest = True def ValidateOptions(self, opt, args): """Validate the user options & arguments before executing. @@ -385,7 +388,7 @@ class Command(object): opt: The command options. """ top = self.outer_manifest - if opt.outer_manifest is False or opt.this_manifest_only: + if not opt.outer_manifest or opt.this_manifest_only: top = self.manifest yield top if not opt.this_manifest_only: diff --git a/main.py b/main.py index 34dfb777..c54f9281 100755 --- a/main.py +++ b/main.py @@ -294,8 +294,7 @@ class _Repo(object): cmd.ValidateOptions(copts, cargs) this_manifest_only = copts.this_manifest_only - # If not specified, default to using the outer manifest. - outer_manifest = copts.outer_manifest is not False + outer_manifest = copts.outer_manifest if cmd.MULTI_MANIFEST_SUPPORT or this_manifest_only: result = cmd.Execute(copts, cargs) elif outer_manifest and repo_client.manifest.is_submanifest: diff --git a/project.py b/project.py index faa6b32b..8668bae9 100644 --- a/project.py +++ b/project.py @@ -3467,6 +3467,67 @@ class ManifestProject(MetaProject): """Return the name of the platform.""" return platform.system().lower() + def SyncWithPossibleInit(self, submanifest, verbose=False, + current_branch_only=False, tags='', git_event_log=None): + """Sync a manifestProject, possibly for the first time. + + Call Sync() with arguments from the most recent `repo init`. If this is a + new sub manifest, then inherit options from the parent's manifestProject. + + This is used by subcmds.Sync() to do an initial download of new sub + manifests. + + Args: + submanifest: an XmlSubmanifest, the submanifest to re-sync. + verbose: a boolean, whether to show all output, rather than only errors. + current_branch_only: a boolean, whether to only fetch the current manifest + branch from the server. + tags: a boolean, whether to fetch tags. + git_event_log: an EventLog, for git tracing. + """ + # TODO(lamontjones): when refactoring sync (and init?) consider how to + # better get the init options that we should use when syncing uncovers a new + # submanifest. + git_event_log = git_event_log or EventLog() + spec = submanifest.ToSubmanifestSpec() + # Use the init options from the existing manifestProject, or the parent if + # it doesn't exist. + # + # Today, we only support changing manifest_groups on the sub-manifest, with + # no supported-for-the-user way to change the other arguments from those + # specified by the outermost manifest. + # + # TODO(lamontjones): determine which of these should come from the outermost + # manifest and which should come from the parent manifest. + mp = self if self.Exists else submanifest.parent.manifestProject + return self.Sync( + manifest_url=spec.manifestUrl, + manifest_branch=spec.revision, + standalone_manifest=mp.standalone_manifest_url, + groups=mp.manifest_groups, + platform=mp.manifest_platform, + mirror=mp.mirror, + dissociate=mp.dissociate, + reference=mp.reference, + worktree=mp.use_worktree, + submodules=mp.submodules, + archive=mp.archive, + partial_clone=mp.partial_clone, + clone_filter=mp.clone_filter, + partial_clone_exclude=mp.partial_clone_exclude, + clone_bundle=mp.clone_bundle, + git_lfs=mp.git_lfs, + use_superproject=mp.use_superproject, + verbose=verbose, + current_branch_only=current_branch_only, + tags=tags, + depth=mp.depth, + git_event_log=git_event_log, + manifest_name=spec.manifestName, + this_manifest_only=True, + outer_manifest=False, + ) + def Sync(self, _kwargs_only=(), manifest_url='', manifest_branch=None, standalone_manifest=False, groups='', mirror=False, reference='', dissociate=False, worktree=False, submodules=False, archive=False, diff --git a/subcmds/init.py b/subcmds/init.py index 6e3951c9..cced44d5 100644 --- a/subcmds/init.py +++ b/subcmds/init.py @@ -89,11 +89,10 @@ to update the working directory files. def _Options(self, p, gitc_init=False): Wrapper().InitParser(p, gitc_init=gitc_init) m = p.add_option_group('Multi-manifest') - m.add_option('--outer-manifest', action='store_true', + m.add_option('--outer-manifest', action='store_true', default=True, help='operate starting at the outermost manifest') m.add_option('--no-outer-manifest', dest='outer_manifest', - action='store_false', default=None, - help='do not operate on outer manifests') + action='store_false', help='do not operate on outer manifests') m.add_option('--this-manifest-only', action='store_true', default=None, help='only operate on this (sub)manifest') m.add_option('--no-this-manifest-only', '--all-manifests', diff --git a/subcmds/sync.py b/subcmds/sync.py index 9a66e48b..0abe23d6 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import collections import functools import http.cookiejar as cookielib import io @@ -66,7 +67,7 @@ _ONE_DAY_S = 24 * 60 * 60 class Sync(Command, MirrorSafeCommand): jobs = 1 COMMON = True - MULTI_MANIFEST_SUPPORT = False + MULTI_MANIFEST_SUPPORT = True helpSummary = "Update working tree to the latest revision" helpUsage = """ %prog [...] @@ -295,52 +296,92 @@ later is required to fix a server side protocol bug. """ return git_superproject.UseSuperproject(opt.use_superproject, manifest) or opt.current_branch_only - def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests, superproject_logging_data, manifest): - """Update revisionId of every project with the SHA from superproject. + def _UpdateProjectsRevisionId(self, opt, args, superproject_logging_data, + manifest): + """Update revisionId of projects with the commit hash from the superproject. - This function updates each project's revisionId with SHA from superproject. - It writes the updated manifest into a file and reloads the manifest from it. + This function updates each project's revisionId with the commit hash from + the superproject. It writes the updated manifest into a file and reloads + the manifest from it. When appropriate, sub manifests are also processed. Args: opt: Program options returned from optparse. See _Options(). args: Arguments to pass to GetProjects. See the GetProjects docstring for details. - load_local_manifests: Whether to load local manifests. - superproject_logging_data: A dictionary of superproject data that is to be logged. + superproject_logging_data: A dictionary of superproject data to log. manifest: The manifest to use. - - Returns: - Returns path to the overriding manifest file instead of None. """ - superproject = self.manifest.superproject - superproject.SetQuiet(opt.quiet) - print_messages = git_superproject.PrintMessages(opt.use_superproject, - self.manifest) - superproject.SetPrintMessages(print_messages) + have_superproject = manifest.superproject or any( + m.superproject for m in manifest.all_children) + if not have_superproject: + return + if opt.local_only: - manifest_path = superproject.manifest_path + manifest_path = manifest.superproject.manifest_path if manifest_path: - self._ReloadManifest(manifest_path, manifest, load_local_manifests) - return manifest_path + self._ReloadManifest(manifest_path, manifest) + return all_projects = self.GetProjects(args, missing_ok=True, - submodules_ok=opt.fetch_submodules) - update_result = superproject.UpdateProjectsRevisionId( - all_projects, git_event_log=self.git_event_log) - manifest_path = update_result.manifest_path - superproject_logging_data['updatedrevisionid'] = bool(manifest_path) - if manifest_path: - self._ReloadManifest(manifest_path, manifest, load_local_manifests) + submodules_ok=opt.fetch_submodules, + manifest=manifest, + all_manifests=not opt.this_manifest_only) + + per_manifest = collections.defaultdict(list) + manifest_paths = {} + if opt.this_manifest_only: + per_manifest[manifest.path_prefix] = all_projects else: - if print_messages: - print('warning: Update of revisionId from superproject has failed, ' - 'repo sync will not use superproject to fetch the source. ', - 'Please resync with the --no-use-superproject option to avoid this repo warning.', - file=sys.stderr) - if update_result.fatal and opt.use_superproject is not None: - sys.exit(1) - return manifest_path + for p in all_projects: + per_manifest[p.manifest.path_prefix].append(p) + + superproject_logging_data = {} + need_unload = False + for m in self.ManifestList(opt): + if not m.path_prefix in per_manifest: + continue + use_super = git_superproject.UseSuperproject(opt.use_superproject, m) + if superproject_logging_data: + superproject_logging_data['multimanifest'] = True + superproject_logging_data.update( + superproject=use_super, + haslocalmanifests=bool(m.HasLocalManifests), + hassuperprojecttag=bool(m.superproject), + ) + if use_super and (m.IsMirror or m.IsArchive): + # Don't use superproject, because we have no working tree. + use_super = False + superproject_logging_data['superproject'] = False + superproject_logging_data['noworktree'] = True + if opt.use_superproject is not False: + print(f'{m.path_prefix}: not using superproject because there is no ' + 'working tree.') + + if not use_super: + continue + m.superproject.SetQuiet(opt.quiet) + print_messages = git_superproject.PrintMessages(opt.use_superproject, m) + m.superproject.SetPrintMessages(print_messages) + update_result = m.superproject.UpdateProjectsRevisionId( + per_manifest[m.path_prefix], git_event_log=self.git_event_log) + manifest_path = update_result.manifest_path + superproject_logging_data['updatedrevisionid'] = bool(manifest_path) + if manifest_path: + m.SetManifestOverride(manifest_path) + need_unload = True + else: + if print_messages: + print(f'{m.path_prefix}: warning: Update of revisionId from ' + 'superproject has failed, repo sync will not use superproject ' + 'to fetch the source. ', + 'Please resync with the --no-use-superproject option to avoid ' + 'this repo warning.', + file=sys.stderr) + if update_result.fatal and opt.use_superproject is not None: + sys.exit(1) + if need_unload: + m.outer_client.manifest.Unload() def _FetchProjectList(self, opt, projects): """Main function of the fetch worker. @@ -485,8 +526,8 @@ later is required to fix a server side protocol bug. return (ret, fetched) - def _FetchMain(self, opt, args, all_projects, err_event, manifest_name, - load_local_manifests, ssh_proxy, manifest): + def _FetchMain(self, opt, args, all_projects, err_event, + ssh_proxy, manifest): """The main network fetch loop. Args: @@ -494,8 +535,6 @@ later is required to fix a server side protocol bug. args: Command line args used to filter out projects. all_projects: List of all projects that should be fetched. err_event: Whether an error was hit while processing. - manifest_name: Manifest file to be reloaded. - load_local_manifests: Whether to load local manifests. ssh_proxy: SSH manager for clients & masters. manifest: The manifest to use. @@ -526,10 +565,12 @@ later is required to fix a server side protocol bug. # Iteratively fetch missing and/or nested unregistered submodules previously_missing_set = set() while True: - self._ReloadManifest(manifest_name, self.manifest, load_local_manifests) + self._ReloadManifest(None, manifest) all_projects = self.GetProjects(args, missing_ok=True, - submodules_ok=opt.fetch_submodules) + submodules_ok=opt.fetch_submodules, + manifest=manifest, + all_manifests=not opt.this_manifest_only) missing = [] for project in all_projects: if project.gitdir not in fetched: @@ -624,7 +665,7 @@ later is required to fix a server side protocol bug. for project in projects: # Make sure pruning never kicks in with shared projects. if (not project.use_git_worktrees and - len(project.manifest.GetProjectsWithName(project.name)) > 1): + len(project.manifest.GetProjectsWithName(project.name, all_manifests=True)) > 1): if not opt.quiet: print('\r%s: Shared project %s found, disabling pruning.' % (project.relpath, project.name)) @@ -698,7 +739,7 @@ later is required to fix a server side protocol bug. t.join() pm.end() - def _ReloadManifest(self, manifest_name, manifest, load_local_manifests=True): + def _ReloadManifest(self, manifest_name, manifest): """Reload the manfiest from the file specified by the |manifest_name|. It unloads the manifest if |manifest_name| is None. @@ -706,17 +747,29 @@ later is required to fix a server side protocol bug. Args: manifest_name: Manifest file to be reloaded. manifest: The manifest to use. - load_local_manifests: Whether to load local manifests. """ if manifest_name: # Override calls Unload already - manifest.Override(manifest_name, load_local_manifests=load_local_manifests) + manifest.Override(manifest_name) else: manifest.Unload() def UpdateProjectList(self, opt, manifest): + """Update the cached projects list for |manifest| + + In a multi-manifest checkout, each manifest has its own project.list. + + Args: + opt: Program options returned from optparse. See _Options(). + manifest: The manifest to use. + + Returns: + 0: success + 1: failure + """ new_project_paths = [] - for project in self.GetProjects(None, missing_ok=True): + for project in self.GetProjects(None, missing_ok=True, manifest=manifest, + all_manifests=False): if project.relpath: new_project_paths.append(project.relpath) file_name = 'project.list' @@ -766,7 +819,8 @@ later is required to fix a server side protocol bug. new_paths = {} new_linkfile_paths = [] new_copyfile_paths = [] - for project in self.GetProjects(None, missing_ok=True): + for project in self.GetProjects(None, missing_ok=True, + manifest=manifest, all_manifests=False): new_linkfile_paths.extend(x.dest for x in project.linkfiles) new_copyfile_paths.extend(x.dest for x in project.copyfiles) @@ -897,8 +951,40 @@ later is required to fix a server side protocol bug. return manifest_name + def _UpdateAllManifestProjects(self, opt, mp, manifest_name): + """Fetch & update the local manifest project. + + After syncing the manifest project, if the manifest has any sub manifests, + those are recursively processed. + + Args: + opt: Program options returned from optparse. See _Options(). + mp: the manifestProject to query. + manifest_name: Manifest file to be reloaded. + """ + if not mp.standalone_manifest_url: + self._UpdateManifestProject(opt, mp, manifest_name) + + if mp.manifest.submanifests: + for submanifest in mp.manifest.submanifests.values(): + child = submanifest.repo_client.manifest + child.manifestProject.SyncWithPossibleInit( + submanifest, + current_branch_only=self._GetCurrentBranchOnly(opt, child), + verbose=opt.verbose, + tags=opt.tags, + git_event_log=self.git_event_log, + ) + self._UpdateAllManifestProjects(opt, child.manifestProject, None) + def _UpdateManifestProject(self, opt, mp, manifest_name): - """Fetch & update the local manifest project.""" + """Fetch & update the local manifest project. + + Args: + opt: Program options returned from optparse. See _Options(). + mp: the manifestProject to query. + manifest_name: Manifest file to be reloaded. + """ if not opt.local_only: start = time.time() success = mp.Sync_NetworkHalf(quiet=opt.quiet, verbose=opt.verbose, @@ -924,6 +1010,7 @@ later is required to fix a server side protocol bug. if not clean: sys.exit(1) self._ReloadManifest(manifest_name, mp.manifest) + if opt.jobs is None: self.jobs = mp.manifest.default.sync_j @@ -948,9 +1035,6 @@ later is required to fix a server side protocol bug. if opt.prune is None: opt.prune = True - if self.outer_client.manifest.is_multimanifest and not opt.this_manifest_only and args: - self.OptionParser.error('partial syncs must use --this-manifest-only') - def Execute(self, opt, args): if opt.jobs: self.jobs = opt.jobs @@ -959,7 +1043,7 @@ later is required to fix a server side protocol bug. self.jobs = min(self.jobs, (soft_limit - 5) // 3) manifest = self.outer_manifest - if opt.this_manifest_only or not opt.outer_manifest: + if not opt.outer_manifest: manifest = self.manifest if opt.manifest_name: @@ -994,39 +1078,26 @@ later is required to fix a server side protocol bug. 'receive updates; run `repo init --repo-rev=stable` to fix.', file=sys.stderr) - mp = manifest.manifestProject - is_standalone_manifest = bool(mp.standalone_manifest_url) - if not is_standalone_manifest: - mp.PreSync() + for m in self.ManifestList(opt): + mp = m.manifestProject + is_standalone_manifest = bool(mp.standalone_manifest_url) + if not is_standalone_manifest: + mp.PreSync() - if opt.repo_upgraded: - _PostRepoUpgrade(manifest, quiet=opt.quiet) + if opt.repo_upgraded: + _PostRepoUpgrade(m, quiet=opt.quiet) - if not opt.mp_update: + if opt.mp_update: + self._UpdateAllManifestProjects(opt, mp, manifest_name) + else: print('Skipping update of local manifest project.') - elif not is_standalone_manifest: - self._UpdateManifestProject(opt, mp, manifest_name) - load_local_manifests = not manifest.HasLocalManifests - use_superproject = git_superproject.UseSuperproject(opt.use_superproject, manifest) - if use_superproject and (manifest.IsMirror or manifest.IsArchive): - # Don't use superproject, because we have no working tree. - use_superproject = False - if opt.use_superproject is not None: - print('Defaulting to no-use-superproject because there is no working tree.') - superproject_logging_data = { - 'superproject': use_superproject, - 'haslocalmanifests': bool(manifest.HasLocalManifests), - 'hassuperprojecttag': bool(manifest.superproject), - } - if use_superproject: - manifest_name = self._UpdateProjectsRevisionId( - opt, args, load_local_manifests, superproject_logging_data, - manifest) or opt.manifest_name + superproject_logging_data = {} + self._UpdateProjectsRevisionId(opt, args, superproject_logging_data, + manifest) if self.gitc_manifest: - gitc_manifest_projects = self.GetProjects(args, - missing_ok=True) + gitc_manifest_projects = self.GetProjects(args, missing_ok=True) gitc_projects = [] opened_projects = [] for project in gitc_manifest_projects: @@ -1059,9 +1130,12 @@ later is required to fix a server side protocol bug. for path in opened_projects] if not args: return + all_projects = self.GetProjects(args, missing_ok=True, - submodules_ok=opt.fetch_submodules) + submodules_ok=opt.fetch_submodules, + manifest=manifest, + all_manifests=not opt.this_manifest_only) err_network_sync = False err_update_projects = False @@ -1073,7 +1147,6 @@ later is required to fix a server side protocol bug. # Initialize the socket dir once in the parent. ssh_proxy.sock() all_projects = self._FetchMain(opt, args, all_projects, err_event, - manifest_name, load_local_manifests, ssh_proxy, manifest) if opt.network_only: @@ -1090,23 +1163,24 @@ later is required to fix a server side protocol bug. file=sys.stderr) sys.exit(1) - if manifest.IsMirror or manifest.IsArchive: - # bail out now, we have no working tree - return + for m in self.ManifestList(opt): + if m.IsMirror or m.IsArchive: + # bail out now, we have no working tree + continue - if self.UpdateProjectList(opt, manifest): - err_event.set() - err_update_projects = True - if opt.fail_fast: - print('\nerror: Local checkouts *not* updated.', file=sys.stderr) - sys.exit(1) + if self.UpdateProjectList(opt, m): + err_event.set() + err_update_projects = True + if opt.fail_fast: + print('\nerror: Local checkouts *not* updated.', file=sys.stderr) + sys.exit(1) - err_update_linkfiles = not self.UpdateCopyLinkfileList(manifest) - if err_update_linkfiles: - err_event.set() - if opt.fail_fast: - print('\nerror: Local update copyfile or linkfile failed.', file=sys.stderr) - sys.exit(1) + err_update_linkfiles = not self.UpdateCopyLinkfileList(m) + if err_update_linkfiles: + err_event.set() + if opt.fail_fast: + print('\nerror: Local update copyfile or linkfile failed.', file=sys.stderr) + sys.exit(1) err_results = [] # NB: We don't exit here because this is the last step. @@ -1114,10 +1188,14 @@ later is required to fix a server side protocol bug. if err_checkout: err_event.set() - # If there's a notice that's supposed to print at the end of the sync, print - # it now... - if manifest.notice: - print(manifest.notice) + printed_notices = set() + # If there's a notice that's supposed to print at the end of the sync, + # print it now... But avoid printing duplicate messages, and preserve + # order. + for m in sorted(self.ManifestList(opt), key=lambda x: x.path_prefix): + if m.notice and m.notice not in printed_notices: + print(m.notice) + printed_notices.add(m.notice) # If we saw an error, exit with code 1 so that other scripts can check. if err_event.is_set():