From a46047a822ad13738af9b17a524a0bd483b98033 Mon Sep 17 00:00:00 2001 From: LaMont Jones Date: Thu, 7 Apr 2022 21:57:06 +0000 Subject: [PATCH] sync: refactor use of self.manifest We need to iterate over multiple manifests, and generally use the outer_client.manifest for multi-manifest support. This refactors the use of self.manifest into a chosen manifest. Change-Id: I992f21d610c929675e99555ece9c38df4b635839 Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/334699 Tested-by: LaMont Jones Reviewed-by: Mike Frysinger --- subcmds/sync.py | 150 ++++++++++++++++++++----------------- tests/test_subcmds_sync.py | 2 +- 2 files changed, 83 insertions(+), 69 deletions(-) diff --git a/subcmds/sync.py b/subcmds/sync.py index 3451ab6b..9a66e48b 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -170,9 +170,9 @@ later is required to fix a server side protocol bug. PARALLEL_JOBS = 1 def _CommonOptions(self, p): - if self.manifest: + if self.outer_client and self.outer_client.manifest: try: - self.PARALLEL_JOBS = self.manifest.default.sync_j + self.PARALLEL_JOBS = self.outer_client.manifest.default.sync_j except ManifestParseError: pass super()._CommonOptions(p) @@ -270,25 +270,32 @@ later is required to fix a server side protocol bug. dest='repo_upgraded', action='store_true', help=SUPPRESS_HELP) - def _GetBranch(self): - """Returns the branch name for getting the approved manifest.""" - p = self.manifest.manifestProject - b = p.GetBranch(p.CurrentBranch) + def _GetBranch(self, manifest_project): + """Returns the branch name for getting the approved smartsync manifest. + + Args: + manifest_project: the manifestProject to query. + """ + b = manifest_project.GetBranch(manifest_project.CurrentBranch) branch = b.merge if branch.startswith(R_HEADS): branch = branch[len(R_HEADS):] return branch - def _GetCurrentBranchOnly(self, opt): + def _GetCurrentBranchOnly(self, opt, manifest): """Returns whether current-branch or use-superproject options are enabled. + Args: + opt: Program options returned from optparse. See _Options(). + manifest: The manifest to use. + Returns: True if a superproject is requested, otherwise the value of the current_branch option (True, False or None). """ - return git_superproject.UseSuperproject(opt.use_superproject, self.manifest) or opt.current_branch_only + return git_superproject.UseSuperproject(opt.use_superproject, manifest) or opt.current_branch_only - def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests, superproject_logging_data): + def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests, superproject_logging_data, manifest): """Update revisionId of every project with the SHA from superproject. This function updates each project's revisionId with SHA from superproject. @@ -300,6 +307,7 @@ later is required to fix a server side protocol bug. docstring for details. load_local_manifests: Whether to load local manifests. superproject_logging_data: A dictionary of superproject data that is to be logged. + manifest: The manifest to use. Returns: Returns path to the overriding manifest file instead of None. @@ -312,7 +320,7 @@ later is required to fix a server side protocol bug. if opt.local_only: manifest_path = superproject.manifest_path if manifest_path: - self._ReloadManifest(manifest_path, load_local_manifests) + self._ReloadManifest(manifest_path, manifest, load_local_manifests) return manifest_path all_projects = self.GetProjects(args, @@ -323,7 +331,7 @@ later is required to fix a server side protocol bug. manifest_path = update_result.manifest_path superproject_logging_data['updatedrevisionid'] = bool(manifest_path) if manifest_path: - self._ReloadManifest(manifest_path, load_local_manifests) + self._ReloadManifest(manifest_path, manifest, load_local_manifests) else: if print_messages: print('warning: Update of revisionId from superproject has failed, ' @@ -366,16 +374,16 @@ later is required to fix a server side protocol bug. quiet=opt.quiet, verbose=opt.verbose, output_redir=buf, - current_branch_only=self._GetCurrentBranchOnly(opt), + current_branch_only=self._GetCurrentBranchOnly(opt, project.manifest), force_sync=opt.force_sync, clone_bundle=opt.clone_bundle, - tags=opt.tags, archive=self.manifest.IsArchive, + tags=opt.tags, archive=project.manifest.IsArchive, optimized_fetch=opt.optimized_fetch, retry_fetches=opt.retry_fetches, prune=opt.prune, ssh_proxy=self.ssh_proxy, - clone_filter=self.manifest.CloneFilter, - partial_clone_exclude=self.manifest.PartialCloneExclude) + clone_filter=project.manifest.CloneFilter, + partial_clone_exclude=project.manifest.PartialCloneExclude) output = buf.getvalue() if (opt.verbose or not success) and output: @@ -472,13 +480,13 @@ later is required to fix a server side protocol bug. pm.end() self._fetch_times.Save() - if not self.manifest.IsArchive: + if not self.outer_client.manifest.IsArchive: self._GCProjects(projects, opt, err_event) return (ret, fetched) def _FetchMain(self, opt, args, all_projects, err_event, manifest_name, - load_local_manifests, ssh_proxy): + load_local_manifests, ssh_proxy, manifest): """The main network fetch loop. Args: @@ -489,11 +497,12 @@ later is required to fix a server side protocol bug. manifest_name: Manifest file to be reloaded. load_local_manifests: Whether to load local manifests. ssh_proxy: SSH manager for clients & masters. + manifest: The manifest to use. Returns: List of all projects that should be checked out. """ - rp = self.manifest.repoProject + rp = manifest.repoProject to_fetch = [] now = time.time() @@ -517,7 +526,7 @@ later is required to fix a server side protocol bug. # Iteratively fetch missing and/or nested unregistered submodules previously_missing_set = set() while True: - self._ReloadManifest(manifest_name, load_local_manifests) + self._ReloadManifest(manifest_name, self.manifest, load_local_manifests) all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=opt.fetch_submodules) @@ -552,7 +561,7 @@ later is required to fix a server side protocol bug. Whether the fetch was successful. """ start = time.time() - syncbuf = SyncBuffer(self.manifest.manifestProject.config, + syncbuf = SyncBuffer(project.manifest.manifestProject.config, detach_head=detach_head) success = False try: @@ -689,28 +698,29 @@ later is required to fix a server side protocol bug. t.join() pm.end() - def _ReloadManifest(self, manifest_name=None, load_local_manifests=True): + def _ReloadManifest(self, manifest_name, manifest, load_local_manifests=True): """Reload the manfiest from the file specified by the |manifest_name|. It unloads the manifest if |manifest_name| is None. Args: manifest_name: Manifest file to be reloaded. + manifest: The manifest to use. load_local_manifests: Whether to load local manifests. """ if manifest_name: # Override calls Unload already - self.manifest.Override(manifest_name, load_local_manifests=load_local_manifests) + manifest.Override(manifest_name, load_local_manifests=load_local_manifests) else: - self.manifest.Unload() + manifest.Unload() - def UpdateProjectList(self, opt): + def UpdateProjectList(self, opt, manifest): new_project_paths = [] for project in self.GetProjects(None, missing_ok=True): if project.relpath: new_project_paths.append(project.relpath) file_name = 'project.list' - file_path = os.path.join(self.manifest.subdir, file_name) + file_path = os.path.join(manifest.subdir, file_name) old_project_paths = [] if os.path.exists(file_path): @@ -722,16 +732,16 @@ later is required to fix a server side protocol bug. continue if path not in new_project_paths: # If the path has already been deleted, we don't need to do it - gitdir = os.path.join(self.manifest.topdir, path, '.git') + gitdir = os.path.join(manifest.topdir, path, '.git') if os.path.exists(gitdir): project = Project( - manifest=self.manifest, + manifest=manifest, name=path, remote=RemoteSpec('origin'), gitdir=gitdir, objdir=gitdir, use_git_worktrees=os.path.isfile(gitdir), - worktree=os.path.join(self.manifest.topdir, path), + worktree=os.path.join(manifest.topdir, path), relpath=path, revisionExpr='HEAD', revisionId=None, @@ -747,7 +757,7 @@ later is required to fix a server side protocol bug. fd.write('\n') return 0 - def UpdateCopyLinkfileList(self): + def UpdateCopyLinkfileList(self, manifest): """Save all dests of copyfile and linkfile, and update them if needed. Returns: @@ -766,7 +776,7 @@ later is required to fix a server side protocol bug. } copylinkfile_name = 'copy-link-files.json' - copylinkfile_path = os.path.join(self.manifest.subdir, copylinkfile_name) + copylinkfile_path = os.path.join(manifest.subdir, copylinkfile_name) old_copylinkfile_paths = {} if os.path.exists(copylinkfile_path): @@ -797,13 +807,13 @@ later is required to fix a server side protocol bug. json.dump(new_paths, fp) return True - def _SmartSyncSetup(self, opt, smart_sync_manifest_path): - if not self.manifest.manifest_server: + def _SmartSyncSetup(self, opt, smart_sync_manifest_path, manifest): + if not manifest.manifest_server: print('error: cannot smart sync: no manifest server defined in ' 'manifest', file=sys.stderr) sys.exit(1) - manifest_server = self.manifest.manifest_server + manifest_server = manifest.manifest_server if not opt.quiet: print('Using manifest server %s' % manifest_server) @@ -844,7 +854,7 @@ later is required to fix a server side protocol bug. try: server = xmlrpc.client.Server(manifest_server, transport=transport) if opt.smart_sync: - branch = self._GetBranch() + branch = self._GetBranch(manifest.manifestProject) if 'SYNC_TARGET' in os.environ: target = os.environ['SYNC_TARGET'] @@ -870,18 +880,18 @@ later is required to fix a server side protocol bug. % (smart_sync_manifest_path, e), file=sys.stderr) sys.exit(1) - self._ReloadManifest(manifest_name) + self._ReloadManifest(manifest_name, manifest) else: print('error: manifest server RPC call failed: %s' % manifest_str, file=sys.stderr) sys.exit(1) except (socket.error, IOError, xmlrpc.client.Fault) as e: print('error: cannot connect to manifest server %s:\n%s' - % (self.manifest.manifest_server, e), file=sys.stderr) + % (manifest.manifest_server, e), file=sys.stderr) sys.exit(1) except xmlrpc.client.ProtocolError as e: print('error: cannot connect to manifest server %s:\n%d %s' - % (self.manifest.manifest_server, e.errcode, e.errmsg), + % (manifest.manifest_server, e.errcode, e.errmsg), file=sys.stderr) sys.exit(1) @@ -892,14 +902,14 @@ later is required to fix a server side protocol bug. if not opt.local_only: start = time.time() success = mp.Sync_NetworkHalf(quiet=opt.quiet, verbose=opt.verbose, - current_branch_only=self._GetCurrentBranchOnly(opt), + current_branch_only=self._GetCurrentBranchOnly(opt, mp.manifest), force_sync=opt.force_sync, tags=opt.tags, optimized_fetch=opt.optimized_fetch, retry_fetches=opt.retry_fetches, - submodules=self.manifest.HasSubmodules, - clone_filter=self.manifest.CloneFilter, - partial_clone_exclude=self.manifest.PartialCloneExclude) + submodules=mp.manifest.HasSubmodules, + clone_filter=mp.manifest.CloneFilter, + partial_clone_exclude=mp.manifest.PartialCloneExclude) finish = time.time() self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK, start, finish, success) @@ -907,15 +917,15 @@ later is required to fix a server side protocol bug. if mp.HasChanges: syncbuf = SyncBuffer(mp.config) start = time.time() - mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules) + mp.Sync_LocalHalf(syncbuf, submodules=mp.manifest.HasSubmodules) clean = syncbuf.Finish() self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL, start, time.time(), clean) if not clean: sys.exit(1) - self._ReloadManifest(manifest_name) + self._ReloadManifest(manifest_name, mp.manifest) if opt.jobs is None: - self.jobs = self.manifest.default.sync_j + self.jobs = mp.manifest.default.sync_j def ValidateOptions(self, opt, args): if opt.force_broken: @@ -938,7 +948,7 @@ later is required to fix a server side protocol bug. if opt.prune is None: opt.prune = True - if self.manifest.is_multimanifest and not opt.this_manifest_only and args: + if self.outer_client.manifest.is_multimanifest and not opt.this_manifest_only and args: self.OptionParser.error('partial syncs must use --this-manifest-only') def Execute(self, opt, args): @@ -948,18 +958,22 @@ later is required to fix a server side protocol bug. soft_limit, _ = _rlimit_nofile() self.jobs = min(self.jobs, (soft_limit - 5) // 3) + manifest = self.outer_manifest + if opt.this_manifest_only or not opt.outer_manifest: + manifest = self.manifest + if opt.manifest_name: - self.manifest.Override(opt.manifest_name) + manifest.Override(opt.manifest_name) manifest_name = opt.manifest_name smart_sync_manifest_path = os.path.join( - self.manifest.manifestProject.worktree, 'smart_sync_override.xml') + manifest.manifestProject.worktree, 'smart_sync_override.xml') if opt.clone_bundle is None: - opt.clone_bundle = self.manifest.CloneBundle + opt.clone_bundle = manifest.CloneBundle if opt.smart_sync or opt.smart_tag: - manifest_name = self._SmartSyncSetup(opt, smart_sync_manifest_path) + manifest_name = self._SmartSyncSetup(opt, smart_sync_manifest_path, manifest) else: if os.path.isfile(smart_sync_manifest_path): try: @@ -970,7 +984,7 @@ later is required to fix a server side protocol bug. err_event = multiprocessing.Event() - rp = self.manifest.repoProject + rp = manifest.repoProject rp.PreSync() cb = rp.CurrentBranch if cb: @@ -980,35 +994,35 @@ later is required to fix a server side protocol bug. 'receive updates; run `repo init --repo-rev=stable` to fix.', file=sys.stderr) - mp = self.manifest.manifestProject + mp = manifest.manifestProject is_standalone_manifest = bool(mp.standalone_manifest_url) if not is_standalone_manifest: mp.PreSync() if opt.repo_upgraded: - _PostRepoUpgrade(self.manifest, quiet=opt.quiet) + _PostRepoUpgrade(manifest, quiet=opt.quiet) if not opt.mp_update: print('Skipping update of local manifest project.') elif not is_standalone_manifest: self._UpdateManifestProject(opt, mp, manifest_name) - load_local_manifests = not self.manifest.HasLocalManifests - use_superproject = git_superproject.UseSuperproject(opt.use_superproject, - self.manifest) - if use_superproject and (self.manifest.IsMirror or self.manifest.IsArchive): + load_local_manifests = not manifest.HasLocalManifests + use_superproject = git_superproject.UseSuperproject(opt.use_superproject, manifest) + if use_superproject and (manifest.IsMirror or manifest.IsArchive): # Don't use superproject, because we have no working tree. use_superproject = False if opt.use_superproject is not None: print('Defaulting to no-use-superproject because there is no working tree.') superproject_logging_data = { 'superproject': use_superproject, - 'haslocalmanifests': bool(self.manifest.HasLocalManifests), - 'hassuperprojecttag': bool(self.manifest.superproject), + 'haslocalmanifests': bool(manifest.HasLocalManifests), + 'hassuperprojecttag': bool(manifest.superproject), } if use_superproject: manifest_name = self._UpdateProjectsRevisionId( - opt, args, load_local_manifests, superproject_logging_data) or opt.manifest_name + opt, args, load_local_manifests, superproject_logging_data, + manifest) or opt.manifest_name if self.gitc_manifest: gitc_manifest_projects = self.GetProjects(args, @@ -1031,7 +1045,7 @@ later is required to fix a server side protocol bug. if manifest_name: manifest.Override(manifest_name) else: - manifest.Override(self.manifest.manifestFile) + manifest.Override(manifest.manifestFile) gitc_utils.generate_gitc_manifest(self.gitc_manifest, manifest, gitc_projects) @@ -1041,7 +1055,7 @@ later is required to fix a server side protocol bug. # generate a new args list to represent the opened projects. # TODO: make this more reliable -- if there's a project name/path overlap, # this may choose the wrong project. - args = [os.path.relpath(self.manifest.paths[path].worktree, os.getcwd()) + args = [os.path.relpath(manifest.paths[path].worktree, os.getcwd()) for path in opened_projects] if not args: return @@ -1052,7 +1066,7 @@ later is required to fix a server side protocol bug. err_network_sync = False err_update_projects = False - self._fetch_times = _FetchTimes(self.manifest) + self._fetch_times = _FetchTimes(manifest) if not opt.local_only: with multiprocessing.Manager() as manager: with ssh.ProxyManager(manager) as ssh_proxy: @@ -1060,7 +1074,7 @@ later is required to fix a server side protocol bug. ssh_proxy.sock() all_projects = self._FetchMain(opt, args, all_projects, err_event, manifest_name, load_local_manifests, - ssh_proxy) + ssh_proxy, manifest) if opt.network_only: return @@ -1076,18 +1090,18 @@ later is required to fix a server side protocol bug. file=sys.stderr) sys.exit(1) - if self.manifest.IsMirror or self.manifest.IsArchive: + if manifest.IsMirror or manifest.IsArchive: # bail out now, we have no working tree return - if self.UpdateProjectList(opt): + if self.UpdateProjectList(opt, manifest): err_event.set() err_update_projects = True if opt.fail_fast: print('\nerror: Local checkouts *not* updated.', file=sys.stderr) sys.exit(1) - err_update_linkfiles = not self.UpdateCopyLinkfileList() + err_update_linkfiles = not self.UpdateCopyLinkfileList(manifest) if err_update_linkfiles: err_event.set() if opt.fail_fast: @@ -1102,8 +1116,8 @@ later is required to fix a server side protocol bug. # If there's a notice that's supposed to print at the end of the sync, print # it now... - if self.manifest.notice: - print(self.manifest.notice) + if manifest.notice: + print(manifest.notice) # If we saw an error, exit with code 1 so that other scripts can check. if err_event.is_set(): diff --git a/tests/test_subcmds_sync.py b/tests/test_subcmds_sync.py index c1d1758e..aad713f2 100644 --- a/tests/test_subcmds_sync.py +++ b/tests/test_subcmds_sync.py @@ -42,4 +42,4 @@ def test_get_current_branch_only(use_superproject, cli_args, result): opts, _ = cmd.OptionParser.parse_args(cli_args) with mock.patch('git_superproject.UseSuperproject', return_value=use_superproject): - assert cmd._GetCurrentBranchOnly(opts) == result + assert cmd._GetCurrentBranchOnly(opts, cmd.manifest) == result