android/git-repo
1
Fork 0
mirror of https://gerrit.googlesource.com/git-repo synced 2024-12-21 07:16:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

sync: refactor use of self.manifest

Browse Source 
We need to iterate over multiple manifests, and generally use the
outer_client.manifest for multi-manifest support.  This refactors the
use of self.manifest into a chosen manifest.

Change-Id: I992f21d610c929675e99555ece9c38df4b635839
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/334699
Tested-by: LaMont Jones <lamontjones@google.com>
Reviewed-by: Mike Frysinger <vapier@google.com>
This commit is contained in:
LaMont Jones 2022-04-07 21:57:06 +00:00
parent 5fa912b0d1
commit a46047a822
2 changed files with 83 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

150
subcmds/sync.py
View File

@ -170,9 +170,9 @@ later is required to fix a server side protocol bug.
PARALLEL_JOBS = 1
def _CommonOptions(self, p):
if self.manifest:
if self.outer_client and self.outer_client.manifest:
try:
self.PARALLEL_JOBS = self.manifest.default.sync_j
self.PARALLEL_JOBS = self.outer_client.manifest.default.sync_j
except ManifestParseError:
pass
super()._CommonOptions(p)
@ -270,25 +270,32 @@ later is required to fix a server side protocol bug.
dest='repo_upgraded', action='store_true',
help=SUPPRESS_HELP)
def _GetBranch(self):
"""Returns the branch name for getting the approved manifest."""
p = self.manifest.manifestProject
b = p.GetBranch(p.CurrentBranch)
def _GetBranch(self, manifest_project):
"""Returns the branch name for getting the approved smartsync manifest.
Args:
manifest_project: the manifestProject to query.
"""
b = manifest_project.GetBranch(manifest_project.CurrentBranch)
branch = b.merge
if branch.startswith(R_HEADS):
branch = branch[len(R_HEADS):]
return branch
def _GetCurrentBranchOnly(self, opt):
def _GetCurrentBranchOnly(self, opt, manifest):
"""Returns whether current-branch or use-superproject options are enabled.
Args:
opt: Program options returned from optparse. See _Options().
manifest: The manifest to use.
Returns:
True if a superproject is requested, otherwise the value of the
current_branch option (True, False or None).
"""
return git_superproject.UseSuperproject(opt.use_superproject, self.manifest) or opt.current_branch_only
return git_superproject.UseSuperproject(opt.use_superproject, manifest) or opt.current_branch_only
def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests, superproject_logging_data):
def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests, superproject_logging_data, manifest):
"""Update revisionId of every project with the SHA from superproject.
This function updates each project's revisionId with SHA from superproject.
@ -300,6 +307,7 @@ later is required to fix a server side protocol bug.
docstring for details.
load_local_manifests: Whether to load local manifests.
superproject_logging_data: A dictionary of superproject data that is to be logged.
manifest: The manifest to use.
Returns:
Returns path to the overriding manifest file instead of None.
@ -312,7 +320,7 @@ later is required to fix a server side protocol bug.
if opt.local_only:
manifest_path = superproject.manifest_path
if manifest_path:
self._ReloadManifest(manifest_path, load_local_manifests)
self._ReloadManifest(manifest_path, manifest, load_local_manifests)
return manifest_path
all_projects = self.GetProjects(args,
@ -323,7 +331,7 @@ later is required to fix a server side protocol bug.
manifest_path = update_result.manifest_path
superproject_logging_data['updatedrevisionid'] = bool(manifest_path)
if manifest_path:
self._ReloadManifest(manifest_path, load_local_manifests)
self._ReloadManifest(manifest_path, manifest, load_local_manifests)
else:
if print_messages:
print('warning: Update of revisionId from superproject has failed, '
@ -366,16 +374,16 @@ later is required to fix a server side protocol bug.
quiet=opt.quiet,
verbose=opt.verbose,
output_redir=buf,
current_branch_only=self._GetCurrentBranchOnly(opt),
current_branch_only=self._GetCurrentBranchOnly(opt, project.manifest),
force_sync=opt.force_sync,
clone_bundle=opt.clone_bundle,
tags=opt.tags, archive=self.manifest.IsArchive,
tags=opt.tags, archive=project.manifest.IsArchive,
optimized_fetch=opt.optimized_fetch,
retry_fetches=opt.retry_fetches,
prune=opt.prune,
ssh_proxy=self.ssh_proxy,
clone_filter=self.manifest.CloneFilter,
partial_clone_exclude=self.manifest.PartialCloneExclude)
clone_filter=project.manifest.CloneFilter,
partial_clone_exclude=project.manifest.PartialCloneExclude)
output = buf.getvalue()
if (opt.verbose or not success) and output:
@ -472,13 +480,13 @@ later is required to fix a server side protocol bug.
pm.end()
self._fetch_times.Save()
if not self.manifest.IsArchive:
if not self.outer_client.manifest.IsArchive:
self._GCProjects(projects, opt, err_event)
return (ret, fetched)
def _FetchMain(self, opt, args, all_projects, err_event, manifest_name,
load_local_manifests, ssh_proxy):
load_local_manifests, ssh_proxy, manifest):
"""The main network fetch loop.
Args:
@ -489,11 +497,12 @@ later is required to fix a server side protocol bug.
manifest_name: Manifest file to be reloaded.
load_local_manifests: Whether to load local manifests.
ssh_proxy: SSH manager for clients & masters.
manifest: The manifest to use.
Returns:
List of all projects that should be checked out.
"""
rp = self.manifest.repoProject
rp = manifest.repoProject
to_fetch = []
now = time.time()
@ -517,7 +526,7 @@ later is required to fix a server side protocol bug.
# Iteratively fetch missing and/or nested unregistered submodules
previously_missing_set = set()
while True:
self._ReloadManifest(manifest_name, load_local_manifests)
self._ReloadManifest(manifest_name, self.manifest, load_local_manifests)
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
@ -552,7 +561,7 @@ later is required to fix a server side protocol bug.
Whether the fetch was successful.
"""
start = time.time()
syncbuf = SyncBuffer(self.manifest.manifestProject.config,
syncbuf = SyncBuffer(project.manifest.manifestProject.config,
detach_head=detach_head)
success = False
try:
@ -689,28 +698,29 @@ later is required to fix a server side protocol bug.
t.join()
pm.end()
def _ReloadManifest(self, manifest_name=None, load_local_manifests=True):
def _ReloadManifest(self, manifest_name, manifest, load_local_manifests=True):
"""Reload the manfiest from the file specified by the |manifest_name|.
It unloads the manifest if |manifest_name| is None.
Args:
manifest_name: Manifest file to be reloaded.
manifest: The manifest to use.
load_local_manifests: Whether to load local manifests.
"""
if manifest_name:
# Override calls Unload already
self.manifest.Override(manifest_name, load_local_manifests=load_local_manifests)
manifest.Override(manifest_name, load_local_manifests=load_local_manifests)
else:
self.manifest.Unload()
manifest.Unload()
def UpdateProjectList(self, opt):
def UpdateProjectList(self, opt, manifest):
new_project_paths = []
for project in self.GetProjects(None, missing_ok=True):
if project.relpath:
new_project_paths.append(project.relpath)
file_name = 'project.list'
file_path = os.path.join(self.manifest.subdir, file_name)
file_path = os.path.join(manifest.subdir, file_name)
old_project_paths = []
if os.path.exists(file_path):
@ -722,16 +732,16 @@ later is required to fix a server side protocol bug.
continue
if path not in new_project_paths:
# If the path has already been deleted, we don't need to do it
gitdir = os.path.join(self.manifest.topdir, path, '.git')
gitdir = os.path.join(manifest.topdir, path, '.git')
if os.path.exists(gitdir):
project = Project(
manifest=self.manifest,
manifest=manifest,
name=path,
remote=RemoteSpec('origin'),
gitdir=gitdir,
objdir=gitdir,
use_git_worktrees=os.path.isfile(gitdir),
worktree=os.path.join(self.manifest.topdir, path),
worktree=os.path.join(manifest.topdir, path),
relpath=path,
revisionExpr='HEAD',
revisionId=None,
@ -747,7 +757,7 @@ later is required to fix a server side protocol bug.
fd.write('\n')
return 0
def UpdateCopyLinkfileList(self):
def UpdateCopyLinkfileList(self, manifest):
"""Save all dests of copyfile and linkfile, and update them if needed.
Returns:
@ -766,7 +776,7 @@ later is required to fix a server side protocol bug.
}
copylinkfile_name = 'copy-link-files.json'
copylinkfile_path = os.path.join(self.manifest.subdir, copylinkfile_name)
copylinkfile_path = os.path.join(manifest.subdir, copylinkfile_name)
old_copylinkfile_paths = {}
if os.path.exists(copylinkfile_path):
@ -797,13 +807,13 @@ later is required to fix a server side protocol bug.
json.dump(new_paths, fp)
return True
def _SmartSyncSetup(self, opt, smart_sync_manifest_path):
if not self.manifest.manifest_server:
def _SmartSyncSetup(self, opt, smart_sync_manifest_path, manifest):
if not manifest.manifest_server:
print('error: cannot smart sync: no manifest server defined in '
'manifest', file=sys.stderr)
sys.exit(1)
manifest_server = self.manifest.manifest_server
manifest_server = manifest.manifest_server
if not opt.quiet:
print('Using manifest server %s' % manifest_server)
@ -844,7 +854,7 @@ later is required to fix a server side protocol bug.
try:
server = xmlrpc.client.Server(manifest_server, transport=transport)
if opt.smart_sync:
branch = self._GetBranch()
branch = self._GetBranch(manifest.manifestProject)
if 'SYNC_TARGET' in os.environ:
target = os.environ['SYNC_TARGET']
@ -870,18 +880,18 @@ later is required to fix a server side protocol bug.
% (smart_sync_manifest_path, e),
file=sys.stderr)
sys.exit(1)
self._ReloadManifest(manifest_name)
self._ReloadManifest(manifest_name, manifest)
else:
print('error: manifest server RPC call failed: %s' %
manifest_str, file=sys.stderr)
sys.exit(1)
except (socket.error, IOError, xmlrpc.client.Fault) as e:
print('error: cannot connect to manifest server %s:\n%s'
% (self.manifest.manifest_server, e), file=sys.stderr)
% (manifest.manifest_server, e), file=sys.stderr)
sys.exit(1)
except xmlrpc.client.ProtocolError as e:
print('error: cannot connect to manifest server %s:\n%d %s'
% (self.manifest.manifest_server, e.errcode, e.errmsg),
% (manifest.manifest_server, e.errcode, e.errmsg),
file=sys.stderr)
sys.exit(1)
@ -892,14 +902,14 @@ later is required to fix a server side protocol bug.
if not opt.local_only:
start = time.time()
success = mp.Sync_NetworkHalf(quiet=opt.quiet, verbose=opt.verbose,
current_branch_only=self._GetCurrentBranchOnly(opt),
current_branch_only=self._GetCurrentBranchOnly(opt, mp.manifest),
force_sync=opt.force_sync,
tags=opt.tags,
optimized_fetch=opt.optimized_fetch,
retry_fetches=opt.retry_fetches,
submodules=self.manifest.HasSubmodules,
clone_filter=self.manifest.CloneFilter,
partial_clone_exclude=self.manifest.PartialCloneExclude)
submodules=mp.manifest.HasSubmodules,
clone_filter=mp.manifest.CloneFilter,
partial_clone_exclude=mp.manifest.PartialCloneExclude)
finish = time.time()
self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK,
start, finish, success)
@ -907,15 +917,15 @@ later is required to fix a server side protocol bug.
if mp.HasChanges:
syncbuf = SyncBuffer(mp.config)
start = time.time()
mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules)
mp.Sync_LocalHalf(syncbuf, submodules=mp.manifest.HasSubmodules)
clean = syncbuf.Finish()
self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL,
start, time.time(), clean)
if not clean:
sys.exit(1)
self._ReloadManifest(manifest_name)
self._ReloadManifest(manifest_name, mp.manifest)
if opt.jobs is None:
self.jobs = self.manifest.default.sync_j
self.jobs = mp.manifest.default.sync_j
def ValidateOptions(self, opt, args):
if opt.force_broken:
@ -938,7 +948,7 @@ later is required to fix a server side protocol bug.
if opt.prune is None:
opt.prune = True
if self.manifest.is_multimanifest and not opt.this_manifest_only and args:
if self.outer_client.manifest.is_multimanifest and not opt.this_manifest_only and args:
self.OptionParser.error('partial syncs must use --this-manifest-only')
def Execute(self, opt, args):
@ -948,18 +958,22 @@ later is required to fix a server side protocol bug.
soft_limit, _ = _rlimit_nofile()
self.jobs = min(self.jobs, (soft_limit - 5) // 3)
manifest = self.outer_manifest
if opt.this_manifest_only or not opt.outer_manifest:
manifest = self.manifest
if opt.manifest_name:
self.manifest.Override(opt.manifest_name)
manifest.Override(opt.manifest_name)
manifest_name = opt.manifest_name
smart_sync_manifest_path = os.path.join(
self.manifest.manifestProject.worktree, 'smart_sync_override.xml')
manifest.manifestProject.worktree, 'smart_sync_override.xml')
if opt.clone_bundle is None:
opt.clone_bundle = self.manifest.CloneBundle
opt.clone_bundle = manifest.CloneBundle
if opt.smart_sync or opt.smart_tag:
manifest_name = self._SmartSyncSetup(opt, smart_sync_manifest_path)
manifest_name = self._SmartSyncSetup(opt, smart_sync_manifest_path, manifest)
else:
if os.path.isfile(smart_sync_manifest_path):
try:
@ -970,7 +984,7 @@ later is required to fix a server side protocol bug.
err_event = multiprocessing.Event()
rp = self.manifest.repoProject
rp = manifest.repoProject
rp.PreSync()
cb = rp.CurrentBranch
if cb:
@ -980,35 +994,35 @@ later is required to fix a server side protocol bug.
'receive updates; run `repo init --repo-rev=stable` to fix.',
file=sys.stderr)
mp = self.manifest.manifestProject
mp = manifest.manifestProject
is_standalone_manifest = bool(mp.standalone_manifest_url)
if not is_standalone_manifest:
mp.PreSync()
if opt.repo_upgraded:
_PostRepoUpgrade(self.manifest, quiet=opt.quiet)
_PostRepoUpgrade(manifest, quiet=opt.quiet)
if not opt.mp_update:
print('Skipping update of local manifest project.')
elif not is_standalone_manifest:
self._UpdateManifestProject(opt, mp, manifest_name)
load_local_manifests = not self.manifest.HasLocalManifests
use_superproject = git_superproject.UseSuperproject(opt.use_superproject,
self.manifest)
if use_superproject and (self.manifest.IsMirror or self.manifest.IsArchive):
load_local_manifests = not manifest.HasLocalManifests
use_superproject = git_superproject.UseSuperproject(opt.use_superproject, manifest)
if use_superproject and (manifest.IsMirror or manifest.IsArchive):
# Don't use superproject, because we have no working tree.
use_superproject = False
if opt.use_superproject is not None:
print('Defaulting to no-use-superproject because there is no working tree.')
superproject_logging_data = {
'superproject': use_superproject,
'haslocalmanifests': bool(self.manifest.HasLocalManifests),
'hassuperprojecttag': bool(self.manifest.superproject),
'haslocalmanifests': bool(manifest.HasLocalManifests),
'hassuperprojecttag': bool(manifest.superproject),
}
if use_superproject:
manifest_name = self._UpdateProjectsRevisionId(
opt, args, load_local_manifests, superproject_logging_data) or opt.manifest_name
opt, args, load_local_manifests, superproject_logging_data,
manifest) or opt.manifest_name
if self.gitc_manifest:
gitc_manifest_projects = self.GetProjects(args,
@ -1031,7 +1045,7 @@ later is required to fix a server side protocol bug.
if manifest_name:
manifest.Override(manifest_name)
else:
manifest.Override(self.manifest.manifestFile)
manifest.Override(manifest.manifestFile)
gitc_utils.generate_gitc_manifest(self.gitc_manifest,
manifest,
gitc_projects)
@ -1041,7 +1055,7 @@ later is required to fix a server side protocol bug.
# generate a new args list to represent the opened projects.
# TODO: make this more reliable -- if there's a project name/path overlap,
# this may choose the wrong project.
args = [os.path.relpath(self.manifest.paths[path].worktree, os.getcwd())
args = [os.path.relpath(manifest.paths[path].worktree, os.getcwd())
for path in opened_projects]
if not args:
return
@ -1052,7 +1066,7 @@ later is required to fix a server side protocol bug.
err_network_sync = False
err_update_projects = False
self._fetch_times = _FetchTimes(self.manifest)
self._fetch_times = _FetchTimes(manifest)
if not opt.local_only:
with multiprocessing.Manager() as manager:
with ssh.ProxyManager(manager) as ssh_proxy:
@ -1060,7 +1074,7 @@ later is required to fix a server side protocol bug.
ssh_proxy.sock()
all_projects = self._FetchMain(opt, args, all_projects, err_event,
manifest_name, load_local_manifests,
ssh_proxy)
ssh_proxy, manifest)
if opt.network_only:
return
@ -1076,18 +1090,18 @@ later is required to fix a server side protocol bug.
file=sys.stderr)
sys.exit(1)
if self.manifest.IsMirror or self.manifest.IsArchive:
if manifest.IsMirror or manifest.IsArchive:
# bail out now, we have no working tree
return
if self.UpdateProjectList(opt):
if self.UpdateProjectList(opt, manifest):
err_event.set()
err_update_projects = True
if opt.fail_fast:
print('\nerror: Local checkouts *not* updated.', file=sys.stderr)
sys.exit(1)
err_update_linkfiles = not self.UpdateCopyLinkfileList()
err_update_linkfiles = not self.UpdateCopyLinkfileList(manifest)
if err_update_linkfiles:
err_event.set()
if opt.fail_fast:
@ -1102,8 +1116,8 @@ later is required to fix a server side protocol bug.
# If there's a notice that's supposed to print at the end of the sync, print
# it now...
if self.manifest.notice:
print(self.manifest.notice)
if manifest.notice:
print(manifest.notice)
# If we saw an error, exit with code 1 so that other scripts can check.
if err_event.is_set():

2
tests/test_subcmds_sync.py
View File

@ -42,4 +42,4 @@ def test_get_current_branch_only(use_superproject, cli_args, result):
opts, _ = cmd.OptionParser.parse_args(cli_args)
with mock.patch('git_superproject.UseSuperproject', return_value=use_superproject):
assert cmd._GetCurrentBranchOnly(opts) == result
assert cmd._GetCurrentBranchOnly(opts, cmd.manifest) == result