android/git-repo
1
Fork 0
mirror of https://gerrit.googlesource.com/git-repo synced 2024-12-21 07:16:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

sync: add multi-manifest support

Browse Source 
With this change, partial syncs (sync with a project list) are again
supported.

If the updated manifest includes new sub manifests, download them
inheriting options from the parent manifestProject.

Change-Id: Id952f85df2e26d34e38b251973be26434443ff56
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/334819
Reviewed-by: Mike Frysinger <vapier@google.com>
Tested-by: LaMont Jones <lamontjones@google.com>
This commit is contained in:
LaMont Jones 2022-04-11 22:50:11 +00:00
parent 1d00a7e2ae
commit bdcba7dc36
5 changed files with 247 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
command.py
View File

@ -144,11 +144,10 @@ class Command(object):
help=f'number of jobs to run in parallel (default: {default})')
m = p.add_option_group('Multi-manifest options')
m.add_option('--outer-manifest', action='store_true',
m.add_option('--outer-manifest', action='store_true', default=None,
help='operate starting at the outermost manifest')
m.add_option('--no-outer-manifest', dest='outer_manifest',
action='store_false', default=None,
help='do not operate on outer manifests')
action='store_false', help='do not operate on outer manifests')
m.add_option('--this-manifest-only', action='store_true', default=None,
help='only operate on this (sub)manifest')
m.add_option('--no-this-manifest-only', '--all-manifests',
@ -186,6 +185,10 @@ class Command(object):
"""Validate common options."""
opt.quiet = opt.output_mode is False
opt.verbose = opt.output_mode is True
if opt.outer_manifest is None:
# By default, treat multi-manifest instances as a single manifest from
# the user's perspective.
opt.outer_manifest = True
def ValidateOptions(self, opt, args):
"""Validate the user options & arguments before executing.
@ -385,7 +388,7 @@ class Command(object):
opt: The command options.
"""
top = self.outer_manifest
if opt.outer_manifest is False or opt.this_manifest_only:
if not opt.outer_manifest or opt.this_manifest_only:
top = self.manifest
yield top
if not opt.this_manifest_only:

3
main.py
View File

@ -294,8 +294,7 @@ class _Repo(object):
cmd.ValidateOptions(copts, cargs)
this_manifest_only = copts.this_manifest_only
# If not specified, default to using the outer manifest.
outer_manifest = copts.outer_manifest is not False
outer_manifest = copts.outer_manifest
if cmd.MULTI_MANIFEST_SUPPORT or this_manifest_only:
result = cmd.Execute(copts, cargs)
elif outer_manifest and repo_client.manifest.is_submanifest:

61
project.py
View File

@ -3467,6 +3467,67 @@ class ManifestProject(MetaProject):
"""Return the name of the platform."""
return platform.system().lower()
def SyncWithPossibleInit(self, submanifest, verbose=False,
current_branch_only=False, tags='', git_event_log=None):
"""Sync a manifestProject, possibly for the first time.
Call Sync() with arguments from the most recent `repo init`. If this is a
new sub manifest, then inherit options from the parent's manifestProject.
This is used by subcmds.Sync() to do an initial download of new sub
manifests.
Args:
submanifest: an XmlSubmanifest, the submanifest to re-sync.
verbose: a boolean, whether to show all output, rather than only errors.
current_branch_only: a boolean, whether to only fetch the current manifest
branch from the server.
tags: a boolean, whether to fetch tags.
git_event_log: an EventLog, for git tracing.
"""
# TODO(lamontjones): when refactoring sync (and init?) consider how to
# better get the init options that we should use when syncing uncovers a new
# submanifest.
git_event_log = git_event_log or EventLog()
spec = submanifest.ToSubmanifestSpec()
# Use the init options from the existing manifestProject, or the parent if
# it doesn't exist.
#
# Today, we only support changing manifest_groups on the sub-manifest, with
# no supported-for-the-user way to change the other arguments from those
# specified by the outermost manifest.
#
# TODO(lamontjones): determine which of these should come from the outermost
# manifest and which should come from the parent manifest.
mp = self if self.Exists else submanifest.parent.manifestProject
return self.Sync(
manifest_url=spec.manifestUrl,
manifest_branch=spec.revision,
standalone_manifest=mp.standalone_manifest_url,
groups=mp.manifest_groups,
platform=mp.manifest_platform,
mirror=mp.mirror,
dissociate=mp.dissociate,
reference=mp.reference,
worktree=mp.use_worktree,
submodules=mp.submodules,
archive=mp.archive,
partial_clone=mp.partial_clone,
clone_filter=mp.clone_filter,
partial_clone_exclude=mp.partial_clone_exclude,
clone_bundle=mp.clone_bundle,
git_lfs=mp.git_lfs,
use_superproject=mp.use_superproject,
verbose=verbose,
current_branch_only=current_branch_only,
tags=tags,
depth=mp.depth,
git_event_log=git_event_log,
manifest_name=spec.manifestName,
this_manifest_only=True,
outer_manifest=False,
)
def Sync(self, _kwargs_only=(), manifest_url='', manifest_branch=None,
standalone_manifest=False, groups='', mirror=False, reference='',
dissociate=False, worktree=False, submodules=False, archive=False,

5
subcmds/init.py
View File

@ -89,11 +89,10 @@ to update the working directory files.
def _Options(self, p, gitc_init=False):
Wrapper().InitParser(p, gitc_init=gitc_init)
m = p.add_option_group('Multi-manifest')
m.add_option('--outer-manifest', action='store_true',
m.add_option('--outer-manifest', action='store_true', default=True,
help='operate starting at the outermost manifest')
m.add_option('--no-outer-manifest', dest='outer_manifest',
action='store_false', default=None,
help='do not operate on outer manifests')
action='store_false', help='do not operate on outer manifests')
m.add_option('--this-manifest-only', action='store_true', default=None,
help='only operate on this (sub)manifest')
m.add_option('--no-this-manifest-only', '--all-manifests',

230
subcmds/sync.py
View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
import functools
import http.cookiejar as cookielib
import io
@ -66,7 +67,7 @@ _ONE_DAY_S = 24 * 60 * 60
class Sync(Command, MirrorSafeCommand):
jobs = 1
COMMON = True
MULTI_MANIFEST_SUPPORT = False
MULTI_MANIFEST_SUPPORT = True
helpSummary = "Update working tree to the latest revision"
helpUsage = """
%prog [<project>...]
@ -295,52 +296,92 @@ later is required to fix a server side protocol bug.
"""
return git_superproject.UseSuperproject(opt.use_superproject, manifest) or opt.current_branch_only
def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests, superproject_logging_data, manifest):
"""Update revisionId of every project with the SHA from superproject.
def _UpdateProjectsRevisionId(self, opt, args, superproject_logging_data,
manifest):
"""Update revisionId of projects with the commit hash from the superproject.
This function updates each project's revisionId with SHA from superproject.
It writes the updated manifest into a file and reloads the manifest from it.
This function updates each project's revisionId with the commit hash from
the superproject. It writes the updated manifest into a file and reloads
the manifest from it. When appropriate, sub manifests are also processed.
Args:
opt: Program options returned from optparse. See _Options().
args: Arguments to pass to GetProjects. See the GetProjects
docstring for details.
load_local_manifests: Whether to load local manifests.
superproject_logging_data: A dictionary of superproject data that is to be logged.
superproject_logging_data: A dictionary of superproject data to log.
manifest: The manifest to use.
Returns:
Returns path to the overriding manifest file instead of None.
"""
superproject = self.manifest.superproject
superproject.SetQuiet(opt.quiet)
print_messages = git_superproject.PrintMessages(opt.use_superproject,
self.manifest)
superproject.SetPrintMessages(print_messages)
have_superproject = manifest.superproject or any(
m.superproject for m in manifest.all_children)
if not have_superproject:
return
if opt.local_only:
manifest_path = superproject.manifest_path
manifest_path = manifest.superproject.manifest_path
if manifest_path:
self._ReloadManifest(manifest_path, manifest, load_local_manifests)
return manifest_path
self._ReloadManifest(manifest_path, manifest)
return
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
update_result = superproject.UpdateProjectsRevisionId(
all_projects, git_event_log=self.git_event_log)
submodules_ok=opt.fetch_submodules,
manifest=manifest,
all_manifests=not opt.this_manifest_only)
per_manifest = collections.defaultdict(list)
manifest_paths = {}
if opt.this_manifest_only:
per_manifest[manifest.path_prefix] = all_projects
else:
for p in all_projects:
per_manifest[p.manifest.path_prefix].append(p)
superproject_logging_data = {}
need_unload = False
for m in self.ManifestList(opt):
if not m.path_prefix in per_manifest:
continue
use_super = git_superproject.UseSuperproject(opt.use_superproject, m)
if superproject_logging_data:
superproject_logging_data['multimanifest'] = True
superproject_logging_data.update(
superproject=use_super,
haslocalmanifests=bool(m.HasLocalManifests),
hassuperprojecttag=bool(m.superproject),
)
if use_super and (m.IsMirror or m.IsArchive):
# Don't use superproject, because we have no working tree.
use_super = False
superproject_logging_data['superproject'] = False
superproject_logging_data['noworktree'] = True
if opt.use_superproject is not False:
print(f'{m.path_prefix}: not using superproject because there is no '
'working tree.')
if not use_super:
continue
m.superproject.SetQuiet(opt.quiet)
print_messages = git_superproject.PrintMessages(opt.use_superproject, m)
m.superproject.SetPrintMessages(print_messages)
update_result = m.superproject.UpdateProjectsRevisionId(
per_manifest[m.path_prefix], git_event_log=self.git_event_log)
manifest_path = update_result.manifest_path
superproject_logging_data['updatedrevisionid'] = bool(manifest_path)
if manifest_path:
self._ReloadManifest(manifest_path, manifest, load_local_manifests)
m.SetManifestOverride(manifest_path)
need_unload = True
else:
if print_messages:
print('warning: Update of revisionId from superproject has failed, '
'repo sync will not use superproject to fetch the source. ',
'Please resync with the --no-use-superproject option to avoid this repo warning.',
print(f'{m.path_prefix}: warning: Update of revisionId from '
'superproject has failed, repo sync will not use superproject '
'to fetch the source. ',
'Please resync with the --no-use-superproject option to avoid '
'this repo warning.',
file=sys.stderr)
if update_result.fatal and opt.use_superproject is not None:
sys.exit(1)
return manifest_path
if need_unload:
m.outer_client.manifest.Unload()
def _FetchProjectList(self, opt, projects):
"""Main function of the fetch worker.
@ -485,8 +526,8 @@ later is required to fix a server side protocol bug.
return (ret, fetched)
def _FetchMain(self, opt, args, all_projects, err_event, manifest_name,
load_local_manifests, ssh_proxy, manifest):
def _FetchMain(self, opt, args, all_projects, err_event,
ssh_proxy, manifest):
"""The main network fetch loop.
Args:
@ -494,8 +535,6 @@ later is required to fix a server side protocol bug.
args: Command line args used to filter out projects.
all_projects: List of all projects that should be fetched.
err_event: Whether an error was hit while processing.
manifest_name: Manifest file to be reloaded.
load_local_manifests: Whether to load local manifests.
ssh_proxy: SSH manager for clients & masters.
manifest: The manifest to use.
@ -526,10 +565,12 @@ later is required to fix a server side protocol bug.
# Iteratively fetch missing and/or nested unregistered submodules
previously_missing_set = set()
while True:
self._ReloadManifest(manifest_name, self.manifest, load_local_manifests)
self._ReloadManifest(None, manifest)
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
submodules_ok=opt.fetch_submodules,
manifest=manifest,
all_manifests=not opt.this_manifest_only)
missing = []
for project in all_projects:
if project.gitdir not in fetched:
@ -624,7 +665,7 @@ later is required to fix a server side protocol bug.
for project in projects:
# Make sure pruning never kicks in with shared projects.
if (not project.use_git_worktrees and
len(project.manifest.GetProjectsWithName(project.name)) > 1):
len(project.manifest.GetProjectsWithName(project.name, all_manifests=True)) > 1):
if not opt.quiet:
print('\r%s: Shared project %s found, disabling pruning.' %
(project.relpath, project.name))
@ -698,7 +739,7 @@ later is required to fix a server side protocol bug.
t.join()
pm.end()
def _ReloadManifest(self, manifest_name, manifest, load_local_manifests=True):
def _ReloadManifest(self, manifest_name, manifest):
"""Reload the manfiest from the file specified by the |manifest_name|.
It unloads the manifest if |manifest_name| is None.
@ -706,17 +747,29 @@ later is required to fix a server side protocol bug.
Args:
manifest_name: Manifest file to be reloaded.
manifest: The manifest to use.
load_local_manifests: Whether to load local manifests.
"""
if manifest_name:
# Override calls Unload already
manifest.Override(manifest_name, load_local_manifests=load_local_manifests)
manifest.Override(manifest_name)
else:
manifest.Unload()
def UpdateProjectList(self, opt, manifest):
"""Update the cached projects list for |manifest|
In a multi-manifest checkout, each manifest has its own project.list.
Args:
opt: Program options returned from optparse. See _Options().
manifest: The manifest to use.
Returns:
0: success
1: failure
"""
new_project_paths = []
for project in self.GetProjects(None, missing_ok=True):
for project in self.GetProjects(None, missing_ok=True, manifest=manifest,
all_manifests=False):
if project.relpath:
new_project_paths.append(project.relpath)
file_name = 'project.list'
@ -766,7 +819,8 @@ later is required to fix a server side protocol bug.
new_paths = {}
new_linkfile_paths = []
new_copyfile_paths = []
for project in self.GetProjects(None, missing_ok=True):
for project in self.GetProjects(None, missing_ok=True,
manifest=manifest, all_manifests=False):
new_linkfile_paths.extend(x.dest for x in project.linkfiles)
new_copyfile_paths.extend(x.dest for x in project.copyfiles)
@ -897,8 +951,40 @@ later is required to fix a server side protocol bug.
return manifest_name
def _UpdateAllManifestProjects(self, opt, mp, manifest_name):
"""Fetch & update the local manifest project.
After syncing the manifest project, if the manifest has any sub manifests,
those are recursively processed.
Args:
opt: Program options returned from optparse. See _Options().
mp: the manifestProject to query.
manifest_name: Manifest file to be reloaded.
"""
if not mp.standalone_manifest_url:
self._UpdateManifestProject(opt, mp, manifest_name)
if mp.manifest.submanifests:
for submanifest in mp.manifest.submanifests.values():
child = submanifest.repo_client.manifest
child.manifestProject.SyncWithPossibleInit(
submanifest,
current_branch_only=self._GetCurrentBranchOnly(opt, child),
verbose=opt.verbose,
tags=opt.tags,
git_event_log=self.git_event_log,
)
self._UpdateAllManifestProjects(opt, child.manifestProject, None)
def _UpdateManifestProject(self, opt, mp, manifest_name):
"""Fetch & update the local manifest project."""
"""Fetch & update the local manifest project.
Args:
opt: Program options returned from optparse. See _Options().
mp: the manifestProject to query.
manifest_name: Manifest file to be reloaded.
"""
if not opt.local_only:
start = time.time()
success = mp.Sync_NetworkHalf(quiet=opt.quiet, verbose=opt.verbose,
@ -924,6 +1010,7 @@ later is required to fix a server side protocol bug.
if not clean:
sys.exit(1)
self._ReloadManifest(manifest_name, mp.manifest)
if opt.jobs is None:
self.jobs = mp.manifest.default.sync_j
@ -948,9 +1035,6 @@ later is required to fix a server side protocol bug.
if opt.prune is None:
opt.prune = True
if self.outer_client.manifest.is_multimanifest and not opt.this_manifest_only and args:
self.OptionParser.error('partial syncs must use --this-manifest-only')
def Execute(self, opt, args):
if opt.jobs:
self.jobs = opt.jobs
@ -959,7 +1043,7 @@ later is required to fix a server side protocol bug.
self.jobs = min(self.jobs, (soft_limit - 5) // 3)
manifest = self.outer_manifest
if opt.this_manifest_only or not opt.outer_manifest:
if not opt.outer_manifest:
manifest = self.manifest
if opt.manifest_name:
@ -994,39 +1078,26 @@ later is required to fix a server side protocol bug.
'receive updates; run `repo init --repo-rev=stable` to fix.',
file=sys.stderr)
mp = manifest.manifestProject
for m in self.ManifestList(opt):
mp = m.manifestProject
is_standalone_manifest = bool(mp.standalone_manifest_url)
if not is_standalone_manifest:
mp.PreSync()
if opt.repo_upgraded:
_PostRepoUpgrade(manifest, quiet=opt.quiet)
_PostRepoUpgrade(m, quiet=opt.quiet)
if not opt.mp_update:
if opt.mp_update:
self._UpdateAllManifestProjects(opt, mp, manifest_name)
else:
print('Skipping update of local manifest project.')
elif not is_standalone_manifest:
self._UpdateManifestProject(opt, mp, manifest_name)
load_local_manifests = not manifest.HasLocalManifests
use_superproject = git_superproject.UseSuperproject(opt.use_superproject, manifest)
if use_superproject and (manifest.IsMirror or manifest.IsArchive):
# Don't use superproject, because we have no working tree.
use_superproject = False
if opt.use_superproject is not None:
print('Defaulting to no-use-superproject because there is no working tree.')
superproject_logging_data = {
'superproject': use_superproject,
'haslocalmanifests': bool(manifest.HasLocalManifests),
'hassuperprojecttag': bool(manifest.superproject),
}
if use_superproject:
manifest_name = self._UpdateProjectsRevisionId(
opt, args, load_local_manifests, superproject_logging_data,
manifest) or opt.manifest_name
superproject_logging_data = {}
self._UpdateProjectsRevisionId(opt, args, superproject_logging_data,
manifest)
if self.gitc_manifest:
gitc_manifest_projects = self.GetProjects(args,
missing_ok=True)
gitc_manifest_projects = self.GetProjects(args, missing_ok=True)
gitc_projects = []
opened_projects = []
for project in gitc_manifest_projects:
@ -1059,9 +1130,12 @@ later is required to fix a server side protocol bug.
for path in opened_projects]
if not args:
return
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
submodules_ok=opt.fetch_submodules,
manifest=manifest,
all_manifests=not opt.this_manifest_only)
err_network_sync = False
err_update_projects = False
@ -1073,7 +1147,6 @@ later is required to fix a server side protocol bug.
# Initialize the socket dir once in the parent.
ssh_proxy.sock()
all_projects = self._FetchMain(opt, args, all_projects, err_event,
manifest_name, load_local_manifests,
ssh_proxy, manifest)
if opt.network_only:
@ -1090,18 +1163,19 @@ later is required to fix a server side protocol bug.
file=sys.stderr)
sys.exit(1)
if manifest.IsMirror or manifest.IsArchive:
for m in self.ManifestList(opt):
if m.IsMirror or m.IsArchive:
# bail out now, we have no working tree
return
continue
if self.UpdateProjectList(opt, manifest):
if self.UpdateProjectList(opt, m):
err_event.set()
err_update_projects = True
if opt.fail_fast:
print('\nerror: Local checkouts *not* updated.', file=sys.stderr)
sys.exit(1)
err_update_linkfiles = not self.UpdateCopyLinkfileList(manifest)
err_update_linkfiles = not self.UpdateCopyLinkfileList(m)
if err_update_linkfiles:
err_event.set()
if opt.fail_fast:
@ -1114,10 +1188,14 @@ later is required to fix a server side protocol bug.
if err_checkout:
err_event.set()
# If there's a notice that's supposed to print at the end of the sync, print
# it now...
if manifest.notice:
print(manifest.notice)
printed_notices = set()
# If there's a notice that's supposed to print at the end of the sync,
# print it now... But avoid printing duplicate messages, and preserve
# order.
for m in sorted(self.ManifestList(opt), key=lambda x: x.path_prefix):
if m.notice and m.notice not in printed_notices:
print(m.notice)
printed_notices.add(m.notice)
# If we saw an error, exit with code 1 so that other scripts can check.
if err_event.is_set():