From e4e94d26ae81dbc9eb6e2f345fac7cd8c533cb9a Mon Sep 17 00:00:00 2001 From: Martin Kelly Date: Tue, 21 Mar 2017 16:05:12 -0700 Subject: [PATCH] init: add --submodules to sync manifest submodules repo sync can sync submodules via the --fetch-submodules option. However, if the manifest repo has submodules, those will not be synced. Having submodules in the manifest repo -- while not commonly done -- can be useful for inheriting a manifest from another project using and layering changes on top of it. In this way, you can avoid having to deal with merge conflicts between your own manifests and the other project's manifests (for example, if you're managing an Android fork). Add a --submodule option to init that automatically syncs the submodules in the manifest repo whenever the manifest repo changes. Change-Id: I45d34f04517774c1462d7f233f482d1d81a332a8 Signed-off-by: Martin Kelly --- manifest_xml.py | 4 ++++ project.py | 49 +++++++++++++++++++++++++++++++++++++++++-------- repo | 3 +++ subcmds/init.py | 10 ++++++++-- subcmds/sync.py | 5 +++-- 5 files changed, 59 insertions(+), 12 deletions(-) diff --git a/manifest_xml.py b/manifest_xml.py index 0859e1fb..73e34964 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -393,6 +393,10 @@ class XmlManifest(object): def IsArchive(self): return self.manifestProject.config.GetBoolean('repo.archive') + @property + def HasSubmodules(self): + return self.manifestProject.config.GetBoolean('repo.submodules') + def _Unload(self): self._loaded = False self._projects = {} diff --git a/project.py b/project.py index 0d60fc6e..e3185b88 100644 --- a/project.py +++ b/project.py @@ -1198,7 +1198,8 @@ class Project(object): no_tags=False, archive=False, optimized_fetch=False, - prune=False): + prune=False, + submodules=False): """Perform only the network IO portion of the sync process. Local working directory/branch state is not affected. """ @@ -1275,7 +1276,8 @@ class Project(object): if (need_to_fetch and not self._RemoteFetch(initial=is_new, quiet=quiet, alt_dir=alt_dir, current_branch_only=current_branch_only, - no_tags=no_tags, prune=prune, depth=depth)): + no_tags=no_tags, prune=prune, depth=depth, + submodules=submodules)): return False if self.worktree: @@ -1331,11 +1333,11 @@ class Project(object): raise ManifestInvalidRevisionError('revision %s in %s not found' % (self.revisionExpr, self.name)) - def Sync_LocalHalf(self, syncbuf, force_sync=False): + def Sync_LocalHalf(self, syncbuf, force_sync=False, submodules=False): """Perform only the local IO portion of the sync process. Network access is not required. """ - self._InitWorkTree(force_sync=force_sync) + self._InitWorkTree(force_sync=force_sync, submodules=submodules) all_refs = self.bare_ref.all self.CleanPublishedCache(all_refs) revid = self.GetRevisionId(all_refs) @@ -1344,6 +1346,9 @@ class Project(object): self._FastForward(revid) self._CopyAndLinkFiles() + def _dosubmodules(): + self._SyncSubmodules(quiet=True) + head = self.work_git.GetHead() if head.startswith(R_HEADS): branch = head[len(R_HEADS):] @@ -1377,6 +1382,8 @@ class Project(object): try: self._Checkout(revid, quiet=True) + if submodules: + self._SyncSubmodules(quiet=True) except GitError as e: syncbuf.fail(self, e) return @@ -1401,6 +1408,8 @@ class Project(object): branch.name) try: self._Checkout(revid, quiet=True) + if submodules: + self._SyncSubmodules(quiet=True) except GitError as e: syncbuf.fail(self, e) return @@ -1426,6 +1435,8 @@ class Project(object): # strict subset. We can fast-forward safely. # syncbuf.later1(self, _doff) + if submodules: + syncbuf.later1(self, _dosubmodules) return # Examine the local commits not in the remote. Find the @@ -1477,19 +1488,28 @@ class Project(object): branch.Save() if cnt_mine > 0 and self.rebase: + def _docopyandlink(): + self._CopyAndLinkFiles() + def _dorebase(): self._Rebase(upstream='%s^1' % last_mine, onto=revid) - self._CopyAndLinkFiles() syncbuf.later2(self, _dorebase) + if submodules: + syncbuf.later2(self, _dosubmodules) + syncbuf.later2(self, _docopyandlink) elif local_changes: try: self._ResetHard(revid) + if submodules: + self._SyncSubmodules(quiet=True) self._CopyAndLinkFiles() except GitError as e: syncbuf.fail(self, e) return else: syncbuf.later1(self, _doff) + if submodules: + syncbuf.later1(self, _dosubmodules) def AddCopyFile(self, src, dest, absdest): # dest should already be an absolute path, but src is project relative @@ -1892,7 +1912,8 @@ class Project(object): alt_dir=None, no_tags=False, prune=False, - depth=None): + depth=None, + submodules=False): is_sha1 = False tag_name = None @@ -2004,6 +2025,9 @@ class Project(object): if prune: cmd.append('--prune') + if submodules: + cmd.append('--recurse-submodules=on-demand') + spec = [] if not current_branch_only: # Fetch whole repo @@ -2224,6 +2248,13 @@ class Project(object): if GitCommand(self, cmd).Wait() != 0: raise GitError('%s reset --hard %s ' % (self.name, rev)) + def _SyncSubmodules(self, quiet=True): + cmd = ['submodule', 'update', '--init', '--recursive'] + if quiet: + cmd.append('-q') + if GitCommand(self, cmd).Wait() != 0: + raise GitError('%s submodule update --init --recursive %s ' % self.name) + def _Rebase(self, upstream, onto=None): cmd = ['rebase'] if onto is not None: @@ -2464,7 +2495,7 @@ class Project(object): else: raise - def _InitWorkTree(self, force_sync=False): + def _InitWorkTree(self, force_sync=False, submodules=False): dotgit = os.path.join(self.worktree, '.git') init_dotgit = not os.path.exists(dotgit) try: @@ -2479,7 +2510,7 @@ class Project(object): if force_sync: try: shutil.rmtree(dotgit) - return self._InitWorkTree(force_sync=False) + return self._InitWorkTree(force_sync=False, submodules=submodules) except: raise e raise e @@ -2493,6 +2524,8 @@ class Project(object): if GitCommand(self, cmd).Wait() != 0: raise GitError("cannot initialize work tree") + if submodules: + self._SyncSubmodules(quiet=True) self._CopyAndLinkFiles() except Exception: if init_dotgit: diff --git a/repo b/repo index dcd48e03..c1d86194 100755 --- a/repo +++ b/repo @@ -192,6 +192,9 @@ group.add_option('--archive', dest='archive', action='store_true', help='checkout an archive instead of a git repository for ' 'each project. See git archive.') +group.add_option('--submodules', + dest='submodules', action='store_true', + help='sync any submodules associated with the manifest repo') group.add_option('-g', '--groups', dest='groups', default='default', help='restrict manifest projects to ones with specified ' diff --git a/subcmds/init.py b/subcmds/init.py index bb7187d7..b260ec0f 100644 --- a/subcmds/init.py +++ b/subcmds/init.py @@ -111,6 +111,9 @@ to update the working directory files. dest='archive', action='store_true', help='checkout an archive instead of a git repository for ' 'each project. See git archive.') + g.add_option('--submodules', + dest='submodules', action='store_true', + help='sync any submodules associated with the manifest repo') g.add_option('-g', '--groups', dest='groups', default='default', help='restrict manifest projects to ones with specified ' @@ -236,10 +239,13 @@ to update the working directory files. 'in another location.', file=sys.stderr) sys.exit(1) + if opt.submodules: + m.config.SetString('repo.submodules', 'true') + if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet, clone_bundle=not opt.no_clone_bundle, current_branch_only=opt.current_branch_only, - no_tags=opt.no_tags): + no_tags=opt.no_tags, submodules=opt.submodules): r = m.GetRemote(m.remote.name) print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr) @@ -253,7 +259,7 @@ to update the working directory files. m.MetaBranchSwitch() syncbuf = SyncBuffer(m.config) - m.Sync_LocalHalf(syncbuf) + m.Sync_LocalHalf(syncbuf, submodules=opt.submodules) syncbuf.Finish() if is_new or m.CurrentBranch is None: diff --git a/subcmds/sync.py b/subcmds/sync.py index 8e8529ee..82056f33 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -723,11 +723,12 @@ later is required to fix a server side protocol bug. mp.Sync_NetworkHalf(quiet=opt.quiet, current_branch_only=opt.current_branch_only, no_tags=opt.no_tags, - optimized_fetch=opt.optimized_fetch) + optimized_fetch=opt.optimized_fetch, + submodules=self.manifest.HasSubmodules) if mp.HasChanges: syncbuf = SyncBuffer(mp.config) - mp.Sync_LocalHalf(syncbuf) + mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules) if not syncbuf.Finish(): sys.exit(1) self._ReloadManifest(manifest_name)