From cd81dd6403fc8dbe6ec5920c517d9083902c3c1f Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 26 Oct 2012 12:18:00 -0700 Subject: [PATCH] Revert "Represent git-submodule as nested projects" This reverts commit 69998b0c6ff724bf620480140ccce648fec7d6a9. Broke Android's non-gitmodule use case. Conflicts: project.py subcmds/sync.py Change-Id: I68ceeb63d8ee3b939f85a64736bdc81dfa352aed --- command.py | 70 +++++---------- docs/manifest-format.txt | 15 +--- manifest_xml.py | 108 ++++++----------------- project.py | 179 +-------------------------------------- subcmds/sync.py | 21 +---- 5 files changed, 54 insertions(+), 339 deletions(-) diff --git a/command.py b/command.py index d543e3a8..0c3b360c 100644 --- a/command.py +++ b/command.py @@ -60,32 +60,6 @@ class Command(object): """ raise NotImplementedError - def _ResetPathToProjectMap(self, projects): - self._by_path = dict((p.worktree, p) for p in projects) - - def _UpdatePathToProjectMap(self, project): - self._by_path[project.worktree] = project - - def _GetProjectByPath(self, path): - project = None - if os.path.exists(path): - oldpath = None - while path \ - and path != oldpath \ - and path != self.manifest.topdir: - try: - project = self._by_path[path] - break - except KeyError: - oldpath = path - path = os.path.dirname(path) - else: - try: - project = self._by_path[path] - except KeyError: - pass - return project - def GetProjects(self, args, missing_ok=False): """A list of projects that match the arguments. """ @@ -100,38 +74,40 @@ class Command(object): groups = [x for x in re.split('[,\s]+', groups) if x] if not args: - all_projects_list = all_projects.values() - derived_projects = [] - for project in all_projects_list: - if project.Registered: - # Do not search registered subproject for derived projects - # since its parent has been searched already - continue - derived_projects.extend(project.GetDerivedSubprojects()) - all_projects_list.extend(derived_projects) - for project in all_projects_list: + for project in all_projects.values(): if ((missing_ok or project.Exists) and project.MatchesGroups(groups)): result.append(project) else: - self._ResetPathToProjectMap(all_projects.values()) + by_path = None for arg in args: project = all_projects.get(arg) if not project: path = os.path.abspath(arg).replace('\\', '/') - project = self._GetProjectByPath(path) - # If it's not a derived project, update path->project mapping and - # search again, as arg might actually point to a derived subproject. - if project and not project.Derived: - search_again = False - for subproject in project.GetDerivedSubprojects(): - self._UpdatePathToProjectMap(subproject) - search_again = True - if search_again: - project = self._GetProjectByPath(path) or project + if not by_path: + by_path = dict() + for p in all_projects.values(): + by_path[p.worktree] = p + + if os.path.exists(path): + oldpath = None + while path \ + and path != oldpath \ + and path != self.manifest.topdir: + try: + project = by_path[path] + break + except KeyError: + oldpath = path + path = os.path.dirname(path) + else: + try: + project = by_path[path] + except KeyError: + pass if not project: raise NoSuchProjectError(arg) diff --git a/docs/manifest-format.txt b/docs/manifest-format.txt index a36af67c..f499868c 100644 --- a/docs/manifest-format.txt +++ b/docs/manifest-format.txt @@ -45,8 +45,7 @@ following DTD: - + @@ -153,10 +152,7 @@ Element project One or more project elements may be specified. Each element describes a single Git repository to be cloned into the repo -client workspace. You may specify Git-submodules by creating a -nested project. Git-submodules will be automatically -recognized and inherit their parent's attributes, but those -may be overridden by an explicitly specified project element. +client workspace. Attribute `name`: A unique name for this project. The project's name is appended onto its remote's fetch URL to generate the actual @@ -167,8 +163,7 @@ URL to configure the Git remote with. The URL gets formed as: where ${remote_fetch} is the remote's fetch attribute and ${project_name} is the project's name attribute. The suffix ".git" is always appended as repo assumes the upstream is a forest of -bare Git repositories. If the project has a parent element, its -name will be prefixed by the parent's. +bare Git repositories. The project name must match the name Gerrit knows, if Gerrit is being used for code reviews. @@ -176,8 +171,6 @@ being used for code reviews. Attribute `path`: An optional path relative to the top directory of the repo client where the Git working directory for this project should be placed. If not supplied the project name is used. -If the project has a parent element, its path will be prefixed -by the parent's. Attribute `remote`: Name of a previously defined remote element. If not supplied the remote given by the default element is used. @@ -197,8 +190,6 @@ its name:`name` and path:`path`. E.g. for definition is implicitly in the following manifest groups: default, name:monkeys, and path:barrel-of. If you place a project in the group "notdefault", it will not be automatically downloaded by repo. -If the project has a parent element, the `name` and `path` here -are the prefixed ones. Element annotation ------------------ diff --git a/manifest_xml.py b/manifest_xml.py index 11e4ee54..dd163bed 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -180,25 +180,20 @@ class XmlManifest(object): root.appendChild(e) root.appendChild(doc.createTextNode('')) - def output_projects(parent, parent_node, projects): - for p in projects: - output_project(parent, parent_node, self.projects[p]) + sort_projects = list(self.projects.keys()) + sort_projects.sort() + + for p in sort_projects: + p = self.projects[p] - def output_project(parent, parent_node, p): if not p.MatchesGroups(groups): - return - - name = p.name - relpath = p.relpath - if parent: - name = self._UnjoinName(parent.name, name) - relpath = self._UnjoinRelpath(parent.relpath, relpath) + continue e = doc.createElement('project') - parent_node.appendChild(e) - e.setAttribute('name', name) - if relpath != name: - e.setAttribute('path', relpath) + root.appendChild(e) + e.setAttribute('name', p.name) + if p.relpath != p.name: + e.setAttribute('path', p.relpath) if not d.remote or p.remote.name != d.remote.name: e.setAttribute('remote', p.remote.name) if peg_rev: @@ -236,16 +231,6 @@ class XmlManifest(object): if p.sync_c: e.setAttribute('sync-c', 'true') - if p.subprojects: - sort_projects = [subp.name for subp in p.subprojects] - sort_projects.sort() - output_projects(p, e, sort_projects) - - sort_projects = [key for key in self.projects.keys() - if not self.projects[key].parent] - sort_projects.sort() - output_projects(None, root, sort_projects) - if self._repo_hooks_project: root.appendChild(doc.createTextNode('')) e = doc.createElement('repo-hooks') @@ -398,15 +383,11 @@ class XmlManifest(object): for node in itertools.chain(*node_list): if node.nodeName == 'project': project = self._ParseProject(node) - def recursively_add_projects(project): - if self._projects.get(project.name): - raise ManifestParseError( - 'duplicate project %s in %s' % - (project.name, self.manifestFile)) - self._projects[project.name] = project - for subproject in project.subprojects: - recursively_add_projects(subproject) - recursively_add_projects(project) + if self._projects.get(project.name): + raise ManifestParseError( + 'duplicate project %s in %s' % + (project.name, self.manifestFile)) + self._projects[project.name] = project if node.nodeName == 'repo-hooks': # Get the name of the project and the (space-separated) list of enabled. repo_hooks_project = self._reqatt(node, 'in-project') @@ -556,19 +537,11 @@ class XmlManifest(object): return '\n'.join(cleanLines) - def _JoinName(self, parent_name, name): - return os.path.join(parent_name, name) - - def _UnjoinName(self, parent_name, name): - return os.path.relpath(name, parent_name) - - def _ParseProject(self, node, parent = None): + def _ParseProject(self, node): """ reads a element from the manifest file """ name = self._reqatt(node, 'name') - if parent: - name = self._JoinName(parent.name, name) remote = self._get_remote(node) if remote is None: @@ -613,66 +586,37 @@ class XmlManifest(object): groups = node.getAttribute('groups') groups = [x for x in re.split('[,\s]+', groups) if x] - if parent is None: - relpath, worktree, gitdir = self.GetProjectPaths(name, path) - else: - relpath, worktree, gitdir = self.GetSubprojectPaths(parent, path) - - default_groups = ['all', 'name:%s' % name, 'path:%s' % relpath] + default_groups = ['all', 'name:%s' % name, 'path:%s' % path] groups.extend(set(default_groups).difference(groups)) + if self.IsMirror: + worktree = None + gitdir = os.path.join(self.topdir, '%s.git' % name) + else: + worktree = os.path.join(self.topdir, path).replace('\\', '/') + gitdir = os.path.join(self.repodir, 'projects/%s.git' % path) + project = Project(manifest = self, name = name, remote = remote.ToRemoteSpec(name), gitdir = gitdir, worktree = worktree, - relpath = relpath, + relpath = path, revisionExpr = revisionExpr, revisionId = None, rebase = rebase, groups = groups, sync_c = sync_c, - upstream = upstream, - parent = parent) + upstream = upstream) for n in node.childNodes: if n.nodeName == 'copyfile': self._ParseCopyFile(project, n) if n.nodeName == 'annotation': self._ParseAnnotation(project, n) - if n.nodeName == 'project': - project.subprojects.append(self._ParseProject(n, parent = project)) return project - def GetProjectPaths(self, name, path): - relpath = path - if self.IsMirror: - worktree = None - gitdir = os.path.join(self.topdir, '%s.git' % name) - else: - worktree = os.path.join(self.topdir, path).replace('\\', '/') - gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path) - return relpath, worktree, gitdir - - def GetSubprojectName(self, parent, submodule_path): - return os.path.join(parent.name, submodule_path) - - def _JoinRelpath(self, parent_relpath, relpath): - return os.path.join(parent_relpath, relpath) - - def _UnjoinRelpath(self, parent_relpath, relpath): - return os.path.relpath(relpath, parent_relpath) - - def GetSubprojectPaths(self, parent, path): - relpath = self._JoinRelpath(parent.relpath, path) - gitdir = os.path.join(parent.gitdir, 'subprojects', '%s.git' % path) - if self.IsMirror: - worktree = None - else: - worktree = os.path.join(parent.worktree, path).replace('\\', '/') - return relpath, worktree, gitdir - def _ParseCopyFile(self, project, node): src = self._reqatt(node, 'src') dest = self._reqatt(node, 'dest') diff --git a/project.py b/project.py index c5ee50fc..2f471692 100644 --- a/project.py +++ b/project.py @@ -22,7 +22,6 @@ import shutil import stat import subprocess import sys -import tempfile import time from color import Coloring @@ -485,28 +484,7 @@ class Project(object): rebase = True, groups = None, sync_c = False, - upstream = None, - parent = None, - is_derived = False): - """Init a Project object. - - Args: - manifest: The XmlManifest object. - name: The `name` attribute of manifest.xml's project element. - remote: RemoteSpec object specifying its remote's properties. - gitdir: Absolute path of git directory. - worktree: Absolute path of git working tree. - relpath: Relative path of git working tree to repo's top directory. - revisionExpr: The `revision` attribute of manifest.xml's project element. - revisionId: git commit id for checking out. - rebase: The `rebase` attribute of manifest.xml's project element. - groups: The `groups` attribute of manifest.xml's project element. - sync_c: The `sync-c` attribute of manifest.xml's project element. - upstream: The `upstream` attribute of manifest.xml's project element. - parent: The parent Project object. - is_derived: False if the project was explicitly defined in the manifest; - True if the project is a discovered submodule. - """ + upstream = None): self.manifest = manifest self.name = name self.remote = remote @@ -529,9 +507,6 @@ class Project(object): self.groups = groups self.sync_c = sync_c self.upstream = upstream - self.parent = parent - self.is_derived = is_derived - self.subprojects = [] self.snapshots = {} self.copyfiles = [] @@ -551,14 +526,6 @@ class Project(object): # project containing repo hooks. self.enabled_repo_hooks = [] - @property - def Registered(self): - return self.parent and not self.is_derived - - @property - def Derived(self): - return self.is_derived - @property def Exists(self): return os.path.isdir(self.gitdir) @@ -1403,150 +1370,6 @@ class Project(object): return kept -## Submodule Management ## - - def GetRegisteredSubprojects(self): - result = [] - def rec(subprojects): - if not subprojects: - return - result.extend(subprojects) - for p in subprojects: - rec(p.subprojects) - rec(self.subprojects) - return result - - def _GetSubmodules(self): - # Unfortunately we cannot call `git submodule status --recursive` here - # because the working tree might not exist yet, and it cannot be used - # without a working tree in its current implementation. - - def get_submodules(gitdir, rev): - # Parse .gitmodules for submodule sub_paths and sub_urls - sub_paths, sub_urls = parse_gitmodules(gitdir, rev) - if not sub_paths: - return [] - # Run `git ls-tree` to read SHAs of submodule object, which happen to be - # revision of submodule repository - sub_revs = git_ls_tree(gitdir, rev, sub_paths) - submodules = [] - for sub_path, sub_url in zip(sub_paths, sub_urls): - try: - sub_rev = sub_revs[sub_path] - except KeyError: - # Ignore non-exist submodules - continue - submodules.append((sub_rev, sub_path, sub_url)) - return submodules - - re_path = re.compile(r'submodule.(\w+).path') - re_url = re.compile(r'submodule.(\w+).url') - def parse_gitmodules(gitdir, rev): - cmd = ['cat-file', 'blob', '%s:.gitmodules' % rev] - try: - p = GitCommand(None, cmd, capture_stdout = True, capture_stderr = True, - bare = True, gitdir = gitdir) - except GitError: - return [], [] - if p.Wait() != 0: - return [], [] - - gitmodules_lines = [] - fd, temp_gitmodules_path = tempfile.mkstemp() - try: - os.write(fd, p.stdout) - os.close(fd) - cmd = ['config', '--file', temp_gitmodules_path, '--list'] - p = GitCommand(None, cmd, capture_stdout = True, capture_stderr = True, - bare = True, gitdir = gitdir) - if p.Wait() != 0: - return [], [] - gitmodules_lines = p.stdout.split('\n') - except GitError: - return [], [] - finally: - os.remove(temp_gitmodules_path) - - names = set() - paths = {} - urls = {} - for line in gitmodules_lines: - if not line: - continue - key, value = line.split('=') - m = re_path.match(key) - if m: - names.add(m.group(1)) - paths[m.group(1)] = value - continue - m = re_url.match(key) - if m: - names.add(m.group(1)) - urls[m.group(1)] = value - continue - names = sorted(names) - return [paths[name] for name in names], [urls[name] for name in names] - - def git_ls_tree(gitdir, rev, paths): - cmd = ['ls-tree', rev, '--'] - cmd.extend(paths) - try: - p = GitCommand(None, cmd, capture_stdout = True, capture_stderr = True, - bare = True, gitdir = gitdir) - except GitError: - return [] - if p.Wait() != 0: - return [] - objects = {} - for line in p.stdout.split('\n'): - if not line.strip(): - continue - object_rev, object_path = line.split()[2:4] - objects[object_path] = object_rev - return objects - - try: - rev = self.GetRevisionId() - except GitError: - return [] - return get_submodules(self.gitdir, rev) - - def GetDerivedSubprojects(self): - result = [] - if not self.Exists: - # If git repo does not exist yet, querying its submodules will - # mess up its states; so return here. - return result - for rev, path, url in self._GetSubmodules(): - name = self.manifest.GetSubprojectName(self, path) - project = self.manifest.projects.get(name) - if project and project.Registered: - # If it has been registered, skip it because we are searching - # derived subprojects, but search for its derived subprojects. - result.extend(project.GetDerivedSubprojects()) - continue - relpath, worktree, gitdir = self.manifest.GetSubprojectPaths(self, path) - remote = RemoteSpec(self.remote.name, - url = url, - review = self.remote.review) - subproject = Project(manifest = self.manifest, - name = name, - remote = remote, - gitdir = gitdir, - worktree = worktree, - relpath = relpath, - revisionExpr = self.revisionExpr, - revisionId = rev, - rebase = self.rebase, - groups = self.groups, - sync_c = self.sync_c, - parent = self, - is_derived = True) - result.append(subproject) - result.extend(subproject.GetDerivedSubprojects()) - return result - - ## Direct Git Commands ## def _RemoteFetch(self, name=None, diff --git a/subcmds/sync.py b/subcmds/sync.py index 7416f4c3..d4637d0c 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -563,31 +563,12 @@ uncommitted changes are present' % project.relpath to_fetch.extend(all_projects) to_fetch.sort(key=self._fetch_times.Get, reverse=True) - fetched = self._Fetch(to_fetch, opt) + self._Fetch(to_fetch, opt) _PostRepoFetch(rp, opt.no_repo_verify) if opt.network_only: # bail out now; the rest touches the working tree return - # Iteratively fetch missing and/or nested unregistered submodules - previously_missing_set = set() - while True: - self.manifest._Unload() - all_projects = self.GetProjects(args, missing_ok=True) - missing = [] - for project in all_projects: - if project.gitdir not in fetched: - missing.append(project) - if not missing: - break - # Stop us from non-stopped fetching actually-missing repos: If set of - # missing repos has not been changed from last fetch, we break. - missing_set = set(p.name for p in missing) - if previously_missing_set == missing_set: - break - previously_missing_set = missing_set - fetched.update(self._Fetch(missing, opt)) - if self.manifest.IsMirror: # bail out now, we have no working tree return