Revert "Represent git-submodule as nested projects"

This reverts commit 69998b0c6f.
Broke Android's non-gitmodule use case.

Conflicts:
	project.py
	subcmds/sync.py

Change-Id: I68ceeb63d8ee3b939f85a64736bdc81dfa352aed

command.py

@@ -60,32 +60,6 @@ class Command(object):
     """
     raise NotImplementedError
 
-  def _ResetPathToProjectMap(self, projects):
-    self._by_path = dict((p.worktree, p) for p in projects)
-
-  def _UpdatePathToProjectMap(self, project):
-    self._by_path[project.worktree] = project
-
-  def _GetProjectByPath(self, path):
-    project = None
-    if os.path.exists(path):
-      oldpath = None
-      while path \
-        and path != oldpath \
-        and path != self.manifest.topdir:
-        try:
-          project = self._by_path[path]
-          break
-        except KeyError:
-          oldpath = path
-          path = os.path.dirname(path)
-    else:
-      try:
-        project = self._by_path[path]
-      except KeyError:
-        pass
-    return project
-
   def GetProjects(self, args, missing_ok=False):
     """A list of projects that match the arguments.
     """
@@ -100,38 +74,40 @@ class Command(object):
     groups = [x for x in re.split('[,\s]+', groups) if x]
 
     if not args:
-      all_projects_list = all_projects.values()
+      for project in all_projects.values():
-      derived_projects = []
-      for project in all_projects_list:
-        if project.Registered:
-          # Do not search registered subproject for derived projects
-          # since its parent has been searched already
-          continue
-        derived_projects.extend(project.GetDerivedSubprojects())
-      all_projects_list.extend(derived_projects)
-      for project in all_projects_list:
         if ((missing_ok or project.Exists) and
             project.MatchesGroups(groups)):
           result.append(project)
     else:
-      self._ResetPathToProjectMap(all_projects.values())
+      by_path = None
 
       for arg in args:
         project = all_projects.get(arg)
 
         if not project:
           path = os.path.abspath(arg).replace('\\', '/')
-          project = self._GetProjectByPath(path)
 
-          # If it's not a derived project, update path->project mapping and
+          if not by_path:
-          # search again, as arg might actually point to a derived subproject.
+            by_path = dict()
-          if project and not project.Derived:
+            for p in all_projects.values():
-            search_again = False
+              by_path[p.worktree] = p
-            for subproject in project.GetDerivedSubprojects():
+
-              self._UpdatePathToProjectMap(subproject)
+          if os.path.exists(path):
-              search_again = True
+            oldpath = None
-            if search_again:
+            while path \
-              project = self._GetProjectByPath(path) or project
+              and path != oldpath \
+              and path != self.manifest.topdir:
+              try:
+                project = by_path[path]
+                break
+              except KeyError:
+                oldpath = path
+                path = os.path.dirname(path)
+          else:
+            try:
+              project = by_path[path]
+            except KeyError:
+              pass
 
         if not project:
           raise NoSuchProjectError(arg)

docs/manifest-format.txt

@@ -45,8 +45,7 @@ following DTD:
     <!ELEMENT manifest-server (EMPTY)>
     <!ATTLIST url              CDATA #REQUIRED>
   
-    <!ELEMENT project (annotation?,
+    <!ELEMENT project (annotation?)>
-                       project*)>
     <!ATTLIST project name     CDATA #REQUIRED>
     <!ATTLIST project path     CDATA #IMPLIED>
     <!ATTLIST project remote   IDREF #IMPLIED>
@@ -153,10 +152,7 @@ Element project
 
 One or more project elements may be specified.  Each element
 describes a single Git repository to be cloned into the repo
-client workspace.  You may specify Git-submodules by creating a
+client workspace.
-nested project.  Git-submodules will be automatically
-recognized and inherit their parent's attributes, but those
-may be overridden by an explicitly specified project element.
 
 Attribute `name`: A unique name for this project.  The project's
 name is appended onto its remote's fetch URL to generate the actual
@@ -167,8 +163,7 @@ URL to configure the Git remote with.  The URL gets formed as:
 where ${remote_fetch} is the remote's fetch attribute and
 ${project_name} is the project's name attribute.  The suffix ".git"
 is always appended as repo assumes the upstream is a forest of
-bare Git repositories.  If the project has a parent element, its
+bare Git repositories.
-name will be prefixed by the parent's.
 
 The project name must match the name Gerrit knows, if Gerrit is
 being used for code reviews.
@@ -176,8 +171,6 @@ being used for code reviews.
 Attribute `path`: An optional path relative to the top directory
 of the repo client where the Git working directory for this project
 should be placed.  If not supplied the project name is used.
-If the project has a parent element, its path will be prefixed
-by the parent's.
 
 Attribute `remote`: Name of a previously defined remote element.
 If not supplied the remote given by the default element is used.
@@ -197,8 +190,6 @@ its name:`name` and path:`path`.  E.g. for
 definition is implicitly in the following manifest groups:
 default, name:monkeys, and path:barrel-of.  If you place a project in the
 group "notdefault", it will not be automatically downloaded by repo.
-If the project has a parent element, the `name` and `path` here
-are the prefixed ones.
 
 Element annotation
 ------------------

manifest_xml.py

@@ -180,25 +180,20 @@ class XmlManifest(object):
       root.appendChild(e)
       root.appendChild(doc.createTextNode(''))
 
-    def output_projects(parent, parent_node, projects):
+    sort_projects = list(self.projects.keys())
-      for p in projects:
+    sort_projects.sort()
-        output_project(parent, parent_node, self.projects[p])
+
+    for p in sort_projects:
+      p = self.projects[p]
 
-    def output_project(parent, parent_node, p):
       if not p.MatchesGroups(groups):
-        return
+        continue
-
-      name = p.name
-      relpath = p.relpath
-      if parent:
-        name = self._UnjoinName(parent.name, name)
-        relpath = self._UnjoinRelpath(parent.relpath, relpath)
 
       e = doc.createElement('project')
-      parent_node.appendChild(e)
+      root.appendChild(e)
-      e.setAttribute('name', name)
+      e.setAttribute('name', p.name)
-      if relpath != name:
+      if p.relpath != p.name:
-        e.setAttribute('path', relpath)
+        e.setAttribute('path', p.relpath)
       if not d.remote or p.remote.name != d.remote.name:
         e.setAttribute('remote', p.remote.name)
       if peg_rev:
@@ -236,16 +231,6 @@ class XmlManifest(object):
       if p.sync_c:
         e.setAttribute('sync-c', 'true')
 
-      if p.subprojects:
-        sort_projects = [subp.name for subp in p.subprojects]
-        sort_projects.sort()
-        output_projects(p, e, sort_projects)
-
-    sort_projects = [key for key in self.projects.keys()
-                     if not self.projects[key].parent]
-    sort_projects.sort()
-    output_projects(None, root, sort_projects)
-
     if self._repo_hooks_project:
       root.appendChild(doc.createTextNode(''))
       e = doc.createElement('repo-hooks')
@@ -398,15 +383,11 @@ class XmlManifest(object):
     for node in itertools.chain(*node_list):
       if node.nodeName == 'project':
         project = self._ParseProject(node)
-        def recursively_add_projects(project):
+        if self._projects.get(project.name):
-          if self._projects.get(project.name):
+          raise ManifestParseError(
-            raise ManifestParseError(
+              'duplicate project %s in %s' %
-                'duplicate project %s in %s' %
+              (project.name, self.manifestFile))
-                (project.name, self.manifestFile))
+        self._projects[project.name] = project
-          self._projects[project.name] = project
-          for subproject in project.subprojects:
-            recursively_add_projects(subproject)
-        recursively_add_projects(project)
       if node.nodeName == 'repo-hooks':
         # Get the name of the project and the (space-separated) list of enabled.
         repo_hooks_project = self._reqatt(node, 'in-project')
@@ -556,19 +537,11 @@ class XmlManifest(object):
 
     return '\n'.join(cleanLines)
 
-  def _JoinName(self, parent_name, name):
+  def _ParseProject(self, node):
-    return os.path.join(parent_name, name)
-
-  def _UnjoinName(self, parent_name, name):
-    return os.path.relpath(name, parent_name)
-
-  def _ParseProject(self, node, parent = None):
     """
     reads a <project> element from the manifest file
     """
     name = self._reqatt(node, 'name')
-    if parent:
-      name = self._JoinName(parent.name, name)
 
     remote = self._get_remote(node)
     if remote is None:
@@ -613,66 +586,37 @@ class XmlManifest(object):
       groups = node.getAttribute('groups')
     groups = [x for x in re.split('[,\s]+', groups) if x]
 
-    if parent is None:
+    default_groups = ['all', 'name:%s' % name, 'path:%s' % path]
-      relpath, worktree, gitdir = self.GetProjectPaths(name, path)
-    else:
-      relpath, worktree, gitdir = self.GetSubprojectPaths(parent, path)
-
-    default_groups = ['all', 'name:%s' % name, 'path:%s' % relpath]
     groups.extend(set(default_groups).difference(groups))
 
+    if self.IsMirror:
+      worktree = None
+      gitdir = os.path.join(self.topdir, '%s.git' % name)
+    else:
+      worktree = os.path.join(self.topdir, path).replace('\\', '/')
+      gitdir = os.path.join(self.repodir, 'projects/%s.git' % path)
+
     project = Project(manifest = self,
                       name = name,
                       remote = remote.ToRemoteSpec(name),
                       gitdir = gitdir,
                       worktree = worktree,
-                      relpath = relpath,
+                      relpath = path,
                       revisionExpr = revisionExpr,
                       revisionId = None,
                       rebase = rebase,
                       groups = groups,
                       sync_c = sync_c,
-                      upstream = upstream,
+                      upstream = upstream)
-                      parent = parent)
 
     for n in node.childNodes:
       if n.nodeName == 'copyfile':
         self._ParseCopyFile(project, n)
       if n.nodeName == 'annotation':
         self._ParseAnnotation(project, n)
-      if n.nodeName == 'project':
-        project.subprojects.append(self._ParseProject(n, parent = project))
 
     return project
 
-  def GetProjectPaths(self, name, path):
-    relpath = path
-    if self.IsMirror:
-      worktree = None
-      gitdir = os.path.join(self.topdir, '%s.git' % name)
-    else:
-      worktree = os.path.join(self.topdir, path).replace('\\', '/')
-      gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path)
-    return relpath, worktree, gitdir
-
-  def GetSubprojectName(self, parent, submodule_path):
-    return os.path.join(parent.name, submodule_path)
-
-  def _JoinRelpath(self, parent_relpath, relpath):
-    return os.path.join(parent_relpath, relpath)
-
-  def _UnjoinRelpath(self, parent_relpath, relpath):
-    return os.path.relpath(relpath, parent_relpath)
-
-  def GetSubprojectPaths(self, parent, path):
-    relpath = self._JoinRelpath(parent.relpath, path)
-    gitdir = os.path.join(parent.gitdir, 'subprojects', '%s.git' % path)
-    if self.IsMirror:
-      worktree = None
-    else:
-      worktree = os.path.join(parent.worktree, path).replace('\\', '/')
-    return relpath, worktree, gitdir
-
   def _ParseCopyFile(self, project, node):
     src = self._reqatt(node, 'src')
     dest = self._reqatt(node, 'dest')

project.py

@@ -22,7 +22,6 @@ import shutil
 import stat
 import subprocess
 import sys
-import tempfile
 import time
 
 from color import Coloring
@@ -485,28 +484,7 @@ class Project(object):
                rebase = True,
                groups = None,
                sync_c = False,
-               upstream = None,
+               upstream = None):
-               parent = None,
-               is_derived = False):
-    """Init a Project object.
-
-    Args:
-      manifest: The XmlManifest object.
-      name: The `name` attribute of manifest.xml's project element.
-      remote: RemoteSpec object specifying its remote's properties.
-      gitdir: Absolute path of git directory.
-      worktree: Absolute path of git working tree.
-      relpath: Relative path of git working tree to repo's top directory.
-      revisionExpr: The `revision` attribute of manifest.xml's project element.
-      revisionId: git commit id for checking out.
-      rebase: The `rebase` attribute of manifest.xml's project element.
-      groups: The `groups` attribute of manifest.xml's project element.
-      sync_c: The `sync-c` attribute of manifest.xml's project element.
-      upstream: The `upstream` attribute of manifest.xml's project element.
-      parent: The parent Project object.
-      is_derived: False if the project was explicitly defined in the manifest;
-                  True if the project is a discovered submodule.
-    """
     self.manifest = manifest
     self.name = name
     self.remote = remote
@@ -529,9 +507,6 @@ class Project(object):
     self.groups = groups
     self.sync_c = sync_c
     self.upstream = upstream
-    self.parent = parent
-    self.is_derived = is_derived
-    self.subprojects = []
 
     self.snapshots = {}
     self.copyfiles = []
@@ -551,14 +526,6 @@ class Project(object):
     # project containing repo hooks.
     self.enabled_repo_hooks = []
 
-  @property
-  def Registered(self):
-    return self.parent and not self.is_derived
-
-  @property
-  def Derived(self):
-    return self.is_derived
-
   @property
   def Exists(self):
     return os.path.isdir(self.gitdir)
@@ -1403,150 +1370,6 @@ class Project(object):
     return kept
 
 
-## Submodule Management ##
-
-  def GetRegisteredSubprojects(self):
-    result = []
-    def rec(subprojects):
-      if not subprojects:
-        return
-      result.extend(subprojects)
-      for p in subprojects:
-        rec(p.subprojects)
-    rec(self.subprojects)
-    return result
-
-  def _GetSubmodules(self):
-    # Unfortunately we cannot call `git submodule status --recursive` here
-    # because the working tree might not exist yet, and it cannot be used
-    # without a working tree in its current implementation.
-
-    def get_submodules(gitdir, rev):
-      # Parse .gitmodules for submodule sub_paths and sub_urls
-      sub_paths, sub_urls = parse_gitmodules(gitdir, rev)
-      if not sub_paths:
-        return []
-      # Run `git ls-tree` to read SHAs of submodule object, which happen to be
-      # revision of submodule repository
-      sub_revs = git_ls_tree(gitdir, rev, sub_paths)
-      submodules = []
-      for sub_path, sub_url in zip(sub_paths, sub_urls):
-        try:
-          sub_rev = sub_revs[sub_path]
-        except KeyError:
-          # Ignore non-exist submodules
-          continue
-        submodules.append((sub_rev, sub_path, sub_url))
-      return submodules
-
-    re_path = re.compile(r'submodule.(\w+).path')
-    re_url = re.compile(r'submodule.(\w+).url')
-    def parse_gitmodules(gitdir, rev):
-      cmd = ['cat-file', 'blob', '%s:.gitmodules' % rev]
-      try:
-        p = GitCommand(None, cmd, capture_stdout = True, capture_stderr = True,
-                       bare = True, gitdir = gitdir)
-      except GitError:
-        return [], []
-      if p.Wait() != 0:
-        return [], []
-
-      gitmodules_lines = []
-      fd, temp_gitmodules_path = tempfile.mkstemp()
-      try:
-        os.write(fd, p.stdout)
-        os.close(fd)
-        cmd = ['config', '--file', temp_gitmodules_path, '--list']
-        p = GitCommand(None, cmd, capture_stdout = True, capture_stderr = True,
-                       bare = True, gitdir = gitdir)
-        if p.Wait() != 0:
-          return [], []
-        gitmodules_lines = p.stdout.split('\n')
-      except GitError:
-        return [], []
-      finally:
-        os.remove(temp_gitmodules_path)
-
-      names = set()
-      paths = {}
-      urls = {}
-      for line in gitmodules_lines:
-        if not line:
-          continue
-        key, value = line.split('=')
-        m = re_path.match(key)
-        if m:
-          names.add(m.group(1))
-          paths[m.group(1)] = value
-          continue
-        m = re_url.match(key)
-        if m:
-          names.add(m.group(1))
-          urls[m.group(1)] = value
-          continue
-      names = sorted(names)
-      return [paths[name] for name in names], [urls[name] for name in names]
-
-    def git_ls_tree(gitdir, rev, paths):
-      cmd = ['ls-tree', rev, '--']
-      cmd.extend(paths)
-      try:
-        p = GitCommand(None, cmd, capture_stdout = True, capture_stderr = True,
-                       bare = True, gitdir = gitdir)
-      except GitError:
-        return []
-      if p.Wait() != 0:
-        return []
-      objects = {}
-      for line in p.stdout.split('\n'):
-        if not line.strip():
-          continue
-        object_rev, object_path = line.split()[2:4]
-        objects[object_path] = object_rev
-      return objects
-
-    try:
-      rev = self.GetRevisionId()
-    except GitError:
-      return []
-    return get_submodules(self.gitdir, rev)
-
-  def GetDerivedSubprojects(self):
-    result = []
-    if not self.Exists:
-      # If git repo does not exist yet, querying its submodules will
-      # mess up its states; so return here.
-      return result
-    for rev, path, url in self._GetSubmodules():
-      name = self.manifest.GetSubprojectName(self, path)
-      project = self.manifest.projects.get(name)
-      if project and project.Registered:
-        # If it has been registered, skip it because we are searching
-        # derived subprojects, but search for its derived subprojects.
-        result.extend(project.GetDerivedSubprojects())
-        continue
-      relpath, worktree, gitdir = self.manifest.GetSubprojectPaths(self, path)
-      remote = RemoteSpec(self.remote.name,
-                          url = url,
-                          review = self.remote.review)
-      subproject = Project(manifest = self.manifest,
-                           name = name,
-                           remote = remote,
-                           gitdir = gitdir,
-                           worktree = worktree,
-                           relpath = relpath,
-                           revisionExpr = self.revisionExpr,
-                           revisionId = rev,
-                           rebase = self.rebase,
-                           groups = self.groups,
-                           sync_c = self.sync_c,
-                           parent = self,
-                           is_derived = True)
-      result.append(subproject)
-      result.extend(subproject.GetDerivedSubprojects())
-    return result
-
-
 ## Direct Git Commands ##
 
   def _RemoteFetch(self, name=None,

subcmds/sync.py

@@ -563,31 +563,12 @@ uncommitted changes are present' % project.relpath
       to_fetch.extend(all_projects)
       to_fetch.sort(key=self._fetch_times.Get, reverse=True)
 
-      fetched = self._Fetch(to_fetch, opt)
+      self._Fetch(to_fetch, opt)
       _PostRepoFetch(rp, opt.no_repo_verify)
       if opt.network_only:
         # bail out now; the rest touches the working tree
         return
 
-      # Iteratively fetch missing and/or nested unregistered submodules
-      previously_missing_set = set()
-      while True:
-        self.manifest._Unload()
-        all_projects = self.GetProjects(args, missing_ok=True)
-        missing = []
-        for project in all_projects:
-          if project.gitdir not in fetched:
-            missing.append(project)
-        if not missing:
-          break
-        # Stop us from non-stopped fetching actually-missing repos: If set of
-        # missing repos has not been changed from last fetch, we break.
-        missing_set = set(p.name for p in missing)
-        if previously_missing_set == missing_set:
-          break
-        previously_missing_set = missing_set
-        fetched.update(self._Fetch(missing, opt))
-
     if self.manifest.IsMirror:
       # bail out now, we have no working tree
       return