superproject: Don't update the commit ids of projects if remote is different.

1) Skip setting the revision id (commit id) for the projects whose
   remote doesn't match superproject's remote.
2) exp-superproject/superproject_override.xml includes local_manfiest's
   projects. When we load this XML, don't reload projects from local.xml
   (otherwise we will get duplicate projects errors).

Tested the code with the following commands.

$ ./run_tests -v

+ Test with local.xml
  $ repo_dev init -u sso://android.git.corp.google.com/platform/manifest -b master --use-superproject --partial-clone --clone-filter=blob:limit=10M && mkdir -p .repo/local_manifests && (gcertstatus -quiet=true || gcert) && ln -s /google/src/head/depot/google3/wireless/android/build_tools/aosp/manifests/mirror-aosp-master-with-vendor/local.xml  .repo/local_manifests/local.xml

  $ repo_dev sync -c -j8

+ Test without local.xml
  $ repo_dev init -u sso://android.git.corp.google.com/platform/manifest -b master --partial-clone --clone-filter=blob:limit=10M --repo-rev=main --use-superproject
  $ repo_dev sync -c -j8

Bug: [google internal] b/186395810
Change-Id: I4e9d4ac2d94a9fc0cef0ccd787b6310758009e86
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/304882
Tested-by: Raman Tenneti <rtenneti@google.com>
Reviewed-by: Mike Frysinger <vapier@google.com>

git_superproject.py

@@ -262,10 +262,20 @@ class Superproject(object):
       return None
 
     projects_missing_commit_ids = []
+    superproject_remote_name = self._manifest.superproject['remote'].name
     for project in projects:
       path = project.relpath
       if not path:
         continue
+      # Some manifests that pull projects from the "chromium" GoB
+      # (remote="chromium"), and have a private manifest that pulls projects
+      # from both the chromium GoB and "chrome-internal" GoB (remote="chrome").
+      # For such projects, one of the remotes will be different from
+      # superproject's remote. Until superproject, supports multiple remotes,
+      # don't update the commit ids of remotes that don't match superproject's
+      # remote.
+      if project.remote.name != superproject_remote_name:
+        continue
       commit_id = commit_ids.get(path)
       if commit_id:
         project.SetRevisionId(commit_id)

manifest_xml.py

@@ -595,6 +595,10 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         'repo.partialcloneexclude') or ''
     return set(x.strip() for x in exclude.split(','))
 
+  @property
+  def HasLocalManifests(self):
+    return self._load_local_manifests and self.local_manifests
+
   @property
   def IsMirror(self):
     return self.manifestProject.config.GetBoolean('repo.mirror')

subcmds/sync.py

@@ -277,7 +277,7 @@ later is required to fix a server side protocol bug.
     """Returns True if current-branch or use-superproject options are enabled."""
     return opt.current_branch_only or self._UseSuperproject(opt)
 
-  def _UpdateProjectsRevisionId(self, opt, args):
+  def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests):
     """Update revisionId of every project with the SHA from superproject.
 
     This function updates each project's revisionId with SHA from superproject.
@@ -287,6 +287,7 @@ later is required to fix a server side protocol bug.
       opt: Program options returned from optparse.  See _Options().
       args: Arguments to pass to GetProjects. See the GetProjects
           docstring for details.
+      load_local_manifests: Whether to load local manifests.
 
     Returns:
       Returns path to the overriding manifest file.
@@ -302,7 +303,7 @@ later is required to fix a server side protocol bug.
       print('error: Update of revsionId from superproject has failed',
             file=sys.stderr)
       sys.exit(1)
-    self._ReloadManifest(manifest_path)
+    self._ReloadManifest(manifest_path, load_local_manifests)
     return manifest_path
 
   def _FetchProjectList(self, opt, projects):
@@ -565,10 +566,18 @@ later is required to fix a server side protocol bug.
       t.join()
     pm.end()
 
-  def _ReloadManifest(self, manifest_name=None):
+  def _ReloadManifest(self, manifest_name=None, load_local_manifests=True):
+    """Reload the manfiest from the file specified by the |manifest_name|.
+
+    It unloads the manifest if |manifest_name| is None.
+
+    Args:
+      manifest_name: Manifest file to be reloaded.
+      load_local_manifests: Whether to load local manifests.
+    """
     if manifest_name:
       # Override calls _Unload already
-      self.manifest.Override(manifest_name)
+      self.manifest.Override(manifest_name, load_local_manifests=load_local_manifests)
     else:
       self.manifest._Unload()
 
@@ -857,8 +866,9 @@ later is required to fix a server side protocol bug.
     else:
       self._UpdateManifestProject(opt, mp, manifest_name)
 
+    load_local_manifests = not self.manifest.HasLocalManifests
     if self._UseSuperproject(opt):
-      manifest_name = self._UpdateProjectsRevisionId(opt, args)
+      manifest_name = self._UpdateProjectsRevisionId(opt, args, load_local_manifests)
 
     if self.gitc_manifest:
       gitc_manifest_projects = self.GetProjects(args,
@@ -926,7 +936,7 @@ later is required to fix a server side protocol bug.
       # Iteratively fetch missing and/or nested unregistered submodules
       previously_missing_set = set()
       while True:
-        self._ReloadManifest(manifest_name)
+        self._ReloadManifest(manifest_name, load_local_manifests)
         all_projects = self.GetProjects(args,
                                         missing_ok=True,
                                         submodules_ok=opt.fetch_submodules)

tests/test_git_superproject.py

@@ -141,12 +141,12 @@ class SuperprojectTestCase(unittest.TestCase):
       manifest_xml = fp.read()
     self.assertEqual(
         manifest_xml,
-        '<?xml version="1.0" ?><manifest>' +
+        '<?xml version="1.0" ?><manifest>'
-        '<remote name="default-remote" fetch="http://localhost"/>' +
+        '<remote name="default-remote" fetch="http://localhost"/>'
-        '<default remote="default-remote" revision="refs/heads/main"/>' +
+        '<default remote="default-remote" revision="refs/heads/main"/>'
-        '<project name="platform/art" path="art" revision="ABCDEF" ' +
+        '<project name="platform/art" path="art" revision="ABCDEF" '
-        'groups="notdefault,platform-' + self.platform + '"/>' +
+        'groups="notdefault,platform-' + self.platform + '"/>'
-        '<superproject name="superproject"/>' +
+        '<superproject name="superproject"/>'
         '</manifest>')
 
   def test_superproject_update_project_revision_id(self):
@@ -168,13 +168,57 @@ class SuperprojectTestCase(unittest.TestCase):
             manifest_xml = fp.read()
           self.assertEqual(
               manifest_xml,
-              '<?xml version="1.0" ?><manifest>' +
+              '<?xml version="1.0" ?><manifest>'
-              '<remote name="default-remote" fetch="http://localhost"/>' +
+              '<remote name="default-remote" fetch="http://localhost"/>'
-              '<default remote="default-remote" revision="refs/heads/main"/>' +
+              '<default remote="default-remote" revision="refs/heads/main"/>'
-              '<project name="platform/art" path="art" ' +
+              '<project name="platform/art" path="art" '
-              'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea" ' +
+              'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea" '
-              'groups="notdefault,platform-' + self.platform + '"/>' +
+              'groups="notdefault,platform-' + self.platform + '"/>'
-              '<superproject name="superproject"/>' +
+              '<superproject name="superproject"/>'
+              '</manifest>')
+
+  def test_superproject_update_project_revision_id_with_different_remotes(self):
+    """Test update of commit ids of a manifest with mutiple remotes."""
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <remote name="goog" fetch="http://localhost2" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <superproject name="superproject"/>
+  <project path="vendor/x" name="platform/vendor/x" remote="goog" groups="vendor"
+           revision="master-with-vendor" clone-depth="1" />
+  <project path="art" name="platform/art" groups="notdefault,platform-""" + self.platform + """
+  " /></manifest>
+""")
+    self.maxDiff = None
+    self._superproject = git_superproject.Superproject(manifest, self.repodir)
+    self.assertEqual(len(self._superproject._manifest.projects), 2)
+    projects = self._superproject._manifest.projects
+    data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00'
+            '160000 commit e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06\tbootable/recovery\x00')
+    with mock.patch.object(self._superproject, '_Init', return_value=True):
+      with mock.patch.object(self._superproject, '_Fetch', return_value=True):
+        with mock.patch.object(self._superproject,
+                               '_LsTree',
+                               return_value=data):
+          # Create temporary directory so that it can write the file.
+          os.mkdir(self._superproject._superproject_path)
+          manifest_path = self._superproject.UpdateProjectsRevisionId(projects)
+          self.assertIsNotNone(manifest_path)
+          with open(manifest_path, 'r') as fp:
+            manifest_xml = fp.read()
+          self.assertEqual(
+              manifest_xml,
+              '<?xml version="1.0" ?><manifest>'
+              '<remote name="default-remote" fetch="http://localhost"/>'
+              '<remote name="goog" fetch="http://localhost2"/>'
+              '<default remote="default-remote" revision="refs/heads/main"/>'
+              '<project name="platform/art" path="art" '
+              'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea" '
+              'groups="notdefault,platform-' + self.platform + '"/>'
+              '<project name="platform/vendor/x" path="vendor/x" remote="goog" '
+              'revision="master-with-vendor" groups="vendor" clone-depth="1"/>'
+              '<superproject name="superproject"/>'
               '</manifest>')