init: added --use-superproject option to clone superproject.

Added --no-use-superproject to repo and init.py to disable use of
manifest superprojects.

Replaced the term "sha" with "commit id".

Added _GetBranch method to Superproject object.

Moved shared code between init and sync into SyncSuperproject function.
This function either does git clone or git fetch. If git fetch fails
it does git clone.

Changed Superproject constructor to accept manifest, repodir and branch
to avoid passing them to multiple functions as argument.

Changed functions that were raising exceptions to return either True
or False.

Saved the --use-superproject option in config as repo.superproject.
Updated internal-fs-layout.md document.

Updated the tests to work with the new API changes in Superproject.

Performance for the first time sync has improved from 20 minutes to
around 15 minutes.

Tested the code with the following commands.

$ ./run_tests -v

Tested the sync code by using repo_dev alias and pointing to this CL.

$ repo init took around 20 seconds longer because of cloning of superproject.

$ time repo_dev init -u sso://android.git.corp.google.com/platform/manifest -b master --partial-clone --clone-filter=blob:limit=10M --repo-rev=main --use-superproject
...
real	0m35.919s
user	0m21.947s
sys	0m8.977s

First run
$ time repo sync --use-superproject
...
real	16m41.982s
user	100m6.916s
sys	19m18.753s

No difference in repo sync time after the first run.

Bug: [google internal] b/179090734
Bug: https://crbug.com/gerrit/13709
Bug: https://crbug.com/gerrit/13707

Change-Id: I12df92112f46e001dfbc6f12cd633c3a15cf924b
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/296382
Reviewed-by: Mike Frysinger <vapier@google.com>
Tested-by: Raman Tenneti <rtenneti@google.com>

subcmds/init.py

@@ -25,6 +25,7 @@ from error import ManifestParseError
 from project import SyncBuffer
 from git_config import GitConfig
 from git_command import git_require, MIN_GIT_VERSION_SOFT, MIN_GIT_VERSION_HARD
+import git_superproject
 import platform_utils
 from wrapper import Wrapper
 
@@ -134,6 +135,11 @@ to update the working directory files.
     g.add_option('--submodules',
                  dest='submodules', action='store_true',
                  help='sync any submodules associated with the manifest repo')
+    g.add_option('--use-superproject', action='store_true',
+                 help='use the manifest superproject to sync projects')
+    g.add_option('--no-use-superproject', action='store_false',
+                 dest='use_superproject',
+                 help='disable use of manifest superprojects')
     g.add_option('-g', '--groups',
                  dest='groups', default='default',
                  help='restrict manifest projects to ones with specified '
@@ -176,6 +182,14 @@ to update the working directory files.
     return {'REPO_MANIFEST_URL': 'manifest_url',
             'REPO_MIRROR_LOCATION': 'reference'}
 
+  def _CloneSuperproject(self):
+    """Clone the superproject based on the superproject's url and branch."""
+    superproject = git_superproject.Superproject(self.manifest,
+                                                 self.repodir)
+    if not superproject.Sync():
+      print('error: git update of superproject failed', file=sys.stderr)
+      sys.exit(1)
+
   def _SyncManifest(self, opt):
     m = self.manifest.manifestProject
     is_new = not m.Exists
@@ -305,6 +319,9 @@ to update the working directory files.
     if opt.submodules:
       m.config.SetBoolean('repo.submodules', opt.submodules)
 
+    if opt.use_superproject is not None:
+      m.config.SetBoolean('repo.superproject', opt.use_superproject)
+
     if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet, verbose=opt.verbose,
                               clone_bundle=opt.clone_bundle,
                               current_branch_only=opt.current_branch_only,
@@ -519,6 +536,9 @@ to update the working directory files.
     self._SyncManifest(opt)
     self._LinkManifest(opt.manifest_name)
 
+    if self.manifest.manifestProject.config.GetBoolean('repo.superproject'):
+      self._CloneSuperproject()
+
     if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
       if opt.config_name or self._ShouldConfigureUser(opt):
         self._ConfigureUser(opt)

subcmds/sync.py

@@ -294,28 +294,12 @@ later is required to fix a server side protocol bug.
     Returns:
       Returns path to the overriding manifest file.
     """
-    if not self.manifest.superproject:
-      print('error: superproject tag is not defined in manifest.xml',
-            file=sys.stderr)
-      sys.exit(1)
-    print('WARNING: --use-superproject is experimental and not '
-          'for general use', file=sys.stderr)
-
-    superproject_url = self.manifest.superproject['remote'].url
-    if not superproject_url:
-      print('error: superproject URL is not defined in manifest.xml',
-            file=sys.stderr)
-      sys.exit(1)
-
-    superproject = git_superproject.Superproject(self.manifest.repodir)
+    superproject = git_superproject.Superproject(self.manifest,
+                                                 self.repodir)
     all_projects = self.GetProjects(args,
                                     missing_ok=True,
                                     submodules_ok=opt.fetch_submodules)
-    branch = self._GetBranch()
-    manifest_path = superproject.UpdateProjectsRevisionId(self.manifest,
-                                                          all_projects,
-                                                          url=superproject_url,
-                                                          branch=branch)
+    manifest_path = superproject.UpdateProjectsRevisionId(all_projects)
     if not manifest_path:
       print('error: Update of revsionId from superproject has failed',
             file=sys.stderr)