Support more url schemes for getting standalone manifest

urllib.requests.urlopen also supports file, so call it unless the
scheme is 'gs'.  This adds http, https, and ftp support.

Change-Id: I3f215c3ebd8e6dee29ba14c7e79ed99d37287109
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/322095
Reviewed-by: Michael Kelly <mkelly@arista.com>
Reviewed-by: Mike Frysinger <vapier@google.com>
Tested-by: Matt Story <mstory@arista.com>

fetch.py

@@ -17,8 +17,10 @@
 import subprocess
 import sys
 from urllib.parse import urlparse
+from urllib.request import urlopen
 
-def fetch_file(url):
+
+def fetch_file(url, verbose=False):
   """Fetch a file from the specified source using the appropriate protocol.
 
   Returns:
@@ -29,13 +31,15 @@ def fetch_file(url):
     cmd = ['gsutil', 'cat', url]
     try:
       result = subprocess.run(
-          cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+          cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+          check=True)
+      if result.stderr and verbose:
+        print('warning: non-fatal error running "gsutil": %s' % result.stderr,
+              file=sys.stderr)
       return result.stdout
     except subprocess.CalledProcessError as e:
-      print('fatal: error running "gsutil": %s' % e.output,
+      print('fatal: error running "gsutil": %s' % e.stderr,
             file=sys.stderr)
     sys.exit(1)
-  if scheme == 'file':
-    with open(url[len('file://'):], 'rb') as f:
+  with urlopen(url) as f:
     return f.read()
-  raise ValueError('unsupported url %s' % url)

ssh.py

@@ -52,6 +52,9 @@ def version():
   """return ssh version as a tuple"""
   try:
     return _parse_ssh_version()
+  except FileNotFoundError:
+    print('fatal: ssh not installed', file=sys.stderr)
+    sys.exit(1)
   except subprocess.CalledProcessError:
     print('fatal: unable to detect ssh version', file=sys.stderr)
     sys.exit(1)

subcmds/init.py

@@ -298,7 +298,7 @@ to update the working directory files.
     if standalone_manifest:
       if is_new:
         manifest_name = 'default.xml'
-        manifest_data = fetch.fetch_file(opt.manifest_url)
+        manifest_data = fetch.fetch_file(opt.manifest_url, verbose=opt.verbose)
         dest = os.path.join(m.worktree, manifest_name)
         os.makedirs(os.path.dirname(dest), exist_ok=True)
         with open(dest, 'wb') as f:

subcmds/sync.py

@@ -605,7 +605,7 @@ later is required to fix a server side protocol bug.
     pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet)
     pm.update(inc=0, msg='prescan')
 
-    gc_gitdirs = {}
+    tidy_dirs = {}
     for project in projects:
       # Make sure pruning never kicks in with shared projects.
       if (not project.use_git_worktrees and
@@ -623,17 +623,28 @@ later is required to fix a server side protocol bug.
                 file=sys.stderr)
           project.config.SetString('gc.pruneExpire', 'never')
       project.config.SetString('gc.autoDetach', 'false')
-      gc_gitdirs[project.gitdir] = project.bare_git
-
-    pm.update(inc=len(projects) - len(gc_gitdirs), msg='warming up')
+      # Only call git gc once per objdir, but call pack-refs for the remainder.
+      if project.objdir not in tidy_dirs:
+        tidy_dirs[project.objdir] = (
+            True,  # Run a full gc.
+            project.bare_git,
+        )
+      elif project.gitdir not in tidy_dirs:
+        tidy_dirs[project.gitdir] = (
+            False,  # Do not run a full gc; just run pack-refs.
+            project.bare_git,
+        )
 
     cpu_count = os.cpu_count()
     jobs = min(self.jobs, cpu_count)
 
     if jobs < 2:
-      for bare_git in gc_gitdirs.values():
+      for (run_gc, bare_git) in tidy_dirs.values():
         pm.update(msg=bare_git._project.name)
+        if run_gc:
           bare_git.gc('--auto')
+        else:
+          bare_git.pack_refs()
       pm.end()
       return
 
@@ -642,11 +653,14 @@ later is required to fix a server side protocol bug.
     threads = set()
     sem = _threading.Semaphore(jobs)
 
-    def GC(bare_git):
+    def tidy_up(run_gc, bare_git):
       pm.start(bare_git._project.name)
       try:
         try:
+          if run_gc:
             bare_git.gc('--auto', config=config)
+          else:
+            bare_git.pack_refs(config=config)
         except GitError:
           err_event.set()
         except Exception:
@@ -656,11 +670,11 @@ later is required to fix a server side protocol bug.
         pm.finish(bare_git._project.name)
         sem.release()
 
-    for bare_git in gc_gitdirs.values():
+    for (run_gc, bare_git) in tidy_dirs.values():
       if err_event.is_set() and opt.fail_fast:
         break
       sem.acquire()
-      t = _threading.Thread(target=GC, args=(bare_git,))
+      t = _threading.Thread(target=tidy_up, args=(run_gc, bare_git,))
       t.daemon = True
       threads.add(t)
       t.start()
@@ -955,6 +969,8 @@ later is required to fix a server side protocol bug.
               file=sys.stderr)
 
     mp = self.manifest.manifestProject
+    is_standalone_manifest = mp.config.GetString('manifest.standalone')
+    if not is_standalone_manifest:
       mp.PreSync()
 
     if opt.repo_upgraded:
@@ -962,7 +978,7 @@ later is required to fix a server side protocol bug.
 
     if not opt.mp_update:
       print('Skipping update of local manifest project.')
-    else:
+    elif not is_standalone_manifest:
       self._UpdateManifestProject(opt, mp, manifest_name)
 
     load_local_manifests = not self.manifest.HasLocalManifests