Support more url schemes for getting standalone manifest

urllib.requests.urlopen also supports file, so call it unless the
scheme is 'gs'.  This adds http, https, and ftp support.

Change-Id: I3f215c3ebd8e6dee29ba14c7e79ed99d37287109
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/322095
Reviewed-by: Michael Kelly <mkelly@arista.com>
Reviewed-by: Mike Frysinger <vapier@google.com>
Tested-by: Matt Story <mstory@arista.com>

fetch.py

@@ -17,8 +17,10 @@
 import subprocess
 import sys
 from urllib.parse import urlparse
+from urllib.request import urlopen
 
-def fetch_file(url):
+
+def fetch_file(url, verbose=False):
   """Fetch a file from the specified source using the appropriate protocol.
 
   Returns:
@@ -29,13 +31,15 @@ def fetch_file(url):
     cmd = ['gsutil', 'cat', url]
     try:
       result = subprocess.run(
-          cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+          cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+          check=True)
+      if result.stderr and verbose:
+        print('warning: non-fatal error running "gsutil": %s' % result.stderr,
+              file=sys.stderr)
       return result.stdout
     except subprocess.CalledProcessError as e:
-      print('fatal: error running "gsutil": %s' % e.output,
+      print('fatal: error running "gsutil": %s' % e.stderr,
             file=sys.stderr)
     sys.exit(1)
-  if scheme == 'file':
+  with urlopen(url) as f:
-    with open(url[len('file://'):], 'rb') as f:
+    return f.read()
-      return f.read()
-  raise ValueError('unsupported url %s' % url)

release/sign-launcher.py

@@ -20,6 +20,7 @@ This is intended to be run only by the official Repo release managers.
 
 import argparse
 import os
+import re
 import subprocess
 import sys
 
@@ -49,18 +50,37 @@ def check(opts):
   util.run(opts, ['gpg', '--verify', f'{opts.launcher}.asc'])
 
 
-def postmsg(opts):
+def get_version(opts):
+  """Get the version from |launcher|."""
+  # Make sure we don't search $PATH when signing the "repo" file in the cwd.
+  launcher = os.path.join('.', opts.launcher)
+  cmd = [launcher, '--version']
+  ret = util.run(opts, cmd, encoding='utf-8', stdout=subprocess.PIPE)
+  m = re.search(r'repo launcher version ([0-9.]+)', ret.stdout)
+  if not m:
+    sys.exit(f'{opts.launcher}: unable to detect repo version')
+  return m.group(1)
+
+
+def postmsg(opts, version):
   """Helpful info to show at the end for release manager."""
   print(f"""
 Repo launcher bucket:
   gs://git-repo-downloads/
 
-To upload this launcher directly:
+You should first upload it with a specific version:
-  gsutil cp -a public-read {opts.launcher} {opts.launcher}.asc gs://git-repo-downloads/
+  gsutil cp -a public-read {opts.launcher} gs://git-repo-downloads/repo-{version}
+  gsutil cp -a public-read {opts.launcher}.asc gs://git-repo-downloads/repo-{version}.asc
 
-NB: You probably want to upload it with a specific version first, e.g.:
+Then to make it the public default:
-  gsutil cp -a public-read {opts.launcher} gs://git-repo-downloads/repo-3.0
+  gsutil cp -a public-read gs://git-repo-downloads/repo-{version} gs://git-repo-downloads/repo
-  gsutil cp -a public-read {opts.launcher}.asc gs://git-repo-downloads/repo-3.0.asc
+  gsutil cp -a public-read gs://git-repo-downloads/repo-{version}.asc gs://git-repo-downloads/repo.asc
+
+NB: If a rollback is necessary, the GS bucket archives old versions, and may be
+    accessed by specifying their unique id number.
+  gsutil ls -la gs://git-repo-downloads/repo gs://git-repo-downloads/repo.asc
+  gsutil cp -a public-read gs://git-repo-downloads/repo#<unique id> gs://git-repo-downloads/repo
+  gsutil cp -a public-read gs://git-repo-downloads/repo.asc#<unique id> gs://git-repo-downloads/repo.asc
 """)
 
 
@@ -103,9 +123,10 @@ def main(argv):
     opts.keys = [util.KEYID_DSA, util.KEYID_RSA, util.KEYID_ECC]
     util.import_release_key(opts)
 
+  version = get_version(opts)
   sign(opts)
   check(opts)
-  postmsg(opts)
+  postmsg(opts, version)
 
   return 0
 

ssh.py

@@ -52,6 +52,9 @@ def version():
   """return ssh version as a tuple"""
   try:
     return _parse_ssh_version()
+  except FileNotFoundError:
+    print('fatal: ssh not installed', file=sys.stderr)
+    sys.exit(1)
   except subprocess.CalledProcessError:
     print('fatal: unable to detect ssh version', file=sys.stderr)
     sys.exit(1)

subcmds/init.py

@@ -127,6 +127,11 @@ to update the working directory files.
     # anew.
     if not is_new:
       was_standalone_manifest = m.config.GetString('manifest.standalone')
+      if was_standalone_manifest and not opt.manifest_url:
+        print('fatal: repo was initialized with a standlone manifest, '
+              'cannot be re-initialized without --manifest-url/-u')
+        sys.exit(1)
+
       if opt.standalone_manifest or (
           was_standalone_manifest and opt.manifest_url):
         m.config.ClearCache()
@@ -166,12 +171,14 @@ to update the working directory files.
     standalone_manifest = False
     if opt.standalone_manifest:
       standalone_manifest = True
-    elif not opt.manifest_url:
+      m.config.SetString('manifest.standalone', opt.manifest_url)
+    elif not opt.manifest_url and not opt.manifest_branch:
       # If -u is set and --standalone-manifest is not, then we're not in
       # standalone mode. Otherwise, use config to infer what we were in the last
       # init.
       standalone_manifest = bool(m.config.GetString('manifest.standalone'))
-    m.config.SetString('manifest.standalone', opt.manifest_url)
+    if not standalone_manifest:
+      m.config.SetString('manifest.standalone', None)
 
     self._ConfigureDepth(opt)
 
@@ -291,7 +298,7 @@ to update the working directory files.
     if standalone_manifest:
       if is_new:
         manifest_name = 'default.xml'
-        manifest_data = fetch.fetch_file(opt.manifest_url)
+        manifest_data = fetch.fetch_file(opt.manifest_url, verbose=opt.verbose)
         dest = os.path.join(m.worktree, manifest_name)
         os.makedirs(os.path.dirname(dest), exist_ok=True)
         with open(dest, 'wb') as f:

subcmds/sync.py

@@ -605,7 +605,7 @@ later is required to fix a server side protocol bug.
     pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet)
     pm.update(inc=0, msg='prescan')
 
-    gc_gitdirs = {}
+    tidy_dirs = {}
     for project in projects:
       # Make sure pruning never kicks in with shared projects.
       if (not project.use_git_worktrees and
@@ -622,17 +622,29 @@ later is required to fix a server side protocol bug.
                 % (project.relpath,),
                 file=sys.stderr)
           project.config.SetString('gc.pruneExpire', 'never')
-      gc_gitdirs[project.gitdir] = project.bare_git
+      project.config.SetString('gc.autoDetach', 'false')
-
+      # Only call git gc once per objdir, but call pack-refs for the remainder.
-    pm.update(inc=len(projects) - len(gc_gitdirs), msg='warming up')
+      if project.objdir not in tidy_dirs:
+        tidy_dirs[project.objdir] = (
+            True,  # Run a full gc.
+            project.bare_git,
+        )
+      elif project.gitdir not in tidy_dirs:
+        tidy_dirs[project.gitdir] = (
+            False,  # Do not run a full gc; just run pack-refs.
+            project.bare_git,
+        )
 
     cpu_count = os.cpu_count()
     jobs = min(self.jobs, cpu_count)
 
     if jobs < 2:
-      for bare_git in gc_gitdirs.values():
+      for (run_gc, bare_git) in tidy_dirs.values():
         pm.update(msg=bare_git._project.name)
-        bare_git.gc('--auto')
+        if run_gc:
+          bare_git.gc('--auto')
+        else:
+          bare_git.pack_refs()
       pm.end()
       return
 
@@ -641,11 +653,14 @@ later is required to fix a server side protocol bug.
     threads = set()
     sem = _threading.Semaphore(jobs)
 
-    def GC(bare_git):
+    def tidy_up(run_gc, bare_git):
       pm.start(bare_git._project.name)
       try:
         try:
-          bare_git.gc('--auto', config=config)
+          if run_gc:
+            bare_git.gc('--auto', config=config)
+          else:
+            bare_git.pack_refs(config=config)
         except GitError:
           err_event.set()
         except Exception:
@@ -655,11 +670,11 @@ later is required to fix a server side protocol bug.
         pm.finish(bare_git._project.name)
         sem.release()
 
-    for bare_git in gc_gitdirs.values():
+    for (run_gc, bare_git) in tidy_dirs.values():
       if err_event.is_set() and opt.fail_fast:
         break
       sem.acquire()
-      t = _threading.Thread(target=GC, args=(bare_git,))
+      t = _threading.Thread(target=tidy_up, args=(run_gc, bare_git,))
       t.daemon = True
       threads.add(t)
       t.start()
@@ -954,14 +969,16 @@ later is required to fix a server side protocol bug.
               file=sys.stderr)
 
     mp = self.manifest.manifestProject
-    mp.PreSync()
+    is_standalone_manifest = mp.config.GetString('manifest.standalone')
+    if not is_standalone_manifest:
+      mp.PreSync()
 
     if opt.repo_upgraded:
       _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
 
     if not opt.mp_update:
       print('Skipping update of local manifest project.')
-    else:
+    elif not is_standalone_manifest:
       self._UpdateManifestProject(opt, mp, manifest_name)
 
     load_local_manifests = not self.manifest.HasLocalManifests