From c474c9cba1a8fbe09c219cc588d9ed334d31cd1e Mon Sep 17 00:00:00 2001 From: Jack Neus Date: Mon, 26 Jul 2021 23:08:54 +0000 Subject: [PATCH] repo: Add support for standalone manifests Added --standalone_manifest to repo tool. If set, the manifest is downloaded directly from the appropriate source (currently, we only support GS) and used instead of creating a manifest git checkout. The manifests.git repo is still created to keep track of various config but is marked as being for a standalone manifest so that the repo tool doesn't try to run networked git commands in it. BUG=b:192664812 TEST=existing tests (no coverage), manual runs Change-Id: I84378cbc7f8e515eabeccdde9665efc8cd2a9d21 Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/312942 Tested-by: Jack Neus Reviewed-by: Mike Frysinger --- docs/internal-fs-layout.md | 1 + fetch.py | 38 +++++++++++++++++ git_config.py | 6 ++- man/repo-gitc-init.1 | 6 ++- man/repo-init.1 | 12 +++++- repo | 4 ++ subcmds/init.py | 83 +++++++++++++++++++++++++++++++------- 7 files changed, 132 insertions(+), 18 deletions(-) create mode 100644 fetch.py diff --git a/docs/internal-fs-layout.md b/docs/internal-fs-layout.md index e3be1731..af6a4523 100644 --- a/docs/internal-fs-layout.md +++ b/docs/internal-fs-layout.md @@ -157,6 +157,7 @@ User controlled settings are initialized when running `repo init`. | Setting | `repo init` Option | Use/Meaning | |------------------- |---------------------------|-------------| | manifest.groups | `--groups` & `--platform` | The manifest groups to sync | +| manifest.standalone | `--standalone-manifest` | Download manifest as static file instead of creating checkout | | repo.archive | `--archive` | Use `git archive` for checkouts | | repo.clonebundle | `--clone-bundle` | Whether the initial sync used clone.bundle explicitly | | repo.clonefilter | `--clone-filter` | Filter setting when using [partial git clones] | diff --git a/fetch.py b/fetch.py new file mode 100644 index 00000000..5b9997a8 --- /dev/null +++ b/fetch.py @@ -0,0 +1,38 @@ +# Copyright (C) 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module contains functions used to fetch files from various sources.""" + +import subprocess +import sys +from urllib.parse import urlparse + +def fetch_file(url): + """Fetch a file from the specified source using the appropriate protocol. + + Returns: + The contents of the file as bytes. + """ + scheme = urlparse(url).scheme + if scheme == 'gs': + cmd = ['gsutil', 'cat', url] + try: + result = subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + return result.stdout + except subprocess.CalledProcessError as e: + print('fatal: error running "gsutil": %s' % e.output, + file=sys.stderr) + sys.exit(1) + raise ValueError('unsupported url %s' % url) diff --git a/git_config.py b/git_config.py index d882239b..778e81a4 100644 --- a/git_config.py +++ b/git_config.py @@ -104,6 +104,10 @@ class GitConfig(object): os.path.dirname(self.file), '.repo_' + os.path.basename(self.file) + '.json') + def ClearCache(self): + """Clear the in-memory cache of config.""" + self._cache_dict = None + def Has(self, name, include_defaults=True): """Return true if this configuration file has the key. """ @@ -399,7 +403,7 @@ class GitConfig(object): if p.Wait() == 0: return p.stdout else: - GitError('git config %s: %s' % (str(args), p.stderr)) + raise GitError('git config %s: %s' % (str(args), p.stderr)) class RepoConfig(GitConfig): diff --git a/man/repo-gitc-init.1 b/man/repo-gitc-init.1 index 1d1b23a8..9b61866e 100644 --- a/man/repo-gitc-init.1 +++ b/man/repo-gitc-init.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man. -.TH REPO "1" "July 2021" "repo gitc-init" "Repo Manual" +.TH REPO "1" "September 2021" "repo gitc-init" "Repo Manual" .SH NAME repo \- repo gitc-init - manual page for repo gitc-init .SH SYNOPSIS @@ -31,6 +31,10 @@ manifest branch or revision (use HEAD for default) \fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml initial manifest file .TP +\fB\-\-standalone\-manifest\fR +download the manifest as a static file rather then +create a git checkout of the manifest repo +.TP \fB\-g\fR GROUP, \fB\-\-groups\fR=\fI\,GROUP\/\fR restrict manifest projects to ones with specified group(s) [default|all|G1,G2,G3|G4,\-G5,\-G6] diff --git a/man/repo-init.1 b/man/repo-init.1 index e860f95d..9957b64d 100644 --- a/man/repo-init.1 +++ b/man/repo-init.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man. -.TH REPO "1" "July 2021" "repo init" "Repo Manual" +.TH REPO "1" "September 2021" "repo init" "Repo Manual" .SH NAME repo \- repo init - manual page for repo init .SH SYNOPSIS @@ -31,6 +31,10 @@ manifest branch or revision (use HEAD for default) \fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml initial manifest file .TP +\fB\-\-standalone\-manifest\fR +download the manifest as a static file rather then +create a git checkout of the manifest repo +.TP \fB\-g\fR GROUP, \fB\-\-groups\fR=\fI\,GROUP\/\fR restrict manifest projects to ones with specified group(s) [default|all|G1,G2,G3|G4,\-G5,\-G6] @@ -137,6 +141,12 @@ equivalent to using \fB\-b\fR HEAD. The optional \fB\-m\fR argument can be used to specify an alternate manifest to be used. If no manifest is specified, the manifest default.xml will be used. .PP +If the \fB\-\-standalone\-manifest\fR argument is set, the manifest will be downloaded +directly from the specified \fB\-\-manifest\-url\fR as a static file (rather than setting +up a manifest git checkout). With \fB\-\-standalone\-manifest\fR, the manifest will be +fully static and will not be re\-downloaded during subsesquent `repo init` and +`repo sync` calls. +.PP The \fB\-\-reference\fR option can be used to point to a directory that has the content of a \fB\-\-mirror\fR sync. This will make the working directory use as much data as possible from the local reference directory when fetching from the server. This diff --git a/repo b/repo index 3b244c16..f61639f3 100755 --- a/repo +++ b/repo @@ -312,6 +312,10 @@ def InitParser(parser, gitc_init=False): metavar='PLATFORM') group.add_option('--submodules', action='store_true', help='sync any submodules associated with the manifest repo') + group.add_option('--standalone-manifest', action='store_true', + help='download the manifest as a static file ' + 'rather then create a git checkout of ' + 'the manifest repo') # Options that only affect manifest project, and not any of the projects # specified in the manifest itself. diff --git a/subcmds/init.py b/subcmds/init.py index 5671fc24..9c6b2ad9 100644 --- a/subcmds/init.py +++ b/subcmds/init.py @@ -15,6 +15,7 @@ import os import platform import re +import subprocess import sys import urllib.parse @@ -24,6 +25,7 @@ from error import ManifestParseError from project import SyncBuffer from git_config import GitConfig from git_command import git_require, MIN_GIT_VERSION_SOFT, MIN_GIT_VERSION_HARD +import fetch import git_superproject import platform_utils from wrapper import Wrapper @@ -53,6 +55,12 @@ The optional -m argument can be used to specify an alternate manifest to be used. If no manifest is specified, the manifest default.xml will be used. +If the --standalone-manifest argument is set, the manifest will be downloaded +directly from the specified --manifest-url as a static file (rather than +setting up a manifest git checkout). With --standalone-manifest, the manifest +will be fully static and will not be re-downloaded during subsesquent +`repo init` and `repo sync` calls. + The --reference option can be used to point to a directory that has the content of a --mirror sync. This will make the working directory use as much data as possible from the local reference @@ -112,6 +120,22 @@ to update the working directory files. m = self.manifest.manifestProject is_new = not m.Exists + # If repo has already been initialized, we take -u with the absence of + # --standalone-manifest to mean "transition to a standard repo set up", + # which necessitates starting fresh. + # If --standalone-manifest is set, we always tear everything down and start + # anew. + if not is_new: + was_standalone_manifest = m.config.GetString('manifest.standalone') + if opt.standalone_manifest or ( + was_standalone_manifest and opt.manifest_url): + m.config.ClearCache() + if m.gitdir and os.path.exists(m.gitdir): + platform_utils.rmtree(m.gitdir) + if m.worktree and os.path.exists(m.worktree): + platform_utils.rmtree(m.worktree) + + is_new = not m.Exists if is_new: if not opt.manifest_url: print('fatal: manifest url is required.', file=sys.stderr) @@ -136,6 +160,19 @@ to update the working directory files. m._InitGitDir(mirror_git=mirrored_manifest_git) + # If standalone_manifest is set, mark the project as "standalone" -- we'll + # still do much of the manifests.git set up, but will avoid actual syncs to + # a remote. + standalone_manifest = False + if opt.standalone_manifest: + standalone_manifest = True + elif not opt.manifest_url: + # If -u is set and --standalone-manifest is not, then we're not in + # standalone mode. Otherwise, use config to infer what we were in the last + # init. + standalone_manifest = bool(m.config.GetString('manifest.standalone')) + m.config.SetString('manifest.standalone', opt.manifest_url) + self._ConfigureDepth(opt) # Set the remote URL before the remote branch as we might need it below. @@ -145,22 +182,23 @@ to update the working directory files. r.ResetFetch() r.Save() - if opt.manifest_branch: - if opt.manifest_branch == 'HEAD': - opt.manifest_branch = m.ResolveRemoteHead() - if opt.manifest_branch is None: - print('fatal: unable to resolve HEAD', file=sys.stderr) - sys.exit(1) - m.revisionExpr = opt.manifest_branch - else: - if is_new: - default_branch = m.ResolveRemoteHead() - if default_branch is None: - # If the remote doesn't have HEAD configured, default to master. - default_branch = 'refs/heads/master' - m.revisionExpr = default_branch + if not standalone_manifest: + if opt.manifest_branch: + if opt.manifest_branch == 'HEAD': + opt.manifest_branch = m.ResolveRemoteHead() + if opt.manifest_branch is None: + print('fatal: unable to resolve HEAD', file=sys.stderr) + sys.exit(1) + m.revisionExpr = opt.manifest_branch else: - m.PreSync() + if is_new: + default_branch = m.ResolveRemoteHead() + if default_branch is None: + # If the remote doesn't have HEAD configured, default to master. + default_branch = 'refs/heads/master' + m.revisionExpr = default_branch + else: + m.PreSync() groups = re.split(r'[,\s]+', opt.groups) all_platforms = ['linux', 'darwin', 'windows'] @@ -250,6 +288,16 @@ to update the working directory files. if opt.use_superproject is not None: m.config.SetBoolean('repo.superproject', opt.use_superproject) + if standalone_manifest: + if is_new: + manifest_name = 'default.xml' + manifest_data = fetch.fetch_file(opt.manifest_url) + dest = os.path.join(m.worktree, manifest_name) + os.makedirs(os.path.dirname(dest), exist_ok=True) + with open(dest, 'wb') as f: + f.write(manifest_data) + return + if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet, verbose=opt.verbose, clone_bundle=opt.clone_bundle, current_branch_only=opt.current_branch_only, @@ -426,6 +474,11 @@ to update the working directory files. if opt.archive and opt.mirror: self.OptionParser.error('--mirror and --archive cannot be used together.') + if opt.standalone_manifest and ( + opt.manifest_branch or opt.manifest_name != 'default.xml'): + self.OptionParser.error('--manifest-branch and --manifest-name cannot' + ' be used with --standalone-manifest.') + if args: if opt.manifest_url: self.OptionParser.error(