From f32f243ff8aaabe5287235015e1ce189da0123e3 Mon Sep 17 00:00:00 2001 From: Raman Tenneti Date: Mon, 12 Apr 2021 20:57:25 -0700 Subject: [PATCH] init: Added --partial-clone-exclude option. partial-clone-exclude option excludes projects during partial clone. This is a comma-delimited project names (from manifest.xml). This option is persisted and it is used by the sync command. A project that has been unparital'ed will remain unpartial if that project's name is specified in the --partial-clone-exclude option. The project name should match exactly. Added $ ./run_tests -v Bug: [google internal] b/175712967 "I can't "unpartial" my androidx-main checkout" $ rm -rf androidx-main/ $ mkdir androidx-main/ $ cd androidx-main/ $ repo_dev init -u https://android.googlesource.com/platform/manifest -b androidx-main --partial-clone --clone-filter=blob:limit=10M -m default.xml $ repo_dev sync -c -j8 + Verify a project is partial $ cd frameworks/support/ $ git config -l | grep 'partial' + Unpartial a project. $ /google/bin/releases/android/git_repack/git_unpartial + Verify project is unpartial $ git config -l | grep 'partial' $ cd ../.. + Exclude the project from being unparial'ed after init and sync. $ repo_dev init -u https://android.googlesource.com/platform/manifest -b androidx-main --partial-clone --clone-filter=blob:limit=10M --partial-clone-exclude="platform/frameworks/support,platform/frameworks/support-golden" -m default.xml + Verify project is unpartial $ cd frameworks/support/ $ git config -l | grep 'partial' $ cd ../.. $ repo_dev sync -c -j8 $ cd frameworks/support/ $ git config -l | grep 'partial' $ cd ../.. + Remove the project from exclude list and verify that project is partially cloned. $ repo_dev init -u https://android.googlesource.com/platform/manifest -b androidx-main --partial-clone --clone-filter=blob:limit=10M --partial-clone-exclude= -m default.xml $ repo_dev sync -c -j8 $ cd frameworks/support/ $ git config -l | grep 'partial' Change-Id: Id5dba418eba1d3f54b54e826000406534c0ec196 Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/303162 Reviewed-by: Mike Frysinger Tested-by: Raman Tenneti --- docs/internal-fs-layout.md | 33 +++++++++++++++++---------------- manifest_xml.py | 6 ++++++ project.py | 7 ++++++- repo | 4 ++++ subcmds/init.py | 6 +++++- subcmds/sync.py | 7 ++++--- 6 files changed, 42 insertions(+), 21 deletions(-) diff --git a/docs/internal-fs-layout.md b/docs/internal-fs-layout.md index 9cbdefb7..0c59f988 100644 --- a/docs/internal-fs-layout.md +++ b/docs/internal-fs-layout.md @@ -147,22 +147,23 @@ repo client checkout. Most settings use the `[repo]` section to avoid conflicts with git. User controlled settings are initialized when running `repo init`. -| Setting | `repo init` Option | Use/Meaning | -|-------------------|---------------------------|-------------| -| manifest.groups | `--groups` & `--platform` | The manifest groups to sync | -| repo.archive | `--archive` | Use `git archive` for checkouts | -| repo.clonebundle | `--clone-bundle` | Whether the initial sync used clone.bundle explicitly | -| repo.clonefilter | `--clone-filter` | Filter setting when using [partial git clones] | -| repo.depth | `--depth` | Create shallow checkouts when cloning | -| repo.dissociate | `--dissociate` | Dissociate from any reference/mirrors after initial clone | -| repo.mirror | `--mirror` | Checkout is a repo mirror | -| repo.partialclone | `--partial-clone` | Create [partial git clones] | -| repo.reference | `--reference` | Reference repo client checkout | -| repo.submodules | `--submodules` | Sync git submodules | -| repo.superproject | `--use-superproject` | Sync [superproject] | -| repo.worktree | `--worktree` | Use [git worktree] for checkouts | -| user.email | `--config-name` | User's e-mail address; Copied into `.git/config` when checking out a new project | -| user.name | `--config-name` | User's name; Copied into `.git/config` when checking out a new project | +| Setting | `repo init` Option | Use/Meaning | +|------------------- |---------------------------|-------------| +| manifest.groups | `--groups` & `--platform` | The manifest groups to sync | +| repo.archive | `--archive` | Use `git archive` for checkouts | +| repo.clonebundle | `--clone-bundle` | Whether the initial sync used clone.bundle explicitly | +| repo.clonefilter | `--clone-filter` | Filter setting when using [partial git clones] | +| repo.depth | `--depth` | Create shallow checkouts when cloning | +| repo.dissociate | `--dissociate` | Dissociate from any reference/mirrors after initial clone | +| repo.mirror | `--mirror` | Checkout is a repo mirror | +| repo.partialclone | `--partial-clone` | Create [partial git clones] | +| repo.partialcloneexclude | `--partial-clone-exclude` | Comma-delimited list of project names (not paths) to exclude while using [partial git clones] | +| repo.reference | `--reference` | Reference repo client checkout | +| repo.submodules | `--submodules` | Sync git submodules | +| repo.superproject | `--use-superproject` | Sync [superproject] | +| repo.worktree | `--worktree` | Use [git worktree] for checkouts | +| user.email | `--config-name` | User's e-mail address; Copied into `.git/config` when checking out a new project | +| user.name | `--config-name` | User's name; Copied into `.git/config` when checking out a new project | [partial git clones]: https://git-scm.com/docs/gitrepository-layout#_code_partialclone_code [superproject]: https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects diff --git a/manifest_xml.py b/manifest_xml.py index d67ba72d..0c2b45e5 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -589,6 +589,12 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md return self.manifestProject.config.GetString('repo.clonefilter') return None + @property + def PartialCloneExclude(self): + exclude = self.manifest.manifestProject.config.GetString( + 'repo.partialcloneexclude') or '' + return set(x.strip() for x in exclude.split(',')) + @property def IsMirror(self): return self.manifestProject.config.GetBoolean('repo.mirror') diff --git a/project.py b/project.py index 2567c57d..9517c5ab 100644 --- a/project.py +++ b/project.py @@ -1050,7 +1050,8 @@ class Project(object): retry_fetches=0, prune=False, submodules=False, - clone_filter=None): + clone_filter=None, + partial_clone_exclude=None): """Perform only the network IO portion of the sync process. Local working directory/branch state is not affected. """ @@ -1087,6 +1088,10 @@ class Project(object): if clone_bundle and os.path.exists(self.objdir): clone_bundle = False + if self.name in partial_clone_exclude: + clone_bundle = True + clone_filter = None + if is_new is None: is_new = not self.Exists if is_new: diff --git a/repo b/repo index 1a494bc0..604f800b 100755 --- a/repo +++ b/repo @@ -350,6 +350,10 @@ def InitParser(parser, gitc_init=False): group.add_option('--no-partial-clone', action='store_false', help='disable use of partial clone (https://git-scm.com/' 'docs/gitrepository-layout#_code_partialclone_code)') + group.add_option('--partial-clone-exclude', action='store', + help='exclude the specified projects (a comma-delimited ' + 'project names) from partial clone (https://git-scm.com' + '/docs/gitrepository-layout#_code_partialclone_code)') group.add_option('--clone-filter', action='store', default='blob:none', help='filter for use with --partial-clone ' '[default: %default]') diff --git a/subcmds/init.py b/subcmds/init.py index 3566b8b6..a23e529d 100644 --- a/subcmds/init.py +++ b/subcmds/init.py @@ -227,6 +227,9 @@ to update the working directory files. else: opt.clone_filter = None + if opt.partial_clone_exclude is not None: + m.config.SetString('repo.partialcloneexclude', opt.partial_clone_exclude) + if opt.clone_bundle is None: opt.clone_bundle = False if opt.partial_clone else True else: @@ -242,7 +245,8 @@ to update the working directory files. clone_bundle=opt.clone_bundle, current_branch_only=opt.current_branch_only, tags=opt.tags, submodules=opt.submodules, - clone_filter=opt.clone_filter): + clone_filter=opt.clone_filter, + partial_clone_exclude=self.manifest.PartialCloneExclude): r = m.GetRemote(m.remote.name) print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr) diff --git a/subcmds/sync.py b/subcmds/sync.py index 4bcd45d5..b8abb1a7 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -348,7 +348,8 @@ later is required to fix a server side protocol bug. optimized_fetch=opt.optimized_fetch, retry_fetches=opt.retry_fetches, prune=opt.prune, - clone_filter=self.manifest.CloneFilter) + clone_filter=self.manifest.CloneFilter, + partial_clone_exclude=self.manifest.PartialCloneExclude) output = buf.getvalue() if opt.verbose and output: @@ -517,7 +518,6 @@ later is required to fix a server side protocol bug. if (not project.use_git_worktrees and len(project.manifest.GetProjectsWithName(project.name)) > 1): if not opt.quiet: - #pm.update(inc=0, msg='Shared project found') print('\r%s: Shared project %s found, disabling pruning.' % (project.relpath, project.name)) if git_require((2, 7, 0)): @@ -726,7 +726,8 @@ later is required to fix a server side protocol bug. optimized_fetch=opt.optimized_fetch, retry_fetches=opt.retry_fetches, submodules=self.manifest.HasSubmodules, - clone_filter=self.manifest.CloneFilter) + clone_filter=self.manifest.CloneFilter, + partial_clone_exclude=self.manifest.PartialCloneExclude) finish = time.time() self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK, start, finish, success)