From 1d00a7e2ae64b6c08aff60c2e7ed5c2d89caf8d6 Mon Sep 17 00:00:00 2001 From: Mike Frysinger Date: Tue, 21 Dec 2021 00:40:31 -0500 Subject: [PATCH] project: initial separation of shared project objects For now, this is opt-in via environment variables: - export REPO_USE_ALTERNATES=1 The shared project logic that shares the internal .git/objects/ dir directly between multiple projects via the project-objects/ tree has a lot of KI with random corruption. It all boils down to projects sharing objects/ but not refs/. Git operations that use refs to see what objects are reachable and discard the rest can easily discard objects that are used by other projects. Consider this project layout: There are unique refs in each of these trees that are not visible in the others. This means it's not safe to run basic operations like git prune or git gc. Since we can't share refs (each project needs to have unique refs like HEAD in order to function), let's change how we share objects. The old way involved symlinking .git/objects/ to the project-objects tree. The new way shares objects using git's info/alternates. This means project-objects/ will only contain objects that exist in the remote project. Local per-project objects (like when creating branches and making changes) will never be shared. When running a prune or gc operation in the per-project state, it will only ever repack or discard those per-project objects. The common shared objects would only be cleaned up when running a common operation (i.e. by repo itself). One downside to this for users is if they try blending unrelated upstream projects. For example, in CrOS we have multiple kernel projects (for diff versions) checked out. If a dev fetched the upstream Linus tree into one of them, the objects & tags would not be shared with the others, so they would have to fetch the upstream state for each project. Annoying, but better than the current corruption situation we're in now. Also if the dev runs a manual `git fetch` in the per-project to sync it up to newer state than the last `repo sync` they ran, the objects would get duplicated. However, git operations later on should eventually dedupe this. Bug: https://crbug.com/gerrit/15553 Change-Id: I313a9b8962f9d439ef98ac0ed37ecfb9e0b3864e Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/328101 Reviewed-by: Mike Frysinger Tested-by: LaMont Jones --- manifest_xml.py | 9 ++++++--- project.py | 16 +++++++++++++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/manifest_xml.py b/manifest_xml.py index 3c43295e..db7a9286 100644 --- a/manifest_xml.py +++ b/manifest_xml.py @@ -768,8 +768,11 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md @property def is_multimanifest(self): - """Whether this is a multimanifest checkout""" - return bool(self.outer_client.submanifests) + """Whether this is a multimanifest checkout. + + This is safe to use as long as the outermost manifest XML has been parsed. + """ + return bool(self._outer_client._submanifests) @property def is_submanifest(self): @@ -1679,7 +1682,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md # We allow people to mix git worktrees & non-git worktrees for now. # This allows for in situ migration of repo clients. if os.path.exists(gitdir) or not self.UseGitWorktrees: - objdir = os.path.join(self.subdir, 'project-objects', namepath) + objdir = os.path.join(self.repodir, 'project-objects', namepath) else: use_git_worktrees = True gitdir = os.path.join(self.repodir, 'worktrees', namepath) diff --git a/project.py b/project.py index b7ed6f33..faa6b32b 100644 --- a/project.py +++ b/project.py @@ -49,6 +49,9 @@ MAXIMUM_RETRY_SLEEP_SEC = 3600.0 # +-10% random jitter is added to each Fetches retry sleep duration. RETRY_JITTER_PERCENT = 0.1 +# Whether to use alternates. +# TODO(vapier): Remove knob once behavior is verified. +_ALTERNATES = os.environ.get('REPO_USE_ALTERNATES') == '1' def _lwrite(path, content): lock = '%s.lock' % path @@ -460,7 +463,7 @@ class RemoteSpec(object): class Project(object): # These objects can be shared between several working trees. - shareable_dirs = ['hooks', 'objects', 'rr-cache'] + shareable_dirs = ['hooks', 'rr-cache'] def __init__(self, manifest, @@ -1143,6 +1146,17 @@ class Project(object): self._UpdateHooks(quiet=quiet) self._InitRemote() + if _ALTERNATES or self.manifest.is_multimanifest: + # If gitdir/objects is a symlink, migrate it from the old layout. + gitdir_objects = os.path.join(self.gitdir, 'objects') + if platform_utils.islink(gitdir_objects): + platform_utils.remove(gitdir_objects, missing_ok=True) + gitdir_alt = os.path.join(self.gitdir, 'objects/info/alternates') + if not os.path.exists(gitdir_alt): + os.makedirs(os.path.dirname(gitdir_alt), exist_ok=True) + _lwrite(gitdir_alt, os.path.join( + os.path.relpath(self.objdir, gitdir_objects), 'objects') + '\n') + if is_new: alt = os.path.join(self.objdir, 'objects/info/alternates') try: