android/git-repo
1
Fork 0
mirror of https://gerrit.googlesource.com/git-repo synced 2024-12-21 07:16:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

add experimental git worktree support

Browse Source 
This provides initial support for using git worktrees internally
instead of our own ad-hoc symlink tree.  It's been lightly tested
which is why it's not currently exposed via --help.

When people opt-in to worktrees in an existing repo client checkout,
no projects are migrated.  Instead, only new projects will use the
worktree method.  This allows for limited testing/opting in without
having to completely blow things away or get a second checkout.

Bug: https://crbug.com/gerrit/11486
Change-Id: Ic3ff891b30940a6ba497b406b2a387e0a8517ed8
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/254075
Tested-by: Mike Frysinger <vapier@google.com>
Reviewed-by: Mike Frysinger <vapier@google.com>
This commit is contained in:
Mike Frysinger 2020-02-09 02:28:34 -05:00
parent 56ce3468b4
commit 979d5bdc3e
6 changed files with 138 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/internal-fs-layout.md
View File

@ -102,6 +102,11 @@ support, see the [manifest-format.md] file.
respective servers ...
* `subprojects/`: Like `projects/`, but for git submodules.
* `subproject-objects/`: Like `project-objects/`, but for git submodules.
* `worktrees/`: Bare checkouts of every project synced by the manifest. The
filesystem layout matches the `<project name=...` setting in the manifest
(i.e. the path on the remote server).
This is used when git worktrees are enabled.
### Global settings
@ -121,6 +126,7 @@ User controlled settings are initialized when running `repo init`.
| repo.partialclone | `--partial-clone` | Create [partial git clones] |
| repo.reference | `--reference` | Reference repo client checkout |
| repo.submodules | `--submodules` | Sync git submodules |
| repo.worktree | `--worktree` | Use `git worktree` for checkouts |
| user.email | `--config-name` | User's e-mail address; Copied into `.git/config` when checking out a new project |
| user.name | `--config-name` | User's name; Copied into `.git/config` when checking out a new project |

35
manifest_xml.py
View File

@ -146,9 +146,17 @@ class XmlManifest(object):
gitdir=os.path.join(repodir, 'repo/.git'),
worktree=os.path.join(repodir, 'repo'))
self.manifestProject = MetaProject(self, 'manifests',
gitdir=os.path.join(repodir, 'manifests.git'),
worktree=os.path.join(repodir, 'manifests'))
mp = MetaProject(self, 'manifests',
gitdir=os.path.join(repodir, 'manifests.git'),
worktree=os.path.join(repodir, 'manifests'))
self.manifestProject = mp
# This is a bit hacky, but we're in a chicken & egg situation: all the
# normal repo settings live in the manifestProject which we just setup
# above, so we couldn't easily query before that. We assume Project()
# init doesn't care if this changes afterwards.
if mp.config.GetBoolean('repo.worktree'):
mp.use_git_worktrees = True
self._Unload()
@ -427,6 +435,10 @@ class XmlManifest(object):
def IsMirror(self):
return self.manifestProject.config.GetBoolean('repo.mirror')
@property
def UseGitWorktrees(self):
return self.manifestProject.config.GetBoolean('repo.worktree')
@property
def IsArchive(self):
return self.manifestProject.config.GetBoolean('repo.archive')
@ -873,8 +885,10 @@ class XmlManifest(object):
groups = self._ParseGroups(groups)
if parent is None:
relpath, worktree, gitdir, objdir = self.GetProjectPaths(name, path)
relpath, worktree, gitdir, objdir, use_git_worktrees = \
self.GetProjectPaths(name, path)
else:
use_git_worktrees = False
relpath, worktree, gitdir, objdir = \
self.GetSubprojectPaths(parent, name, path)
@ -903,6 +917,7 @@ class XmlManifest(object):
upstream=upstream,
parent=parent,
dest_branch=dest_branch,
use_git_worktrees=use_git_worktrees,
**extra_proj_attrs)
for n in node.childNodes:
@ -918,6 +933,7 @@ class XmlManifest(object):
return project
def GetProjectPaths(self, name, path):
use_git_worktrees = False
relpath = path
if self.IsMirror:
worktree = None
@ -926,8 +942,15 @@ class XmlManifest(object):
else:
worktree = os.path.join(self.topdir, path).replace('\\', '/')
gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path)
objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name)
return relpath, worktree, gitdir, objdir
# We allow people to mix git worktrees & non-git worktrees for now.
# This allows for in situ migration of repo clients.
if os.path.exists(gitdir) or not self.UseGitWorktrees:
objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name)
else:
use_git_worktrees = True
gitdir = os.path.join(self.repodir, 'worktrees', '%s.git' % name)
objdir = gitdir
return relpath, worktree, gitdir, objdir, use_git_worktrees
def GetProjectsWithName(self, name):
return self._projects.get(name, [])

70
project.py
View File

@ -866,6 +866,7 @@ class Project(object):
clone_depth=None,
upstream=None,
parent=None,
use_git_worktrees=False,
is_derived=False,
dest_branch=None,
optimized_fetch=False,
@ -889,6 +890,7 @@ class Project(object):
sync_tags: The `sync-tags` attribute of manifest.xml's project element.
upstream: The `upstream` attribute of manifest.xml's project element.
parent: The parent Project object.
use_git_worktrees: Whether to use `git worktree` for this project.
is_derived: False if the project was explicitly defined in the manifest;
True if the project is a discovered submodule.
dest_branch: The branch to which to push changes for review by default.
@ -923,6 +925,10 @@ class Project(object):
self.clone_depth = clone_depth
self.upstream = upstream
self.parent = parent
# NB: Do not use this setting in __init__ to change behavior so that the
# manifest.git checkout can inspect & change it after instantiating. See
# the XmlManifest init code for more info.
self.use_git_worktrees = use_git_worktrees
self.is_derived = is_derived
self.optimized_fetch = optimized_fetch
self.subprojects = []
@ -1872,15 +1878,19 @@ class Project(object):
except KeyError:
head = None
if revid and head and revid == head:
ref = os.path.join(self.gitdir, R_HEADS + name)
try:
os.makedirs(os.path.dirname(ref))
except OSError:
pass
_lwrite(ref, '%s\n' % revid)
_lwrite(self.GetHeadPath(), 'ref: %s%s\n' % (R_HEADS, name))
branch.Save()
return True
if self.use_git_worktrees:
self.work_git.update_ref(HEAD, revid)
branch.Save()
else:
ref = os.path.join(self.gitdir, R_HEADS + name)
try:
os.makedirs(os.path.dirname(ref))
except OSError:
pass
_lwrite(ref, '%s\n' % revid)
_lwrite(self.GetHeadPath(), 'ref: %s%s\n' % (R_HEADS, name))
branch.Save()
return True
if GitCommand(self,
['checkout', '-b', branch.name, revid],
@ -2617,6 +2627,11 @@ class Project(object):
os.makedirs(self.objdir)
self.bare_objdir.init()
# Enable per-worktree config file support if possible. This is more a
# nice-to-have feature for users rather than a hard requirement.
if self.use_git_worktrees and git_require((2, 19, 0)):
self.config.SetString('extensions.worktreeConfig', 'true')
# If we have a separate directory to hold refs, initialize it as well.
if self.objdir != self.gitdir:
if init_git_dir:
@ -2651,13 +2666,15 @@ class Project(object):
mirror_git = os.path.join(ref_dir, self.name + '.git')
repo_git = os.path.join(ref_dir, '.repo', 'projects',
self.relpath + '.git')
worktrees_git = os.path.join(ref_dir, '.repo', 'worktrees',
self.name + '.git')
if os.path.exists(mirror_git):
ref_dir = mirror_git
elif os.path.exists(repo_git):
ref_dir = repo_git
elif os.path.exists(worktrees_git):
ref_dir = worktrees_git
else:
ref_dir = None
@ -2765,6 +2782,10 @@ class Project(object):
self.bare_git.symbolic_ref('-m', msg, ref, dst)
def _CheckDirReference(self, srcdir, destdir, share_refs):
# Git worktrees don't use symlinks to share at all.
if self.use_git_worktrees:
return
symlink_files = self.shareable_files[:]
symlink_dirs = self.shareable_dirs[:]
if share_refs:
@ -2864,11 +2885,38 @@ class Project(object):
else:
raise
def _InitGitWorktree(self):
"""Init the project using git worktrees."""
self.bare_git.worktree('prune')
self.bare_git.worktree('add', '-ff', '--checkout', '--detach', '--lock',
self.worktree, self.GetRevisionId())
# Rewrite the internal state files to use relative paths between the
# checkouts & worktrees.
dotgit = os.path.join(self.worktree, '.git')
with open(dotgit, 'r') as fp:
# Figure out the checkout->worktree path.
setting = fp.read()
assert setting.startswith('gitdir:')
git_worktree_path = setting.split(':', 1)[1].strip()
# Use relative path from checkout->worktree.
with open(dotgit, 'w') as fp:
print('gitdir:', os.path.relpath(git_worktree_path, self.worktree),
file=fp)
# Use relative path from worktree->checkout.
with open(os.path.join(git_worktree_path, 'gitdir'), 'w') as fp:
print(os.path.relpath(dotgit, git_worktree_path), file=fp)
def _InitWorkTree(self, force_sync=False, submodules=False):
realdotgit = os.path.join(self.worktree, '.git')
tmpdotgit = realdotgit + '.tmp'
init_dotgit = not os.path.exists(realdotgit)
if init_dotgit:
if self.use_git_worktrees:
self._InitGitWorktree()
self._CopyAndLinkFiles()
return
dotgit = tmpdotgit
platform_utils.rmtree(tmpdotgit, ignore_errors=True)
os.makedirs(tmpdotgit)

2
repo
View File

@ -302,6 +302,8 @@ def GetParser(gitc_init=False):
group.add_option('--clone-filter', action='store', default='blob:none',
help='filter for use with --partial-clone '
'[default: %default]')
group.add_option('--worktree', action='store_true',
help=optparse.SUPPRESS_HELP)
group.add_option('--archive', action='store_true',
help='checkout an archive instead of a git repository for '
'each project. See git archive.')

24
subcmds/init.py
View File

@ -15,6 +15,8 @@
# limitations under the License.
from __future__ import print_function
import optparse
import os
import platform
import re
@ -128,6 +130,10 @@ to update the working directory files.
g.add_option('--clone-filter', action='store', default='blob:none',
dest='clone_filter',
help='filter for use with --partial-clone [default: %default]')
# TODO(vapier): Expose option with real help text once this has been in the
# wild for a while w/out significant bug reports. Goal is by ~Sep 2020.
g.add_option('--worktree', action='store_true',
help=optparse.SUPPRESS_HELP)
g.add_option('--archive',
dest='archive', action='store_true',
help='checkout an archive instead of a git repository for '
@ -246,6 +252,20 @@ to update the working directory files.
if opt.dissociate:
m.config.SetString('repo.dissociate', 'true')
if opt.worktree:
if opt.mirror:
print('fatal: --mirror and --worktree are incompatible',
file=sys.stderr)
sys.exit(1)
if opt.submodules:
print('fatal: --submodules and --worktree are incompatible',
file=sys.stderr)
sys.exit(1)
m.config.SetString('repo.worktree', 'true')
if is_new:
m.use_git_worktrees = True
print('warning: --worktree is experimental!', file=sys.stderr)
if opt.archive:
if is_new:
m.config.SetString('repo.archive', 'true')
@ -459,6 +479,10 @@ to update the working directory files.
% ('.'.join(str(x) for x in MIN_GIT_VERSION_SOFT),),
file=sys.stderr)
if opt.worktree:
# Older versions of git supported worktree, but had dangerous gc bugs.
git_require((2, 15, 0), fail=True, msg='git gc worktree corruption')
self._SyncManifest(opt)
self._LinkManifest(opt.manifest_name)

24
subcmds/sync.py
View File

@ -15,6 +15,8 @@
# limitations under the License.
from __future__ import print_function
import errno
import json
import netrc
from optparse import SUPPRESS_HELP
@ -569,7 +571,8 @@ later is required to fix a server side protocol bug.
gc_gitdirs = {}
for project in projects:
# Make sure pruning never kicks in with shared projects.
if len(project.manifest.GetProjectsWithName(project.name)) > 1:
if (not project.use_git_worktrees and
len(project.manifest.GetProjectsWithName(project.name)) > 1):
print('%s: Shared project %s found, disabling pruning.' %
(project.relpath, project.name))
if git_require((2, 7, 0)):
@ -637,13 +640,22 @@ later is required to fix a server side protocol bug.
# Delete the .git directory first, so we're less likely to have a partially
# working git repository around. There shouldn't be any git projects here,
# so rmtree works.
dotgit = os.path.join(path, '.git')
# Try to remove plain files first in case of git worktrees. If this fails
# for any reason, we'll fall back to rmtree, and that'll display errors if
# it can't remove things either.
try:
platform_utils.rmtree(os.path.join(path, '.git'))
platform_utils.remove(dotgit)
except OSError:
pass
try:
platform_utils.rmtree(dotgit)
except OSError as e:
print('Failed to remove %s (%s)' % (os.path.join(path, '.git'), str(e)), file=sys.stderr)
print('error: Failed to delete obsolete path %s' % path, file=sys.stderr)
print(' remove manually, then run sync again', file=sys.stderr)
return 1
if e.errno != errno.ENOENT:
print('error: %s: %s' % (dotgit, str(e)), file=sys.stderr)
print('error: %s: Failed to delete obsolete path; remove manually, then '
'run sync again' % (path,), file=sys.stderr)
return 1
# Delete everything under the worktree, except for directories that contain
# another git project