project: mark gc.log as safe to discard when migrating .git/

This is just a log file that, while useful for humans when gc aborts,
doesn't contain any data, so it's safe to throw away.

Bug: https://crbug.com/gerrit/15619
Change-Id: Ia95e0e281f52260668f7a80b5d5f990e32a8597a
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/328999
Tested-by: Mike Frysinger <vapier@google.com>
Reviewed-by: Peter Kjellerstedt <peter.kjellerstedt@axis.com>
Reviewed-by: Mike Frysinger <vapier@google.com>

docs/internal-fs-layout.md

@@ -163,6 +163,7 @@ User controlled settings are initialized when running `repo init`.
 | repo.clonefilter         | `--clone-filter`          | Filter setting when using [partial git clones] |
 | repo.depth               | `--depth`                 | Create shallow checkouts when cloning |
 | repo.dissociate          | `--dissociate`            | Dissociate from any reference/mirrors after initial clone |
+| repo.git-lfs             | `--git-lfs`               | Enable [Git LFS] support |
 | repo.mirror              | `--mirror`                | Checkout is a repo mirror |
 | repo.partialclone        | `--partial-clone`         | Create [partial git clones] |
 | repo.partialcloneexclude | `--partial-clone-exclude` | Comma-delimited list of project names (not paths) to exclude while using [partial git clones] |
@@ -254,6 +255,7 @@ Repo will create & maintain a few files in the user's home directory.
 
 
 [git-config]: https://git-scm.com/docs/git-config
+[Git LFS]: https://git-lfs.github.com/
 [git worktree]: https://git-scm.com/docs/git-worktree
 [gitsubmodules]: https://git-scm.com/docs/gitsubmodules
 [manifest-format.md]: ./manifest-format.md

git_command.py

@@ -169,7 +169,8 @@ class GitCommand(object):
                disable_editor=False,
                ssh_proxy=None,
                cwd=None,
-               gitdir=None):
+               gitdir=None,
+               objdir=None):
     env = self._GetBasicEnv()
 
     if disable_editor:
@@ -194,13 +195,24 @@ class GitCommand(object):
         cwd = project.worktree
       if not gitdir:
         gitdir = project.gitdir
+    # Git on Windows wants its paths only using / for reliability.
+    if platform_utils.isWindows():
+      if objdir:
+        objdir = objdir.replace('\\', '/')
+      if gitdir:
+        gitdir = gitdir.replace('\\', '/')
+
+    if objdir:
+      # Set to the place we want to save the objects.
+      env['GIT_OBJECT_DIRECTORY'] = objdir
+      if gitdir:
+        # Allow git to search the original place in case of local or unique refs
+        # that git will attempt to resolve even if we aren't fetching them.
+        env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = gitdir + '/objects'
 
     command = [GIT]
     if bare:
       if gitdir:
-        # Git on Windows wants its paths only using / for reliability.
-        if platform_utils.isWindows():
-          gitdir = gitdir.replace('\\', '/')
         env[GIT_DIR] = gitdir
       cwd = None
     command.append(cmdv[0])
@@ -234,6 +246,11 @@ class GitCommand(object):
         dbg += ': export GIT_DIR=%s\n' % env[GIT_DIR]
         LAST_GITDIR = env[GIT_DIR]
 
+      if 'GIT_OBJECT_DIRECTORY' in env:
+        dbg += ': export GIT_OBJECT_DIRECTORY=%s\n' % env['GIT_OBJECT_DIRECTORY']
+      if 'GIT_ALTERNATE_OBJECT_DIRECTORIES' in env:
+        dbg += ': export GIT_ALTERNATE_OBJECT_DIRECTORIES=%s\n' % env['GIT_ALTERNATE_OBJECT_DIRECTORIES']
+
       dbg += ': '
       dbg += ' '.join(command)
       if stdin == subprocess.PIPE:

manifest_xml.py

@@ -666,6 +666,10 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
   def HasSubmodules(self):
     return self.manifestProject.config.GetBoolean('repo.submodules')
 
+  @property
+  def EnableGitLfs(self):
+    return self.manifestProject.config.GetBoolean('repo.git-lfs')
+
   def GetDefaultGroupsStr(self):
     """Returns the default group string for the platform."""
     return 'default,platform-' + platform.system().lower()

project.py

@@ -1120,7 +1120,7 @@ class Project(object):
     self._InitRemote()
 
     if is_new:
-      alt = os.path.join(self.gitdir, 'objects/info/alternates')
+      alt = os.path.join(self.objdir, 'objects/info/alternates')
       try:
         with open(alt) as fd:
           # This works for both absolute and relative alternate directories.
@@ -1169,7 +1169,7 @@ class Project(object):
     mp = self.manifest.manifestProject
     dissociate = mp.config.GetBoolean('repo.dissociate')
     if dissociate:
-      alternates_file = os.path.join(self.gitdir, 'objects/info/alternates')
+      alternates_file = os.path.join(self.objdir, 'objects/info/alternates')
       if os.path.exists(alternates_file):
         cmd = ['repack', '-a', '-d']
         p = GitCommand(self, cmd, bare=True, capture_stdout=bool(output_redir),
@@ -2192,8 +2192,10 @@ class Project(object):
     retry_cur_sleep = retry_sleep_initial_sec
     ok = prune_tried = False
     for try_n in range(retry_fetches):
-      gitcmd = GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy,
-                          merge_output=True, capture_stdout=quiet or bool(output_redir))
+      gitcmd = GitCommand(
+          self, cmd, bare=True, objdir=os.path.join(self.objdir, 'objects'),
+          ssh_proxy=ssh_proxy,
+          merge_output=True, capture_stdout=quiet or bool(output_redir))
       if gitcmd.stdout and not quiet and output_redir:
         output_redir.write(gitcmd.stdout)
       ret = gitcmd.Wait()
@@ -2309,7 +2311,8 @@ class Project(object):
       cmd.append(str(f))
     cmd.append('+refs/tags/*:refs/tags/*')
 
-    ok = GitCommand(self, cmd, bare=True).Wait() == 0
+    ok = GitCommand(
+        self, cmd, bare=True, objdir=os.path.join(self.objdir, 'objects')).Wait() == 0
     platform_utils.remove(bundle_dst, missing_ok=True)
     platform_utils.remove(bundle_tmp, missing_ok=True)
     return ok
@@ -2504,8 +2507,8 @@ class Project(object):
         if ref_dir or mirror_git:
           if not mirror_git:
             mirror_git = os.path.join(ref_dir, self.name + '.git')
-          repo_git = os.path.join(ref_dir, '.repo', 'projects',
-                                  self.relpath + '.git')
+          repo_git = os.path.join(ref_dir, '.repo', 'project-objects',
+                                  self.name + '.git')
           worktrees_git = os.path.join(ref_dir, '.repo', 'worktrees',
                                        self.name + '.git')
 
@@ -2523,15 +2526,16 @@ class Project(object):
               # The alternate directory is relative to the object database.
               ref_dir = os.path.relpath(ref_dir,
                                         os.path.join(self.objdir, 'objects'))
-            _lwrite(os.path.join(self.gitdir, 'objects/info/alternates'),
+            _lwrite(os.path.join(self.objdir, 'objects/info/alternates'),
                     os.path.join(ref_dir, 'objects') + '\n')
 
         m = self.manifest.manifestProject.config
         for key in ['user.name', 'user.email']:
           if m.Has(key, include_defaults=False):
             self.config.SetString(key, m.GetString(key))
-        self.config.SetString('filter.lfs.smudge', 'git-lfs smudge --skip -- %f')
-        self.config.SetString('filter.lfs.process', 'git-lfs filter-process --skip')
+        if not self.manifest.EnableGitLfs:
+          self.config.SetString('filter.lfs.smudge', 'git-lfs smudge --skip -- %f')
+          self.config.SetString('filter.lfs.process', 'git-lfs filter-process --skip')
         self.config.SetBoolean('core.bare', True if self.manifest.IsMirror else None)
     except Exception:
       if init_obj_dir and os.path.exists(self.objdir):
@@ -2551,7 +2555,10 @@ class Project(object):
 
     # Delete sample hooks.  They're noise.
     for hook in glob.glob(os.path.join(hooks, '*.sample')):
-      platform_utils.remove(hook, missing_ok=True)
+      try:
+        platform_utils.remove(hook, missing_ok=True)
+      except PermissionError:
+        pass
 
     for stock_hook in _ProjectHooks():
       name = os.path.basename(stock_hook)
@@ -2813,7 +2820,8 @@ class Project(object):
     }
     # Paths that we know will be in both, but are safe to clobber in .repo/projects/.
     SAFE_TO_CLOBBER = {
-        'COMMIT_EDITMSG', 'FETCH_HEAD', 'HEAD', 'gitk.cache', 'index', 'ORIG_HEAD',
+        'COMMIT_EDITMSG', 'FETCH_HEAD', 'HEAD', 'gc.log', 'gitk.cache', 'index',
+        'ORIG_HEAD',
     }
 
     # First see if we'd succeed before starting the migration.

repo

@@ -149,7 +149,7 @@ if not REPO_REV:
 BUG_URL = 'https://bugs.chromium.org/p/gerrit/issues/entry?template=Repo+tool+issue'
 
 # increment this whenever we make important changes to this script
-VERSION = (2, 17)
+VERSION = (2, 21)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (2, 3)
@@ -382,6 +382,11 @@ def InitParser(parser, gitc_init=False):
   group.add_option('--no-clone-bundle',
                    dest='clone_bundle', action='store_false',
                    help='disable use of /clone.bundle on HTTP/HTTPS (default if --partial-clone)')
+  group.add_option('--git-lfs', action='store_true',
+                   help='enable Git LFS support')
+  group.add_option('--no-git-lfs',
+                   dest='git_lfs', action='store_false',
+                   help='disable Git LFS support')
 
   # Tool.
   group = parser.add_option_group('repo Version options')
@@ -618,6 +623,7 @@ def _Init(args, gitc_init=False):
             "REPO_URL set correctly?" % url, file=sys.stderr)
 
   except CloneFailure:
+    print('fatal: double check your --repo-rev setting.', file=sys.stderr)
     if opt.quiet:
       print('fatal: repo init failed; run without --quiet to see why',
             file=sys.stderr)

subcmds/branches.py

@@ -151,7 +151,7 @@ is shown, then the branch appears in all projects.
         fmt = out.write
         paths = []
         non_cur_paths = []
-        if i.IsSplitCurrent or (in_cnt < project_cnt - in_cnt):
+        if i.IsSplitCurrent or (in_cnt <= project_cnt - in_cnt):
           in_type = 'in'
           for b in i.projects:
             if not i.IsSplitCurrent or b.current:
@@ -163,9 +163,9 @@ is shown, then the branch appears in all projects.
           in_type = 'not in'
           have = set()
           for b in i.projects:
-            have.add(b.project)
+            have.add(b.project.relpath)
           for p in projects:
-            if p not in have:
+            if p.relpath not in have:
               paths.append(p.relpath)
 
         s = ' %s %s' % (in_type, ', '.join(paths))

subcmds/init.py

@@ -291,6 +291,15 @@ to update the working directory files.
     if opt.submodules:
       m.config.SetBoolean('repo.submodules', opt.submodules)
 
+    if opt.git_lfs is not None:
+      if opt.git_lfs:
+        git_require((2, 17, 0), fail=True, msg='Git LFS support')
+
+      m.config.SetBoolean('repo.git-lfs', opt.git_lfs)
+      if not is_new:
+        print('warning: Changing --git-lfs settings will only affect new project checkouts.\n'
+              '         Existing projects will require manual updates.\n', file=sys.stderr)
+
     if opt.use_superproject is not None:
       m.config.SetBoolean('repo.superproject', opt.use_superproject)
 
@@ -520,8 +529,12 @@ to update the working directory files.
     # Handle new --repo-rev requests.
     if opt.repo_rev:
       wrapper = Wrapper()
-      remote_ref, rev = wrapper.check_repo_rev(
-          rp.gitdir, opt.repo_rev, repo_verify=opt.repo_verify, quiet=opt.quiet)
+      try:
+        remote_ref, rev = wrapper.check_repo_rev(
+            rp.gitdir, opt.repo_rev, repo_verify=opt.repo_verify, quiet=opt.quiet)
+      except wrapper.CloneFailure:
+        print('fatal: double check your --repo-rev setting.', file=sys.stderr)
+        sys.exit(1)
       branch = rp.GetBranch('default')
       branch.merge = remote_ref
       rp.work_git.reset('--hard', rev)

tests/test_project.py

@@ -382,7 +382,7 @@ class MigrateWorkTreeTests(unittest.TestCase):
 
       # Make sure the dir was transformed into a symlink.
       self.assertTrue(dotgit.is_symlink())
-      self.assertEqual(str(dotgit.readlink()), '../../.repo/projects/src/test.git')
+      self.assertEqual(os.readlink(dotgit), '../../.repo/projects/src/test.git')
 
       # Make sure files were moved over.
       gitdir = tempdir / '.repo/projects/src/test.git'