project: migrate worktree .git/ dirs to symlinks

Historically we created a .git/ subdir in each source checkout and
symlinked individual files to the .repo/projects/ paths.  This layer
of indirection isn't actually needed: the .repo/projects/ paths are
guaranteed to only ever have a 1-to-1 mapping with the actual git
checkout.  So we don't need to worry about having files in .git/ be
isolated.

To that end, change how we manage the actual project checkouts from
a dir full of symlinks (and a few files) to a symlink to the internal
.repo/projects/ dir.  This makes the code simpler & faster.

The directory structure we have today is:
.repo/
  project-objects/chromiumos/third_party/kernel.git/
    <paths omitted as not relevant to this change>
  projects/src/third_party/kernel/
    v3.8.git/
      config
      description   -> …/project-objects/…/config
      FETCH_HEAD
      HEAD
      hooks/        -> …/project-objects/…/hooks/
      info/         -> …/project-objects/…/info/
      logs/
      objects/      -> …/project-objects/…/objects/
      packed-refs
      refs/
      rr-cache/     -> …/project-objects/…/rr-cache/
src/third_party/kernel/
  v3.8/
    .git/
      config        -> …/projects/…/v3.8.git/config
      description   -> …/project-objects/…/v3.8.git/description
      HEAD
      hooks/        -> …/project-objects/…/v3.8.git/hooks/
      index
      info/         -> …/project-objects/…/v3.8.git/info/
      logs/         -> …/projects/…/v3.8.git/logs/
      objects/      -> …/project-objects/…/v3.8.git/objects/
      packed-refs   -> …/projects/…/v3.8.git/packed-refs
      refs/         -> …/projects/…/v3.8.git/refs/
      rr-cache/     -> …/project-objects/…/v3.8.git/rr-cache/

The directory structure we have after this commit:
.repo/
  <nothing changes>
src/third_party/kernel/
  v3.8/
    .git            -> …/projects/…/v3.8.git

Bug: https://crbug.com/gerrit/15273
Change-Id: I9dd8def23fbfb2f4cb209a93f8b1b2b24002a444
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/323695
Reviewed-by: Mike Nichols <mikenichols@google.com>
Reviewed-by: Xin Li <delphij@google.com>
Tested-by: Mike Frysinger <vapier@google.com>

.gitreview

@@ -0,0 +1,5 @@
+[gerrit]
+host=gerrit-review.googlesource.com
+scheme=https
+project=git-repo.git
+defaultbranch=main

SUBMITTING_PATCHES.md

@@ -3,7 +3,7 @@
 # Short Version
 
  - Make small logical changes.
- - Provide a meaningful commit message.
+ - [Provide a meaningful commit message][commit-message-style].
  - Check for coding errors and style nits with flake8.
  - Make sure all code is under the Apache License, 2.0.
  - Publish your changes for review.
@@ -26,10 +26,11 @@ yourself with the following relevant bits.
 
 ## Make separate commits for logically separate changes.
 
-Unless your patch is really trivial, you should not be sending
+Unless your patch is really trivial, you should not be sending out a patch that
-out a patch that was generated between your working tree and your
+was generated between your working tree and your commit head.
-commit head.  Instead, always make a commit with complete commit
+Instead, always make a commit with a complete
-message and generate a series of patches from your repository.
+[commit message][commit-message-style] and generate a series of patches from
+your repository.
 It is a good discipline.
 
 Describe the technical detail of the change(s).
@@ -171,3 +172,6 @@ After you receive a Code-Review+2 from the maintainer, select the Verified
 button on the gerrit page for the change. This verifies that you have tested
 your changes and notifies the maintainer that they are ready to be submitted.
 The maintainer will then submit your changes to the repository.
+
+
+[commit-message-style]: https://chris.beams.io/posts/git-commit/

fetch.py

@@ -17,8 +17,10 @@
 import subprocess
 import sys
 from urllib.parse import urlparse
+from urllib.request import urlopen
 
-def fetch_file(url):
+
+def fetch_file(url, verbose=False):
   """Fetch a file from the specified source using the appropriate protocol.
 
   Returns:
@@ -29,13 +31,15 @@ def fetch_file(url):
     cmd = ['gsutil', 'cat', url]
     try:
       result = subprocess.run(
-          cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+          cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+          check=True)
+      if result.stderr and verbose:
+        print('warning: non-fatal error running "gsutil": %s' % result.stderr,
+              file=sys.stderr)
       return result.stdout
     except subprocess.CalledProcessError as e:
-      print('fatal: error running "gsutil": %s' % e.output,
+      print('fatal: error running "gsutil": %s' % e.stderr,
             file=sys.stderr)
     sys.exit(1)
-  if scheme == 'file':
+  with urlopen(url) as f:
-    with open(url[len('file://'):], 'rb') as f:
     return f.read()
-  raise ValueError('unsupported url %s' % url)

git_config.py

@@ -352,7 +352,7 @@ class GitConfig(object):
       Trace(': parsing %s', self.file)
       with open(self._json) as fd:
         return json.load(fd)
-    except (IOError, ValueErrorl):
+    except (IOError, ValueError):
       platform_utils.remove(self._json, missing_ok=True)
       return None
 

man/repo-gitc-init.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "September 2021" "repo gitc-init" "Repo Manual"
+.TH REPO "1" "November 2021" "repo gitc-init" "Repo Manual"
 .SH NAME
 repo \- repo gitc-init - manual page for repo gitc-init
 .SH SYNOPSIS
@@ -31,10 +31,6 @@ manifest branch or revision (use HEAD for default)
 \fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
 initial manifest file
 .TP
-\fB\-\-standalone\-manifest\fR
-download the manifest as a static file rather then
-create a git checkout of the manifest repo
-.TP
 \fB\-g\fR GROUP, \fB\-\-groups\fR=\fI\,GROUP\/\fR
 restrict manifest projects to ones with specified
 group(s) [default|all|G1,G2,G3|G4,\-G5,\-G6]
@@ -45,6 +41,10 @@ platform group [auto|all|none|linux|darwin|...]
 .TP
 \fB\-\-submodules\fR
 sync any submodules associated with the manifest repo
+.TP
+\fB\-\-standalone\-manifest\fR
+download the manifest as a static file rather then
+create a git checkout of the manifest repo
 .SS Manifest (only) checkout options:
 .TP
 \fB\-\-current\-branch\fR
@@ -96,7 +96,8 @@ filter for use with \fB\-\-partial\-clone\fR [default:
 blob:none]
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects

man/repo-init.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "September 2021" "repo init" "Repo Manual"
+.TH REPO "1" "November 2021" "repo init" "Repo Manual"
 .SH NAME
 repo \- repo init - manual page for repo init
 .SH SYNOPSIS
@@ -31,10 +31,6 @@ manifest branch or revision (use HEAD for default)
 \fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
 initial manifest file
 .TP
-\fB\-\-standalone\-manifest\fR
-download the manifest as a static file rather then
-create a git checkout of the manifest repo
-.TP
 \fB\-g\fR GROUP, \fB\-\-groups\fR=\fI\,GROUP\/\fR
 restrict manifest projects to ones with specified
 group(s) [default|all|G1,G2,G3|G4,\-G5,\-G6]
@@ -45,6 +41,10 @@ platform group [auto|all|none|linux|darwin|...]
 .TP
 \fB\-\-submodules\fR
 sync any submodules associated with the manifest repo
+.TP
+\fB\-\-standalone\-manifest\fR
+download the manifest as a static file rather then
+create a git checkout of the manifest repo
 .SS Manifest (only) checkout options:
 .TP
 \fB\-c\fR, \fB\-\-current\-branch\fR
@@ -96,7 +96,8 @@ filter for use with \fB\-\-partial\-clone\fR [default:
 blob:none]
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects

man/repo-manifest.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo manifest" "Repo Manual"
+.TH REPO "1" "November 2021" "repo manifest" "Repo Manual"
 .SH NAME
 repo \- repo manifest - manual page for repo manifest
 .SH SYNOPSIS
@@ -161,6 +161,7 @@ CDATA #IMPLIED>
 <!ELEMENT extend\-project EMPTY>
 <!ATTLIST extend\-project name CDATA #REQUIRED>
 <!ATTLIST extend\-project path CDATA #IMPLIED>
+<!ATTLIST extend\-project dest\-path CDATA #IMPLIED>
 <!ATTLIST extend\-project groups CDATA #IMPLIED>
 <!ATTLIST extend\-project revision CDATA #IMPLIED>
 <!ATTLIST extend\-project remote CDATA #IMPLIED>
@@ -176,6 +177,7 @@ CDATA #IMPLIED>
 <!ELEMENT superproject EMPTY>
 <!ATTLIST superproject name     CDATA #REQUIRED>
 <!ATTLIST superproject remote   IDREF #IMPLIED>
+<!ATTLIST superproject revision CDATA #IMPLIED>
 .IP
 <!ELEMENT contactinfo EMPTY>
 <!ATTLIST contactinfo bugurl  CDATA #REQUIRED>
@@ -385,6 +387,11 @@ original manifest.
 Attribute `path`: If specified, limit the change to projects checked out at the
 specified path, rather than all projects with the given name.
 .PP
+Attribute `dest\-path`: If specified, a path relative to the top directory of the
+repo client where the Git working directory for this project should be placed.
+This is used to move a project in the checkout by overriding the existing `path`
+setting.
+.PP
 Attribute `groups`: List of additional groups to which this project belongs.
 Same syntax as the corresponding element of `project`.
 .PP
@@ -477,6 +484,10 @@ project](#element\-project) for more information.
 Attribute `remote`: Name of a previously defined remote element. If not supplied
 the remote given by the default element is used.
 .PP
+Attribute `revision`: Name of the Git branch the manifest wants to track for
+this superproject. If not supplied the revision given by the remote element is
+used if applicable, else the default element is used.
+.PP
 Element contactinfo
 .PP
 *** *Note*: This is currently a WIP. ***

man/repo-smartsync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo smartsync" "Repo Manual"
+.TH REPO "1" "November 2021" "repo smartsync" "Repo Manual"
 .SH NAME
 repo \- repo smartsync - manual page for repo smartsync
 .SH SYNOPSIS
@@ -80,7 +80,8 @@ password to authenticate with the manifest server
 fetch submodules from server
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects
@@ -89,7 +90,7 @@ disable use of manifest superprojects
 fetch tags
 .TP
 \fB\-\-no\-tags\fR
-don't fetch tags
+don't fetch tags (default)
 .TP
 \fB\-\-optimized\-fetch\fR
 only fetch projects fixed to sha1 if revision does not
@@ -100,6 +101,10 @@ number of times to retry fetches on transient errors
 .TP
 \fB\-\-prune\fR
 delete refs that no longer exist on the remote
+(default)
+.TP
+\fB\-\-no\-prune\fR
+do not delete refs that no longer exist on the remote
 .SS Logging options:
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR

man/repo-sync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo sync" "Repo Manual"
+.TH REPO "1" "November 2021" "repo sync" "Repo Manual"
 .SH NAME
 repo \- repo sync - manual page for repo sync
 .SH SYNOPSIS
@@ -80,7 +80,8 @@ password to authenticate with the manifest server
 fetch submodules from server
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects
@@ -89,7 +90,7 @@ disable use of manifest superprojects
 fetch tags
 .TP
 \fB\-\-no\-tags\fR
-don't fetch tags
+don't fetch tags (default)
 .TP
 \fB\-\-optimized\-fetch\fR
 only fetch projects fixed to sha1 if revision does not
@@ -100,6 +101,10 @@ number of times to retry fetches on transient errors
 .TP
 \fB\-\-prune\fR
 delete refs that no longer exist on the remote
+(default)
+.TP
+\fB\-\-no\-prune\fR
+do not delete refs that no longer exist on the remote
 .TP
 \fB\-s\fR, \fB\-\-smart\-sync\fR
 smart sync using manifest from the latest known good

man/repo.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo" "Repo Manual"
+.TH REPO "1" "November 2021" "repo" "Repo Manual"
 .SH NAME
 repo \- repository management tool built on top of git
 .SH SYNOPSIS
@@ -43,7 +43,7 @@ filename of event log to append timeline to
 .TP
 \fB\-\-git\-trace2\-event\-log\fR=\fI\,GIT_TRACE2_EVENT_LOG\/\fR
 directory to write git trace2 event log to
-.SS "The complete list of recognized repo commands are:"
+.SS "The complete list of recognized repo commands is:"
 .TP
 abandon
 Permanently abandon a development branch

project.py

@@ -2044,8 +2044,11 @@ class Project(object):
 
     if current_branch_only:
       if self.revisionExpr.startswith(R_TAGS):
-        # this is a tag and its sha1 value should never change
+        # This is a tag and its commit id should never change.
         tag_name = self.revisionExpr[len(R_TAGS):]
+      elif self.upstream and self.upstream.startswith(R_TAGS):
+        # This is a tag and its commit id should never change.
+        tag_name = self.upstream[len(R_TAGS):]
 
       if is_sha1 or tag_name is not None:
         if self._CheckForImmutableRevision():
@@ -2440,7 +2443,7 @@ class Project(object):
     if quiet:
       cmd.append('-q')
     if GitCommand(self, cmd).Wait() != 0:
-      raise GitError('%s submodule update --init --recursive %s ' % self.name)
+      raise GitError('%s submodule update --init --recursive ' % self.name)
 
   def _Rebase(self, upstream, onto=None):
     cmd = ['rebase']
@@ -2466,6 +2469,8 @@ class Project(object):
         os.makedirs(self.objdir)
         self.bare_objdir.init()
 
+        self._UpdateHooks(quiet=quiet)
+
         if self.use_git_worktrees:
           # Enable per-worktree config file support if possible.  This is more a
           # nice-to-have feature for users rather than a hard requirement.
@@ -2526,8 +2531,6 @@ class Project(object):
             _lwrite(os.path.join(self.gitdir, 'objects/info/alternates'),
                     os.path.join(ref_dir, 'objects') + '\n')
 
-        self._UpdateHooks(quiet=quiet)
-
         m = self.manifest.manifestProject.config
         for key in ['user.name', 'user.email']:
           if m.Has(key, include_defaults=False):
@@ -2543,11 +2546,11 @@ class Project(object):
       raise
 
   def _UpdateHooks(self, quiet=False):
-    if os.path.exists(self.gitdir):
+    if os.path.exists(self.objdir):
       self._InitHooks(quiet=quiet)
 
   def _InitHooks(self, quiet=False):
-    hooks = platform_utils.realpath(self._gitdir_path('hooks'))
+    hooks = platform_utils.realpath(os.path.join(self.objdir, 'hooks'))
     if not os.path.exists(hooks):
       os.makedirs(hooks)
     for stock_hook in _ProjectHooks():
@@ -2778,44 +2781,45 @@ class Project(object):
     self._InitMRef()
 
   def _InitWorkTree(self, force_sync=False, submodules=False):
-    realdotgit = os.path.join(self.worktree, '.git')
+    """Setup the worktree .git path.
-    tmpdotgit = realdotgit + '.tmp'
+
-    init_dotgit = not os.path.exists(realdotgit)
+    This is the user-visible path like src/foo/.git/.
-    if init_dotgit:
+
+    With non-git-worktrees, this will be a symlink to the .repo/projects/ path.
+    With git-worktrees, this will be a .git file using "gitdir: ..." syntax.
+
+    Older checkouts had .git/ directories.  If we see that, migrate it.
+
+    This also handles changes in the manifest.  Maybe this project was backed
+    by "foo/bar" on the server, but now it's "new/foo/bar".  We have to update
+    the path we point to under .repo/projects/ to match.
+    """
+    dotgit = os.path.join(self.worktree, '.git')
+
+    # If using an old layout style (a directory), migrate it.
+    if not platform_utils.islink(dotgit) and platform_utils.isdir(dotgit):
+      self._MigrateOldWorkTreeGitDir(dotgit)
+
+    init_dotgit = not os.path.exists(dotgit)
     if self.use_git_worktrees:
+      if init_dotgit:
         self._InitGitWorktree()
         self._CopyAndLinkFiles()
-        return
-
-      dotgit = tmpdotgit
-      platform_utils.rmtree(tmpdotgit, ignore_errors=True)
-      os.makedirs(tmpdotgit)
-      self._ReferenceGitDir(self.gitdir, tmpdotgit, share_refs=True,
-                            copy_all=False)
     else:
-      dotgit = realdotgit
+      if not init_dotgit:
+        # See if the project has changed.
+        if platform_utils.realpath(self.gitdir) != platform_utils.realpath(dotgit):
+          platform_utils.remove(dotgit)
 
-    try:
+      if init_dotgit or not os.path.exists(dotgit):
-      self._CheckDirReference(self.gitdir, dotgit, share_refs=True)
+        os.makedirs(self.worktree, exist_ok=True)
-    except GitError as e:
+        platform_utils.symlink(os.path.relpath(self.gitdir, self.worktree), dotgit)
-      if force_sync and not init_dotgit:
-        try:
-          platform_utils.rmtree(dotgit)
-          return self._InitWorkTree(force_sync=False, submodules=submodules)
-        except Exception:
-          raise e
-      raise e
 
       if init_dotgit:
-      _lwrite(os.path.join(tmpdotgit, HEAD), '%s\n' % self.GetRevisionId())
+        _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
-
-      # Now that the .git dir is fully set up, move it to its final home.
-      platform_utils.rename(tmpdotgit, realdotgit)
 
         # Finish checking out the worktree.
-      cmd = ['read-tree', '--reset', '-u']
+        cmd = ['read-tree', '--reset', '-u', '-v', HEAD]
-      cmd.append('-v')
-      cmd.append(HEAD)
         if GitCommand(self, cmd).Wait() != 0:
           raise GitError('Cannot initialize work tree for ' + self.name)
 
@@ -2823,6 +2827,50 @@ class Project(object):
           self._SyncSubmodules(quiet=True)
         self._CopyAndLinkFiles()
 
+  @classmethod
+  def _MigrateOldWorkTreeGitDir(cls, dotgit):
+    """Migrate the old worktree .git/ dir style to a symlink.
+
+    This logic specifically only uses state from |dotgit| to figure out where to
+    move content and not |self|.  This way if the backing project also changed
+    places, we only do the .git/ dir to .git symlink migration here.  The path
+    updates will happen independently.
+    """
+    # Figure out where in .repo/projects/ it's pointing to.
+    if not os.path.islink(os.path.join(dotgit, 'refs')):
+      raise GitError(f'{dotgit}: unsupported checkout state')
+    gitdir = os.path.dirname(os.path.realpath(os.path.join(dotgit, 'refs')))
+
+    # Remove known symlink paths that exist in .repo/projects/.
+    KNOWN_LINKS = {
+        'config', 'description', 'hooks', 'info', 'logs', 'objects',
+        'packed-refs', 'refs', 'rr-cache', 'shallow', 'svn',
+    }
+    # Paths that we know will be in both, but are safe to clobber in .repo/projects/.
+    SAFE_TO_CLOBBER = {
+        'COMMIT_EDITMSG', 'FETCH_HEAD', 'HEAD', 'index', 'ORIG_HEAD',
+    }
+
+    # Now walk the paths and sync the .git/ to .repo/projects/.
+    for name in platform_utils.listdir(dotgit):
+      dotgit_path = os.path.join(dotgit, name)
+      if name in KNOWN_LINKS:
+        if platform_utils.islink(dotgit_path):
+          platform_utils.remove(dotgit_path)
+        else:
+          raise GitError(f'{dotgit_path}: should be a symlink')
+      else:
+        gitdir_path = os.path.join(gitdir, name)
+        if name in SAFE_TO_CLOBBER or not os.path.exists(gitdir_path):
+          platform_utils.remove(gitdir_path, missing_ok=True)
+          platform_utils.rename(dotgit_path, gitdir_path)
+        else:
+          raise GitError(f'{dotgit_path}: unknown file; please file a bug')
+
+    # Now that the dir should be empty, clear it out, and symlink it over.
+    platform_utils.rmdir(dotgit)
+    platform_utils.symlink(os.path.relpath(gitdir, os.path.dirname(dotgit)), dotgit)
+
   def _get_symlink_error_message(self):
     if platform_utils.isWindows():
       return ('Unable to create symbolic link. Please re-run the command as '
@@ -2831,9 +2879,6 @@ class Project(object):
               'for other options.')
     return 'filesystem must support symlinks'
 
-  def _gitdir_path(self, path):
-    return platform_utils.realpath(os.path.join(self.gitdir, path))
-
   def _revlist(self, *args, **kw):
     a = []
     a.extend(args)

repo

@@ -372,7 +372,7 @@ def InitParser(parser, gitc_init=False):
                    help='filter for use with --partial-clone '
                         '[default: %default]')
   group.add_option('--use-superproject', action='store_true', default=None,
-                   help='use the manifest superproject to sync projects')
+                   help='use the manifest superproject to sync projects; implies -c')
   group.add_option('--no-use-superproject', action='store_false',
                    dest='use_superproject',
                    help='disable use of manifest superprojects')

ssh.py

@@ -52,6 +52,9 @@ def version():
   """return ssh version as a tuple"""
   try:
     return _parse_ssh_version()
+  except FileNotFoundError:
+    print('fatal: ssh not installed', file=sys.stderr)
+    sys.exit(1)
   except subprocess.CalledProcessError:
     print('fatal: unable to detect ssh version', file=sys.stderr)
     sys.exit(1)

subcmds/help.py

@@ -53,7 +53,7 @@ Displays detailed usage information about a command.
     self.PrintAllCommandsBody()
 
   def PrintAllCommandsBody(self):
-    print('The complete list of recognized repo commands are:')
+    print('The complete list of recognized repo commands is:')
     commandNames = list(sorted(all_commands))
     self._PrintCommands(commandNames)
     print("See 'repo help <command>' for more information on a "

subcmds/init.py

@@ -15,7 +15,6 @@
 import os
 import platform
 import re
-import subprocess
 import sys
 import urllib.parse
 
@@ -132,8 +131,8 @@ to update the working directory files.
               'cannot be re-initialized without --manifest-url/-u')
         sys.exit(1)
 
-      if opt.standalone_manifest or (
+      if opt.standalone_manifest or (was_standalone_manifest and
-          was_standalone_manifest and opt.manifest_url):
+                                     opt.manifest_url):
         m.config.ClearCache()
         if m.gitdir and os.path.exists(m.gitdir):
           platform_utils.rmtree(m.gitdir)
@@ -298,7 +297,7 @@ to update the working directory files.
     if standalone_manifest:
       if is_new:
         manifest_name = 'default.xml'
-        manifest_data = fetch.fetch_file(opt.manifest_url)
+        manifest_data = fetch.fetch_file(opt.manifest_url, verbose=opt.verbose)
         dest = os.path.join(m.worktree, manifest_name)
         os.makedirs(os.path.dirname(dest), exist_ok=True)
         with open(dest, 'wb') as f:
@@ -478,11 +477,16 @@ to update the working directory files.
 
     # Check this here, else manifest will be tagged "not new" and init won't be
     # possible anymore without removing the .repo/manifests directory.
-    if opt.archive and opt.mirror:
+    if opt.mirror:
-      self.OptionParser.error('--mirror and --archive cannot be used together.')
+      if opt.archive:
+        self.OptionParser.error('--mirror and --archive cannot be used '
+                                'together.')
+      if opt.use_superproject is not None:
+        self.OptionParser.error('--mirror and --use-superproject cannot be '
+                                'used together.')
 
-    if opt.standalone_manifest and (
+    if opt.standalone_manifest and (opt.manifest_branch or
-        opt.manifest_branch or opt.manifest_name != 'default.xml'):
+                                    opt.manifest_name != 'default.xml'):
       self.OptionParser.error('--manifest-branch and --manifest-name cannot'
                               ' be used with --standalone-manifest.')
 

subcmds/sync.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import errno
 import functools
 import http.cookiejar as cookielib
 import io
@@ -235,24 +234,25 @@ later is required to fix a server side protocol bug.
                  dest='fetch_submodules', action='store_true',
                  help='fetch submodules from server')
     p.add_option('--use-superproject', action='store_true',
-                 help='use the manifest superproject to sync projects')
+                 help='use the manifest superproject to sync projects; implies -c')
     p.add_option('--no-use-superproject', action='store_false',
                  dest='use_superproject',
                  help='disable use of manifest superprojects')
-    p.add_option('--tags',
+    p.add_option('--tags', action='store_true',
-                 action='store_false',
                  help='fetch tags')
     p.add_option('--no-tags',
                  dest='tags', action='store_false',
-                 help="don't fetch tags")
+                 help="don't fetch tags (default)")
     p.add_option('--optimized-fetch',
                  dest='optimized_fetch', action='store_true',
                  help='only fetch projects fixed to sha1 if revision does not exist locally')
     p.add_option('--retry-fetches',
                  default=0, action='store', type='int',
                  help='number of times to retry fetches on transient errors')
-    p.add_option('--prune', dest='prune', action='store_true',
+    p.add_option('--prune', action='store_true',
-                 help='delete refs that no longer exist on the remote')
+                 help='delete refs that no longer exist on the remote (default)')
+    p.add_option('--no-prune', dest='prune', action='store_false',
+                 help='do not delete refs that no longer exist on the remote')
     if show_smart:
       p.add_option('-s', '--smart-sync',
                    dest='smart_sync', action='store_true',
@@ -448,8 +448,8 @@ later is required to fix a server side protocol bug.
       else:
         pm.update(inc=0, msg='warming up')
         chunksize = 4
-      with multiprocessing.Pool(
+      with multiprocessing.Pool(jobs, initializer=self._FetchInitChild,
-          jobs, initializer=self._FetchInitChild, initargs=(ssh_proxy,)) as pool:
+                                initargs=(ssh_proxy,)) as pool:
         results = pool.imap_unordered(
             functools.partial(self._FetchProjectList, opt),
             projects_list,
@@ -605,7 +605,7 @@ later is required to fix a server side protocol bug.
     pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet)
     pm.update(inc=0, msg='prescan')
 
-    gc_gitdirs = {}
+    tidy_dirs = {}
     for project in projects:
       # Make sure pruning never kicks in with shared projects.
       if (not project.use_git_worktrees and
@@ -623,17 +623,28 @@ later is required to fix a server side protocol bug.
                 file=sys.stderr)
           project.config.SetString('gc.pruneExpire', 'never')
       project.config.SetString('gc.autoDetach', 'false')
-      gc_gitdirs[project.gitdir] = project.bare_git
+      # Only call git gc once per objdir, but call pack-refs for the remainder.
-
+      if project.objdir not in tidy_dirs:
-    pm.update(inc=len(projects) - len(gc_gitdirs), msg='warming up')
+        tidy_dirs[project.objdir] = (
+            True,  # Run a full gc.
+            project.bare_git,
+        )
+      elif project.gitdir not in tidy_dirs:
+        tidy_dirs[project.gitdir] = (
+            False,  # Do not run a full gc; just run pack-refs.
+            project.bare_git,
+        )
 
     cpu_count = os.cpu_count()
     jobs = min(self.jobs, cpu_count)
 
     if jobs < 2:
-      for bare_git in gc_gitdirs.values():
+      for (run_gc, bare_git) in tidy_dirs.values():
         pm.update(msg=bare_git._project.name)
+        if run_gc:
           bare_git.gc('--auto')
+        else:
+          bare_git.pack_refs()
       pm.end()
       return
 
@@ -642,11 +653,14 @@ later is required to fix a server side protocol bug.
     threads = set()
     sem = _threading.Semaphore(jobs)
 
-    def GC(bare_git):
+    def tidy_up(run_gc, bare_git):
       pm.start(bare_git._project.name)
       try:
         try:
+          if run_gc:
             bare_git.gc('--auto', config=config)
+          else:
+            bare_git.pack_refs(config=config)
         except GitError:
           err_event.set()
         except Exception:
@@ -656,11 +670,11 @@ later is required to fix a server side protocol bug.
         pm.finish(bare_git._project.name)
         sem.release()
 
-    for bare_git in gc_gitdirs.values():
+    for (run_gc, bare_git) in tidy_dirs.values():
       if err_event.is_set() and opt.fail_fast:
         break
       sem.acquire()
-      t = _threading.Thread(target=GC, args=(bare_git,))
+      t = _threading.Thread(target=tidy_up, args=(run_gc, bare_git,))
       t.daemon = True
       threads.add(t)
       t.start()
@@ -753,7 +767,7 @@ later is required to fix a server side protocol bug.
       with open(copylinkfile_path, 'rb') as fp:
         try:
           old_copylinkfile_paths = json.load(fp)
-        except:
+        except Exception:
           print('error: %s is not a json formatted file.' %
                 copylinkfile_path, file=sys.stderr)
           platform_utils.remove(copylinkfile_path)
@@ -915,6 +929,9 @@ later is required to fix a server side protocol bug.
       if None in [opt.manifest_server_username, opt.manifest_server_password]:
         self.OptionParser.error('both -u and -p must be given')
 
+    if opt.prune is None:
+      opt.prune = True
+
   def Execute(self, opt, args):
     if opt.jobs:
       self.jobs = opt.jobs
@@ -955,6 +972,8 @@ later is required to fix a server side protocol bug.
               file=sys.stderr)
 
     mp = self.manifest.manifestProject
+    is_standalone_manifest = mp.config.GetString('manifest.standalone')
+    if not is_standalone_manifest:
       mp.PreSync()
 
     if opt.repo_upgraded:
@@ -962,11 +981,15 @@ later is required to fix a server side protocol bug.
 
     if not opt.mp_update:
       print('Skipping update of local manifest project.')
-    else:
+    elif not is_standalone_manifest:
       self._UpdateManifestProject(opt, mp, manifest_name)
 
     load_local_manifests = not self.manifest.HasLocalManifests
     use_superproject = git_superproject.UseSuperproject(opt, self.manifest)
+    if self.manifest.IsMirror or self.manifest.IsArchive:
+      # Don't use superproject, because we have no working tree.
+      use_superproject = False
+      print('Defaulting to no-use-superproject because there is no working tree.')
     superproject_logging_data = {
         'superproject': use_superproject,
         'haslocalmanifests': bool(self.manifest.HasLocalManifests),
@@ -1102,6 +1125,15 @@ later is required to fix a server side protocol bug.
 
 
 def _PostRepoUpgrade(manifest, quiet=False):
+  # Link the docs for the internal .repo/ layout for people
+  link = os.path.join(manifest.repodir, 'internal-fs-layout.md')
+  if not platform_utils.islink(link):
+    target = os.path.join('repo', 'docs', 'internal-fs-layout.md')
+    try:
+      platform_utils.symlink(target, link)
+    except Exception:
+      pass
+
   wrapper = Wrapper()
   if wrapper.NeedSetupGnuPG():
     wrapper.SetupGnuPG(quiet)

tests/test_project.py

@@ -16,6 +16,7 @@
 
 import contextlib
 import os
+from pathlib import Path
 import shutil
 import subprocess
 import tempfile
@@ -335,3 +336,52 @@ class LinkFile(CopyLinkTestCase):
     platform_utils.symlink(self.tempdir, dest)
     lf._Link()
     self.assertEqual(os.path.join('git-project', 'foo.txt'), os.readlink(dest))
+
+
+class MigrateWorkTreeTests(unittest.TestCase):
+  """Check _MigrateOldWorkTreeGitDir handling."""
+
+  _SYMLINKS = {
+      'config', 'description', 'hooks', 'info', 'logs', 'objects',
+      'packed-refs', 'refs', 'rr-cache', 'shallow', 'svn',
+  }
+  _FILES = {
+      'COMMIT_EDITMSG', 'FETCH_HEAD', 'HEAD', 'index', 'ORIG_HEAD',
+  }
+
+  @classmethod
+  @contextlib.contextmanager
+  def _simple_layout(cls):
+    """Create a simple repo client checkout to test against."""
+    with tempfile.TemporaryDirectory() as tempdir:
+      tempdir = Path(tempdir)
+
+      gitdir = tempdir / '.repo/projects/src/test.git'
+      gitdir.mkdir(parents=True)
+      cmd = ['git', 'init', '--bare', str(gitdir)]
+      subprocess.check_call(cmd)
+
+      dotgit = tempdir / 'src/test/.git'
+      dotgit.mkdir(parents=True)
+      for name in cls._SYMLINKS:
+        (dotgit / name).symlink_to(f'../../../.repo/projects/src/test.git/{name}')
+      for name in cls._FILES:
+        (dotgit / name).write_text(name)
+
+      subprocess.run(['tree', '-a', str(dotgit)])
+      yield tempdir
+
+  def test_standard(self):
+    """Migrate a standard checkout that we expect."""
+    with self._simple_layout() as tempdir:
+      dotgit = tempdir / 'src/test/.git'
+      project.Project._MigrateOldWorkTreeGitDir(str(dotgit))
+
+      # Make sure the dir was transformed into a symlink.
+      self.assertTrue(dotgit.is_symlink())
+      self.assertEqual(str(dotgit.readlink()), '../../.repo/projects/src/test.git')
+
+      # Make sure files were moved over.
+      gitdir = tempdir / '.repo/projects/src/test.git'
+      for name in self._FILES:
+        self.assertEqual(name, (gitdir / name).read_text())