tests: drop old unittest.main logic

We use pytest now which doesn't need this boilerplate.

Change-Id: Ib71d90b3f1669897814ee768927b5b595ca8d789
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/354355
Reviewed-by: Josip Sokcevic <sokcevic@google.com>
Tested-by: Mike Frysinger <vapier@google.com>

.github/workflows/test-ci.yml

@@ -14,18 +14,18 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10']
     runs-on: ${{ matrix.os }}
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install tox tox-gh-actions
+        python -m pip install tox tox-gh-actions
     - name: Test with tox
       run: tox

command.py

@@ -144,11 +144,10 @@ class Command(object):
           help=f'number of jobs to run in parallel (default: {default})')
 
     m = p.add_option_group('Multi-manifest options')
-    m.add_option('--outer-manifest', action='store_true',
+    m.add_option('--outer-manifest', action='store_true', default=None,
                  help='operate starting at the outermost manifest')
     m.add_option('--no-outer-manifest', dest='outer_manifest',
-                 action='store_false', default=None,
-                 help='do not operate on outer manifests')
+                 action='store_false', help='do not operate on outer manifests')
     m.add_option('--this-manifest-only', action='store_true', default=None,
                  help='only operate on this (sub)manifest')
     m.add_option('--no-this-manifest-only', '--all-manifests',
@@ -186,6 +185,10 @@ class Command(object):
     """Validate common options."""
     opt.quiet = opt.output_mode is False
     opt.verbose = opt.output_mode is True
+    if opt.outer_manifest is None:
+      # By default, treat multi-manifest instances as a single manifest from
+      # the user's perspective.
+      opt.outer_manifest = True
 
   def ValidateOptions(self, opt, args):
     """Validate the user options & arguments before executing.
@@ -274,6 +277,18 @@ class Command(object):
   def GetProjects(self, args, manifest=None, groups='', missing_ok=False,
                   submodules_ok=False, all_manifests=False):
     """A list of projects that match the arguments.
+
+    Args:
+      args: a list of (case-insensitive) strings, projects to search for.
+      manifest: an XmlManifest, the manifest to use, or None for default.
+      groups: a string, the manifest groups in use.
+      missing_ok: a boolean, whether to allow missing projects.
+      submodules_ok: a boolean, whether to allow submodules.
+      all_manifests: a boolean, if True then all manifests and submanifests are
+                     used.  If False, then only the local (sub)manifest is used.
+
+    Returns:
+      A list of matching Project instances.
     """
     if all_manifests:
       if not manifest:
@@ -385,7 +400,7 @@ class Command(object):
       opt: The command options.
     """
     top = self.outer_manifest
-    if opt.outer_manifest is False or opt.this_manifest_only:
+    if not opt.outer_manifest or opt.this_manifest_only:
       top = self.manifest
     yield top
     if not opt.this_manifest_only:

docs/manifest-format.md

@@ -66,6 +66,7 @@ following DTD:
   <!ATTLIST submanifest revision       CDATA #IMPLIED>
   <!ATTLIST submanifest path           CDATA #IMPLIED>
   <!ATTLIST submanifest groups         CDATA #IMPLIED>
+  <!ATTLIST submanifest default-groups CDATA #IMPLIED>
 
   <!ELEMENT project (annotation*,
                      project*,
@@ -104,6 +105,8 @@ following DTD:
   <!ATTLIST extend-project groups CDATA #IMPLIED>
   <!ATTLIST extend-project revision CDATA #IMPLIED>
   <!ATTLIST extend-project remote CDATA #IMPLIED>
+  <!ATTLIST extend-project dest-branch CDATA #IMPLIED>
+  <!ATTLIST extend-project upstream CDATA #IMPLIED>
 
   <!ELEMENT remove-project EMPTY>
   <!ATTLIST remove-project name  CDATA #REQUIRED>
@@ -302,6 +305,9 @@ in the included submanifest belong. This appends and recurses, meaning
 all projects in submanifests carry all parent submanifest groups.
 Same syntax as the corresponding element of `project`.
 
+Attribute `default-groups`: The list of manifest groups to sync if no
+`--groups=` parameter was specified at init.  When that list is empty, use this
+list instead of "default" as the list of groups to sync.
 
 ### Element project
 
@@ -419,6 +425,12 @@ project.  Same syntax as the corresponding element of `project`.
 Attribute `remote`: If specified, overrides the remote of the original
 project.  Same syntax as the corresponding element of `project`.
 
+Attribute `dest-branch`: If specified, overrides the dest-branch of the original
+project.  Same syntax as the corresponding element of `project`.
+
+Attribute `upstream`: If specified, overrides the upstream of the original
+project.  Same syntax as the corresponding element of `project`.
+
 ### Element annotation
 
 Zero or more annotation elements may be specified as children of a

docs/release-process.md

@@ -143,23 +143,14 @@ internal processes for accessing the restricted keys.
 ***
 
 ```sh
-# Set the gpg key directory.
-$ export GNUPGHOME=~/.gnupg/repo/
-
-# Verify the listed key is “Repo Maintainer”.
-$ gpg -K
-
-# Pick whatever branch or commit you want to tag.
-$ r=main
-
 # Pick the new version.
-$ t=1.12.10
+$ t=v2.30
 
-# Create the signed tag.
-$ git tag -s v$t -u "Repo Maintainer <repo@android.kernel.org>" -m "repo $t" $r
+# Create a new signed tag with the current HEAD.
+$ ./release/sign-tag.py $t
 
 # Verify the signed tag.
-$ git show v$t
+$ git show $t
 ```
 
 ### Push the new release
@@ -168,11 +159,11 @@ Once you're ready to make the release available to everyone, push it to the
 `stable` branch.
 
 Make sure you never push the tag itself to the stable branch!
-Only push the commit -- notice the use of `$t` and `$r` below.
+Only push the commit -- note the use of `^0` below.
 
 ```sh
-$ git push https://gerrit-review.googlesource.com/git-repo v$t
-$ git push https://gerrit-review.googlesource.com/git-repo $r:stable
+$ git push https://gerrit-review.googlesource.com/git-repo $t
+$ git push https://gerrit-review.googlesource.com/git-repo $t^0:stable
 ```
 
 If something goes horribly wrong, you can force push the previous version to the
@@ -195,7 +186,9 @@ You can create a short changelog using the command:
 ```sh
 # If you haven't pushed to the stable branch yet, you can use origin/stable.
 # If you have pushed, change origin/stable to the previous release tag.
-$ git log --format="%h (%aN) %s" --no-merges origin/stable..$r
+# This assumes "main" is the current tagged release.  If it's newer, change it
+# to the current release tag too.
+$ git log --format="%h (%aN) %s" --no-merges origin/stable..main
 ```
 
 ## Project References

git_command.py

@@ -16,6 +16,7 @@ import functools
 import os
 import sys
 import subprocess
+from typing import Any, Optional
 
 from error import GitError
 from git_refs import HEAD
@@ -157,7 +158,56 @@ def git_require(min_version, fail=False, msg=''):
   return False
 
 
+def _build_env(
+  _kwargs_only=(),
+  bare: Optional[bool] = False,
+  disable_editor: Optional[bool] = False,
+  ssh_proxy: Optional[Any] = None,
+  gitdir: Optional[str] = None,
+  objdir: Optional[str] = None
+):
+  """Constucts an env dict for command execution."""
+
+  assert _kwargs_only == (), '_build_env only accepts keyword arguments.'
+
+  env = GitCommand._GetBasicEnv()
+
+  if disable_editor:
+    env['GIT_EDITOR'] = ':'
+  if ssh_proxy:
+    env['REPO_SSH_SOCK'] = ssh_proxy.sock()
+    env['GIT_SSH'] = ssh_proxy.proxy
+    env['GIT_SSH_VARIANT'] = 'ssh'
+  if 'http_proxy' in env and 'darwin' == sys.platform:
+    s = "'http.proxy=%s'" % (env['http_proxy'],)
+    p = env.get('GIT_CONFIG_PARAMETERS')
+    if p is not None:
+      s = p + ' ' + s
+    env['GIT_CONFIG_PARAMETERS'] = s
+  if 'GIT_ALLOW_PROTOCOL' not in env:
+    env['GIT_ALLOW_PROTOCOL'] = (
+        'file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc')
+  env['GIT_HTTP_USER_AGENT'] = user_agent.git
+
+  if objdir:
+    # Set to the place we want to save the objects.
+    env['GIT_OBJECT_DIRECTORY'] = objdir
+
+    alt_objects = os.path.join(gitdir, 'objects') if gitdir else None
+    if (alt_objects and
+        os.path.realpath(alt_objects) != os.path.realpath(objdir)):
+      # Allow git to search the original place in case of local or unique refs
+      # that git will attempt to resolve even if we aren't fetching them.
+      env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = alt_objects
+  if bare and gitdir is not None:
+      env[GIT_DIR] = gitdir
+
+  return env
+
+
 class GitCommand(object):
+  """Wrapper around a single git invocation."""
+
   def __init__(self,
                project,
                cmdv,
@@ -171,30 +221,13 @@ class GitCommand(object):
                cwd=None,
                gitdir=None,
                objdir=None):
-    env = self._GetBasicEnv()
-
-    if disable_editor:
-      env['GIT_EDITOR'] = ':'
-    if ssh_proxy:
-      env['REPO_SSH_SOCK'] = ssh_proxy.sock()
-      env['GIT_SSH'] = ssh_proxy.proxy
-      env['GIT_SSH_VARIANT'] = 'ssh'
-    if 'http_proxy' in env and 'darwin' == sys.platform:
-      s = "'http.proxy=%s'" % (env['http_proxy'],)
-      p = env.get('GIT_CONFIG_PARAMETERS')
-      if p is not None:
-        s = p + ' ' + s
-      env['GIT_CONFIG_PARAMETERS'] = s
-    if 'GIT_ALLOW_PROTOCOL' not in env:
-      env['GIT_ALLOW_PROTOCOL'] = (
-          'file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc')
-    env['GIT_HTTP_USER_AGENT'] = user_agent.git
 
     if project:
       if not cwd:
         cwd = project.worktree
       if not gitdir:
         gitdir = project.gitdir
+
     # Git on Windows wants its paths only using / for reliability.
     if platform_utils.isWindows():
       if objdir:
@@ -202,18 +235,16 @@ class GitCommand(object):
       if gitdir:
         gitdir = gitdir.replace('\\', '/')
 
-    if objdir:
-      # Set to the place we want to save the objects.
-      env['GIT_OBJECT_DIRECTORY'] = objdir
-      if gitdir:
-        # Allow git to search the original place in case of local or unique refs
-        # that git will attempt to resolve even if we aren't fetching them.
-        env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = gitdir + '/objects'
+    env = _build_env(
+      disable_editor=disable_editor,
+      ssh_proxy=ssh_proxy,
+      objdir=objdir,
+      gitdir=gitdir,
+      bare=bare,
+    )
 
     command = [GIT]
     if bare:
-      if gitdir:
-        env[GIT_DIR] = gitdir
       cwd = None
     command.append(cmdv[0])
     # Need to use the --progress flag for fetch/clone so output will be
@@ -228,12 +259,11 @@ class GitCommand(object):
     stderr = (subprocess.STDOUT if merge_output else
               (subprocess.PIPE if capture_stderr else None))
 
+    dbg = ''
     if IsTrace():
       global LAST_CWD
       global LAST_GITDIR
 
-      dbg = ''
-
       if cwd and LAST_CWD != cwd:
         if LAST_GITDIR or LAST_CWD:
           dbg += '\n'
@@ -261,36 +291,31 @@ class GitCommand(object):
         dbg += ' 2>|'
       elif stderr == subprocess.STDOUT:
         dbg += ' 2>&1'
-      Trace('%s', dbg)
 
-    try:
-      p = subprocess.Popen(command,
-                           cwd=cwd,
-                           env=env,
-                           encoding='utf-8',
-                           errors='backslashreplace',
-                           stdin=stdin,
-                           stdout=stdout,
-                           stderr=stderr)
-    except Exception as e:
-      raise GitError('%s: %s' % (command[1], e))
+    with Trace('git command %s %s with debug: %s', LAST_GITDIR, command, dbg):
+      try:
+        p = subprocess.Popen(command,
+                            cwd=cwd,
+                            env=env,
+                            encoding='utf-8',
+                            errors='backslashreplace',
+                            stdin=stdin,
+                            stdout=stdout,
+                            stderr=stderr)
+      except Exception as e:
+        raise GitError('%s: %s' % (command[1], e))
 
-    if ssh_proxy:
-      ssh_proxy.add_client(p)
-
-    self.process = p
-    if input:
-      if isinstance(input, str):
-        input = input.encode('utf-8')
-      p.stdin.write(input)
-      p.stdin.close()
-
-    try:
-      self.stdout, self.stderr = p.communicate()
-    finally:
       if ssh_proxy:
-        ssh_proxy.remove_client(p)
-    self.rc = p.wait()
+        ssh_proxy.add_client(p)
+
+      self.process = p
+
+      try:
+        self.stdout, self.stderr = p.communicate(input=input)
+      finally:
+        if ssh_proxy:
+          ssh_proxy.remove_client(p)
+      self.rc = p.wait()
 
   @staticmethod
   def _GetBasicEnv():

git_config.py

@@ -22,6 +22,7 @@ import re
 import ssl
 import subprocess
 import sys
+from typing import Union
 import urllib.error
 import urllib.request
 
@@ -117,7 +118,7 @@ class GitConfig(object):
       return self.defaults.Has(name, include_defaults=True)
     return False
 
-  def GetInt(self, name):
+  def GetInt(self, name: str) -> Union[int, None]:
     """Returns an integer from the configuration file.
 
     This follows the git config syntax.
@@ -126,7 +127,7 @@ class GitConfig(object):
       name: The key to lookup.
 
     Returns:
-      None if the value was not defined, or is not a boolean.
+      None if the value was not defined, or is not an int.
       Otherwise, the number itself.
     """
     v = self.GetString(name)
@@ -152,6 +153,9 @@ class GitConfig(object):
     try:
       return int(v, base=base) * mult
     except ValueError:
+      print(
+          f"warning: expected {name} to represent an integer, got {v} instead",
+          file=sys.stderr)
       return None
 
   def DumpConfigDict(self):
@@ -169,7 +173,7 @@ class GitConfig(object):
       config_dict[key] = self.GetString(key)
     return config_dict
 
-  def GetBoolean(self, name):
+  def GetBoolean(self, name: str) -> Union[str, None]:
     """Returns a boolean from the configuration file.
        None : The value was not defined, or is not a boolean.
        True : The value was set to true or yes.
@@ -183,6 +187,8 @@ class GitConfig(object):
       return True
     if v in ('false', 'no'):
       return False
+    print(f"warning: expected {name} to represent a boolean, got {v} instead",
+            file=sys.stderr)
     return None
 
   def SetBoolean(self, name, value):
@@ -191,7 +197,7 @@ class GitConfig(object):
       value = 'true' if value else 'false'
     self.SetString(name, value)
 
-  def GetString(self, name, all_keys=False):
+  def GetString(self, name: str, all_keys: bool = False) -> Union[str,  None]:
     """Get the first value for a key, or None if it is not defined.
 
        This configuration file is used first, if the key is not
@@ -219,8 +225,8 @@ class GitConfig(object):
     """Set the value(s) for a key.
        Only this configuration file is modified.
 
-       The supplied value should be either a string,
-       or a list of strings (to store multiple values).
+       The supplied value should be either a string, or a list of strings (to
+       store multiple values), or None (to delete the key).
     """
     key = _key(name)
 
@@ -349,9 +355,9 @@ class GitConfig(object):
     except OSError:
       return None
     try:
-      Trace(': parsing %s', self.file)
-      with open(self._json) as fd:
-        return json.load(fd)
+      with Trace(': parsing %s', self.file):
+        with open(self._json) as fd:
+          return json.load(fd)
     except (IOError, ValueError):
       platform_utils.remove(self._json, missing_ok=True)
       return None

git_refs.py

@@ -67,38 +67,37 @@ class GitRefs(object):
       self._LoadAll()
 
   def _NeedUpdate(self):
-    Trace(': scan refs %s', self._gitdir)
-
-    for name, mtime in self._mtime.items():
-      try:
-        if mtime != os.path.getmtime(os.path.join(self._gitdir, name)):
+    with Trace(': scan refs %s', self._gitdir):
+      for name, mtime in self._mtime.items():
+        try:
+          if mtime != os.path.getmtime(os.path.join(self._gitdir, name)):
+            return True
+        except OSError:
           return True
-      except OSError:
-        return True
-    return False
+      return False
 
   def _LoadAll(self):
-    Trace(': load refs %s', self._gitdir)
+    with Trace(': load refs %s', self._gitdir):
 
-    self._phyref = {}
-    self._symref = {}
-    self._mtime = {}
+      self._phyref = {}
+      self._symref = {}
+      self._mtime = {}
 
-    self._ReadPackedRefs()
-    self._ReadLoose('refs/')
-    self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD)
+      self._ReadPackedRefs()
+      self._ReadLoose('refs/')
+      self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD)
 
-    scan = self._symref
-    attempts = 0
-    while scan and attempts < 5:
-      scan_next = {}
-      for name, dest in scan.items():
-        if dest in self._phyref:
-          self._phyref[name] = self._phyref[dest]
-        else:
-          scan_next[name] = dest
-      scan = scan_next
-      attempts += 1
+      scan = self._symref
+      attempts = 0
+      while scan and attempts < 5:
+        scan_next = {}
+        for name, dest in scan.items():
+          if dest in self._phyref:
+            self._phyref[name] = self._phyref[dest]
+          else:
+            scan_next[name] = dest
+        scan = scan_next
+        attempts += 1
 
   def _ReadPackedRefs(self):
     path = os.path.join(self._gitdir, 'packed-refs')

git_superproject.py

@@ -18,7 +18,7 @@ For more information on superproject, check out:
 https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects
 
 Examples:
-  superproject = Superproject()
+  superproject = Superproject(manifest, name, remote, revision)
   UpdateProjectsResult = superproject.UpdateProjectsRevisionId(projects)
 """
 
@@ -99,8 +99,8 @@ class Superproject(object):
     self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME
     self._work_git = os.path.join(self._superproject_path, self._work_git_name)
 
-    # The following are command arguemnts, rather then superproject attributes,
-    # and where included here originally.  They should eventually become
+    # The following are command arguemnts, rather than superproject attributes,
+    # and were included here originally.  They should eventually become
     # arguments that are passed down from the public methods, instead of being
     # treated as attributes.
     self._git_event_log = None
@@ -295,7 +295,8 @@ class Superproject(object):
     if not os.path.exists(self._superproject_path):
       self._LogWarning(f'missing superproject directory: {self._superproject_path}')
       return None
-    manifest_str = self._manifest.ToXml(groups=self._manifest.GetGroupsStr()).toxml()
+    manifest_str = self._manifest.ToXml(groups=self._manifest.GetGroupsStr(),
+                                        omit_local=True).toxml()
     manifest_path = self._manifest_path
     try:
       with open(manifest_path, 'w', encoding='utf-8') as fp:
@@ -329,7 +330,8 @@ class Superproject(object):
     """Update revisionId of every project in projects with the commit id.
 
     Args:
-      projects: List of projects whose revisionId needs to be updated.
+      projects: a list of projects whose revisionId needs to be updated.
+      git_event_log: an EventLog, for git tracing.
 
     Returns:
       UpdateProjectsResult
@@ -431,9 +433,15 @@ def UseSuperproject(use_superproject, manifest):
   Args:
     use_superproject: option value from optparse.
     manifest: manifest to use.
+
+  Returns:
+    Whether the superproject should be used.
   """
 
-  if use_superproject is not None:
+  if not manifest.superproject:
+    # This (sub) manifest does not have a superproject definition.
+    return False
+  elif use_superproject is not None:
     return use_superproject
   else:
     client_value = manifest.manifestProject.use_superproject

git_trace2_event_log.py

@@ -110,7 +110,7 @@ class EventLog(object):
     return {
         'event': event_name,
         'sid': self._full_sid,
-        'thread': threading.currentThread().getName(),
+        'thread': threading.current_thread().name,
         'time': datetime.datetime.utcnow().isoformat() + 'Z',
     }
 

hooks/commit-msg

@@ -1,5 +1,5 @@
 #!/bin/sh
-# From Gerrit Code Review 3.1.3
+# From Gerrit Code Review 3.6.1 c67916dbdc07555c44e32a68f92ffc484b9b34f0
 #
 # Part of Gerrit Code Review (https://www.gerritcodereview.com/)
 #
@@ -17,6 +17,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -u
+
 # avoid [[ which is not POSIX sh.
 if test "$#" != 1 ; then
   echo "$0 requires an argument."
@@ -29,15 +31,25 @@ if test ! -f "$1" ; then
 fi
 
 # Do not create a change id if requested
-if test "false" = "`git config --bool --get gerrit.createChangeId`" ; then
+if test "false" = "$(git config --bool --get gerrit.createChangeId)" ; then
   exit 0
 fi
 
-# $RANDOM will be undefined if not using bash, so don't use set -u
-random=$( (whoami ; hostname ; date; cat $1 ; echo $RANDOM) | git hash-object --stdin)
+# Do not create a change id for squash commits.
+if head -n1 "$1" | grep -q '^squash! '; then
+  exit 0
+fi
+
+if git rev-parse --verify HEAD >/dev/null 2>&1; then
+  refhash="$(git rev-parse HEAD)"
+else
+  refhash="$(git hash-object -t tree /dev/null)"
+fi
+
+random=$({ git var GIT_COMMITTER_IDENT ; echo "$refhash" ; cat "$1"; } | git hash-object --stdin)
 dest="$1.tmp.${random}"
 
-trap 'rm -f "${dest}"' EXIT
+trap 'rm -f "$dest" "$dest-2"' EXIT
 
 if ! git stripspace --strip-comments < "$1" > "${dest}" ; then
    echo "cannot strip comments from $1"
@@ -49,11 +61,40 @@ if test ! -s "${dest}" ; then
   exit 1
 fi
 
+reviewurl="$(git config --get gerrit.reviewUrl)"
+if test -n "${reviewurl}" ; then
+  token="Link"
+  value="${reviewurl%/}/id/I$random"
+  pattern=".*/id/I[0-9a-f]\{40\}$"
+else
+  token="Change-Id"
+  value="I$random"
+  pattern=".*"
+fi
+
+if git interpret-trailers --parse < "$1" | grep -q "^$token: $pattern$" ; then
+  exit 0
+fi
+
+# There must be a Signed-off-by trailer for the code below to work. Insert a
+# sentinel at the end to make sure there is one.
 # Avoid the --in-place option which only appeared in Git 2.8
-# Avoid the --if-exists option which only appeared in Git 2.15
-if ! git -c trailer.ifexists=doNothing interpret-trailers \
-      --trailer "Change-Id: I${random}" < "$1" > "${dest}" ; then
-  echo "cannot insert change-id line in $1"
+if ! git interpret-trailers \
+         --trailer "Signed-off-by: SENTINEL" < "$1" > "$dest-2" ; then
+  echo "cannot insert Signed-off-by sentinel line in $1"
+  exit 1
+fi
+
+# Make sure the trailer appears before any Signed-off-by trailers by inserting
+# it as if it was a Signed-off-by trailer and then use sed to remove the
+# Signed-off-by prefix and the Signed-off-by sentinel line.
+# Avoid the --in-place option which only appeared in Git 2.8
+# Avoid the --where option which only appeared in Git 2.15
+if ! git -c trailer.where=before interpret-trailers \
+         --trailer "Signed-off-by: $token: $value" < "$dest-2" |
+     sed -re "s/^Signed-off-by: ($token: )/\1/" \
+         -e "/^Signed-off-by: SENTINEL/d" > "$dest" ; then
+  echo "cannot insert $token line in $1"
   exit 1
 fi
 

main.py

@@ -37,7 +37,7 @@ except ImportError:
 
 from color import SetDefaultColoring
 import event_log
-from repo_trace import SetTrace
+from repo_trace import SetTrace, Trace, SetTraceToStderr
 from git_command import user_agent
 from git_config import RepoConfig
 from git_trace2_event_log import EventLog
@@ -109,6 +109,9 @@ global_options.add_option('--color',
 global_options.add_option('--trace',
                           dest='trace', action='store_true',
                           help='trace git command execution (REPO_TRACE=1)')
+global_options.add_option('--trace-to-stderr',
+                          dest='trace_to_stderr', action='store_true',
+                          help='trace outputs go to stderr in addition to .repo/TRACE_FILE')
 global_options.add_option('--trace-python',
                           dest='trace_python', action='store_true',
                           help='trace python command execution')
@@ -198,9 +201,6 @@ class _Repo(object):
     """Execute the requested subcommand."""
     result = 0
 
-    if gopts.trace:
-      SetTrace()
-
     # Handle options that terminate quickly first.
     if gopts.help or gopts.help_all:
       self._PrintHelp(short=False, all_commands=gopts.help_all)
@@ -216,6 +216,21 @@ class _Repo(object):
       self._PrintHelp(short=True)
       return 1
 
+    run = lambda: self._RunLong(name, gopts, argv) or 0
+    with Trace('starting new command: %s', ', '.join([name] + argv),
+               first_trace=True):
+      if gopts.trace_python:
+        import trace
+        tracer = trace.Trace(count=False, trace=True, timing=True,
+                             ignoredirs=set(sys.path[1:]))
+        result = tracer.runfunc(run)
+      else:
+        result = run()
+    return result
+
+  def _RunLong(self, name, gopts, argv):
+    """Execute the (longer running) requested subcommand."""
+    result = 0
     SetDefaultColoring(gopts.color)
 
     git_trace2_event_log = EventLog()
@@ -294,8 +309,7 @@ class _Repo(object):
       cmd.ValidateOptions(copts, cargs)
 
       this_manifest_only = copts.this_manifest_only
-      # If not specified, default to using the outer manifest.
-      outer_manifest = copts.outer_manifest is not False
+      outer_manifest = copts.outer_manifest
       if cmd.MULTI_MANIFEST_SUPPORT or this_manifest_only:
         result = cmd.Execute(copts, cargs)
       elif outer_manifest and repo_client.manifest.is_submanifest:
@@ -653,17 +667,18 @@ def _Main(argv):
   Version.wrapper_path = opt.wrapper_path
 
   repo = _Repo(opt.repodir)
+
   try:
     init_http()
     name, gopts, argv = repo._ParseArgs(argv)
-    run = lambda: repo._Run(name, gopts, argv) or 0
-    if gopts.trace_python:
-      import trace
-      tracer = trace.Trace(count=False, trace=True, timing=True,
-                           ignoredirs=set(sys.path[1:]))
-      result = tracer.runfunc(run)
-    else:
-      result = run()
+
+    if gopts.trace:
+      SetTrace()
+
+    if gopts.trace_to_stderr:
+      SetTraceToStderr()
+
+    result = repo._Run(name, gopts, argv) or 0
   except KeyboardInterrupt:
     print('aborted by user', file=sys.stderr)
     result = 1

man/repo-abandon.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo abandon" "Repo Manual"
+.TH REPO "1" "July 2022" "repo abandon" "Repo Manual"
 .SH NAME
 repo \- repo abandon - manual page for repo abandon
 .SH SYNOPSIS
@@ -32,5 +32,18 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help abandon` to view the detailed manual.

man/repo-branches.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo branches" "Repo Manual"
+.TH REPO "1" "July 2022" "repo branches" "Repo Manual"
 .SH NAME
 repo \- repo branches - manual page for repo branches
 .SH SYNOPSIS
@@ -55,5 +55,18 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help branches` to view the detailed manual.

man/repo-checkout.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo checkout" "Repo Manual"
+.TH REPO "1" "July 2022" "repo checkout" "Repo Manual"
 .SH NAME
 repo \- repo checkout - manual page for repo checkout
 .SH SYNOPSIS
@@ -24,6 +24,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help checkout` to view the detailed manual.
 .SH DETAILS

man/repo-cherry-pick.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo cherry-pick" "Repo Manual"
+.TH REPO "1" "July 2022" "repo cherry-pick" "Repo Manual"
 .SH NAME
 repo \- repo cherry-pick - manual page for repo cherry-pick
 .SH SYNOPSIS
@@ -20,6 +20,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help cherry\-pick` to view the detailed manual.
 .SH DETAILS

man/repo-diff.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo diff" "Repo Manual"
+.TH REPO "1" "July 2022" "repo diff" "Repo Manual"
 .SH NAME
 repo \- repo diff - manual page for repo diff
 .SH SYNOPSIS
@@ -31,5 +31,18 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help diff` to view the detailed manual.

man/repo-diffmanifests.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo diffmanifests" "Repo Manual"
+.TH REPO "1" "July 2022" "repo diffmanifests" "Repo Manual"
 .SH NAME
 repo \- repo diffmanifests - manual page for repo diffmanifests
 .SH SYNOPSIS
@@ -29,6 +29,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help diffmanifests` to view the detailed manual.
 .SH DETAILS

man/repo-download.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo download" "Repo Manual"
+.TH REPO "1" "July 2022" "repo download" "Repo Manual"
 .SH NAME
 repo \- repo download - manual page for repo download
 .SH SYNOPSIS
@@ -35,6 +35,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help download` to view the detailed manual.
 .SH DETAILS

man/repo-forall.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo forall" "Repo Manual"
+.TH REPO "1" "July 2022" "repo forall" "Repo Manual"
 .SH NAME
 repo \- repo forall - manual page for repo forall
 .SH SYNOPSIS
@@ -54,6 +54,19 @@ only show errors
 .TP
 \fB\-p\fR
 show project headers before output
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help forall` to view the detailed manual.
 .SH DETAILS
@@ -93,6 +106,11 @@ REPO_PROJECT is set to the unique name of the project.
 .PP
 REPO_PATH is the path relative the the root of the client.
 .PP
+REPO_OUTERPATH is the path of the sub manifest's root relative to the root of
+the client.
+.PP
+REPO_INNERPATH is the path relative to the root of the sub manifest.
+.PP
 REPO_REMOTE is the name of the remote system from the manifest.
 .PP
 REPO_LREV is the name of the revision from the manifest, translated to a local

man/repo-gitc-delete.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo gitc-delete" "Repo Manual"
+.TH REPO "1" "July 2022" "repo gitc-delete" "Repo Manual"
 .SH NAME
 repo \- repo gitc-delete - manual page for repo gitc-delete
 .SH SYNOPSIS
@@ -23,6 +23,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help gitc\-delete` to view the detailed manual.
 .SH DETAILS

man/repo-gitc-init.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "November 2021" "repo gitc-init" "Repo Manual"
+.TH REPO "1" "October 2022" "repo gitc-init" "Repo Manual"
 .SH NAME
 repo \- repo gitc-init - manual page for repo gitc-init
 .SH SYNOPSIS
@@ -45,10 +45,15 @@ sync any submodules associated with the manifest repo
 \fB\-\-standalone\-manifest\fR
 download the manifest as a static file rather then
 create a git checkout of the manifest repo
+.TP
+\fB\-\-manifest\-depth\fR=\fI\,DEPTH\/\fR
+create a shallow clone of the manifest repo with given
+depth (0 for full clone); see git clone (default: 0)
 .SS Manifest (only) checkout options:
 .TP
 \fB\-\-current\-branch\fR
 fetch only current manifest branch from server
+(default)
 .TP
 \fB\-\-no\-current\-branch\fR
 fetch all manifest branches from server
@@ -109,6 +114,12 @@ not \fB\-\-partial\-clone\fR)
 \fB\-\-no\-clone\-bundle\fR
 disable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS (default if
 \fB\-\-partial\-clone\fR)
+.TP
+\fB\-\-git\-lfs\fR
+enable Git LFS support
+.TP
+\fB\-\-no\-git\-lfs\fR
+disable Git LFS support
 .SS repo Version options:
 .TP
 \fB\-\-repo\-url\fR=\fI\,URL\/\fR
@@ -130,6 +141,19 @@ Optional manifest file to use for this GITC client.
 .TP
 \fB\-c\fR GITC_CLIENT, \fB\-\-gitc\-client\fR=\fI\,GITC_CLIENT\/\fR
 Name of the gitc_client instance to create or modify.
+.SS Multi\-manifest:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help gitc\-init` to view the detailed manual.
 .SH DETAILS

man/repo-grep.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo grep" "Repo Manual"
+.TH REPO "1" "July 2022" "repo grep" "Repo Manual"
 .SH NAME
 repo \- repo grep - manual page for repo grep
 .SH SYNOPSIS
@@ -24,6 +24,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .SS Sources:
 .TP
 \fB\-\-cached\fR

man/repo-help.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo help" "Repo Manual"
+.TH REPO "1" "July 2022" "repo help" "Repo Manual"
 .SH NAME
 repo \- repo help - manual page for repo help
 .SH SYNOPSIS
@@ -26,6 +26,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help help` to view the detailed manual.
 .SH DETAILS

man/repo-info.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo info" "Repo Manual"
+.TH REPO "1" "July 2022" "repo info" "Repo Manual"
 .SH NAME
 repo \- repo info - manual page for repo info
 .SH SYNOPSIS
@@ -36,5 +36,18 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help info` to view the detailed manual.

man/repo-init.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "November 2021" "repo init" "Repo Manual"
+.TH REPO "1" "October 2022" "repo init" "Repo Manual"
 .SH NAME
 repo \- repo init - manual page for repo init
 .SH SYNOPSIS
@@ -45,10 +45,15 @@ sync any submodules associated with the manifest repo
 \fB\-\-standalone\-manifest\fR
 download the manifest as a static file rather then
 create a git checkout of the manifest repo
+.TP
+\fB\-\-manifest\-depth\fR=\fI\,DEPTH\/\fR
+create a shallow clone of the manifest repo with given
+depth (0 for full clone); see git clone (default: 0)
 .SS Manifest (only) checkout options:
 .TP
 \fB\-c\fR, \fB\-\-current\-branch\fR
 fetch only current manifest branch from server
+(default)
 .TP
 \fB\-\-no\-current\-branch\fR
 fetch all manifest branches from server
@@ -109,6 +114,12 @@ not \fB\-\-partial\-clone\fR)
 \fB\-\-no\-clone\-bundle\fR
 disable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS (default if
 \fB\-\-partial\-clone\fR)
+.TP
+\fB\-\-git\-lfs\fR
+enable Git LFS support
+.TP
+\fB\-\-no\-git\-lfs\fR
+disable Git LFS support
 .SS repo Version options:
 .TP
 \fB\-\-repo\-url\fR=\fI\,URL\/\fR
@@ -123,6 +134,19 @@ do not verify repo source code
 .TP
 \fB\-\-config\-name\fR
 Always prompt for name/e\-mail
+.SS Multi\-manifest:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help init` to view the detailed manual.
 .SH DETAILS

man/repo-list.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo list" "Repo Manual"
+.TH REPO "1" "July 2022" "repo list" "Repo Manual"
 .SH NAME
 repo \- repo list - manual page for repo list
 .SH SYNOPSIS
@@ -47,6 +47,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help list` to view the detailed manual.
 .SH DETAILS

man/repo-manifest.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "November 2021" "repo manifest" "Repo Manual"
+.TH REPO "1" "October 2022" "repo manifest" "Repo Manual"
 .SH NAME
 repo \- repo manifest - manual page for repo manifest
 .SH SYNOPSIS
@@ -40,7 +40,8 @@ format output for humans to read
 ignore local manifests
 .TP
 \fB\-o\fR \-|NAME.xml, \fB\-\-output\-file\fR=\fI\,\-\/\fR|NAME.xml
-file to save the manifest to
+file to save the manifest to. (Filename prefix for
+multi\-tree.)
 .SS Logging options:
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR
@@ -48,6 +49,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help manifest` to view the detailed manual.
 .SH DETAILS
@@ -88,6 +102,7 @@ A manifest XML file (e.g. `default.xml`) roughly conforms to the following DTD:
 remote*,
 default?,
 manifest\-server?,
+submanifest*?,
 remove\-project*,
 project*,
 extend\-project*,
@@ -118,6 +133,16 @@ include*)>
 .IP
 <!ELEMENT manifest\-server EMPTY>
 <!ATTLIST manifest\-server url CDATA #REQUIRED>
+.IP
+<!ELEMENT submanifest EMPTY>
+<!ATTLIST submanifest name           ID #REQUIRED>
+<!ATTLIST submanifest remote         IDREF #IMPLIED>
+<!ATTLIST submanifest project        CDATA #IMPLIED>
+<!ATTLIST submanifest manifest\-name  CDATA #IMPLIED>
+<!ATTLIST submanifest revision       CDATA #IMPLIED>
+<!ATTLIST submanifest path           CDATA #IMPLIED>
+<!ATTLIST submanifest groups         CDATA #IMPLIED>
+<!ATTLIST submanifest default\-groups CDATA #IMPLIED>
 .TP
 <!ELEMENT project (annotation*,
 project*,
@@ -165,6 +190,8 @@ CDATA #IMPLIED>
 <!ATTLIST extend\-project groups CDATA #IMPLIED>
 <!ATTLIST extend\-project revision CDATA #IMPLIED>
 <!ATTLIST extend\-project remote CDATA #IMPLIED>
+<!ATTLIST extend\-project dest\-branch CDATA #IMPLIED>
+<!ATTLIST extend\-project upstream CDATA #IMPLIED>
 .IP
 <!ELEMENT remove\-project EMPTY>
 <!ATTLIST remove\-project name  CDATA #REQUIRED>
@@ -295,6 +322,65 @@ GetManifest(tag)
 Return a manifest in which each project is pegged to the revision at the
 specified tag. This is used by repo sync when the \fB\-\-smart\-tag\fR option is given.
 .PP
+Element submanifest
+.PP
+One or more submanifest elements may be specified. Each element describes a
+single manifest to be checked out as a child.
+.PP
+Attribute `name`: A unique name (within the current (sub)manifest) for this
+submanifest. It acts as a default for `revision` below. The same name can be
+used for submanifests with different parent (sub)manifests.
+.PP
+Attribute `remote`: Name of a previously defined remote element. If not supplied
+the remote given by the default element is used.
+.PP
+Attribute `project`: The manifest project name. The project's name is appended
+onto its remote's fetch URL to generate the actual URL to configure the Git
+remote with. The URL gets formed as:
+.IP
+${remote_fetch}/${project_name}.git
+.PP
+where ${remote_fetch} is the remote's fetch attribute and ${project_name} is the
+project's name attribute. The suffix ".git" is always appended as repo assumes
+the upstream is a forest of bare Git repositories. If the project has a parent
+element, its name will be prefixed by the parent's.
+.PP
+The project name must match the name Gerrit knows, if Gerrit is being used for
+code reviews.
+.PP
+`project` must not be empty, and may not be an absolute path or use "." or ".."
+path components. It is always interpreted relative to the remote's fetch
+settings, so if a different base path is needed, declare a different remote with
+the new settings needed.
+.PP
+If not supplied the remote and project for this manifest will be used: `remote`
+cannot be supplied.
+.PP
+Projects from a submanifest and its submanifests are added to the
+submanifest::path:<path_prefix> group.
+.PP
+Attribute `manifest\-name`: The manifest filename in the manifest project. If not
+supplied, `default.xml` is used.
+.PP
+Attribute `revision`: Name of a Git branch (e.g. "main" or "refs/heads/main"),
+tag (e.g. "refs/tags/stable"), or a commit hash. If not supplied, `name` is
+used.
+.PP
+Attribute `path`: An optional path relative to the top directory of the repo
+client where the submanifest repo client top directory should be placed. If not
+supplied, `revision` is used.
+.PP
+`path` may not be an absolute path or use "." or ".." path components.
+.PP
+Attribute `groups`: List of additional groups to which all projects in the
+included submanifest belong. This appends and recurses, meaning all projects in
+submanifests carry all parent submanifest groups. Same syntax as the
+corresponding element of `project`.
+.PP
+Attribute `default\-groups`: The list of manifest groups to sync if no
+`\-\-groups=` parameter was specified at init. When that list is empty, use this
+list instead of "default" as the list of groups to sync.
+.PP
 Element project
 .PP
 One or more project elements may be specified. Each element describes a single
@@ -401,6 +487,12 @@ project. Same syntax as the corresponding element of `project`.
 Attribute `remote`: If specified, overrides the remote of the original project.
 Same syntax as the corresponding element of `project`.
 .PP
+Attribute `dest\-branch`: If specified, overrides the dest\-branch of the original
+project. Same syntax as the corresponding element of `project`.
+.PP
+Attribute `upstream`: If specified, overrides the upstream of the original
+project. Same syntax as the corresponding element of `project`.
+.PP
 Element annotation
 .PP
 Zero or more annotation elements may be specified as children of a project or
@@ -513,10 +605,10 @@ restrictions are not enforced for [Local Manifests].
 .PP
 Attribute `groups`: List of additional groups to which all projects in the
 included manifest belong. This appends and recurses, meaning all projects in
-sub\-manifests carry all parent include groups. Same syntax as the corresponding
-element of `project`.
+included manifests carry all parent include groups. Same syntax as the
+corresponding element of `project`.
 .PP
-Local Manifests
+Local Manifests 
 .PP
 Additional remotes and projects may be added through local manifest files stored
 in `$TOP_DIR/.repo/local_manifests/*.xml`.

man/repo-overview.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo overview" "Repo Manual"
+.TH REPO "1" "July 2022" "repo overview" "Repo Manual"
 .SH NAME
 repo \- repo overview - manual page for repo overview
 .SH SYNOPSIS
@@ -26,6 +26,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help overview` to view the detailed manual.
 .SH DETAILS

man/repo-prune.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo prune" "Repo Manual"
+.TH REPO "1" "July 2022" "repo prune" "Repo Manual"
 .SH NAME
 repo \- repo prune - manual page for repo prune
 .SH SYNOPSIS
@@ -24,5 +24,18 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help prune` to view the detailed manual.

man/repo-rebase.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo rebase" "Repo Manual"
+.TH REPO "1" "July 2022" "repo rebase" "Repo Manual"
 .SH NAME
 repo \- repo rebase - manual page for repo rebase
 .SH SYNOPSIS
@@ -46,6 +46,19 @@ only show errors
 .TP
 \fB\-i\fR, \fB\-\-interactive\fR
 interactive rebase (single project only)
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help rebase` to view the detailed manual.
 .SH DETAILS

man/repo-selfupdate.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo selfupdate" "Repo Manual"
+.TH REPO "1" "July 2022" "repo selfupdate" "Repo Manual"
 .SH NAME
 repo \- repo selfupdate - manual page for repo selfupdate
 .SH SYNOPSIS
@@ -20,6 +20,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .SS repo Version options:
 .TP
 \fB\-\-no\-repo\-verify\fR

man/repo-smartsync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "November 2021" "repo smartsync" "Repo Manual"
+.TH REPO "1" "November 2022" "repo smartsync" "Repo Manual"
 .SH NAME
 repo \- repo smartsync - manual page for repo smartsync
 .SH SYNOPSIS
@@ -20,11 +20,11 @@ number of CPU cores)
 .TP
 \fB\-\-jobs\-network\fR=\fI\,JOBS\/\fR
 number of network jobs to run in parallel (defaults to
-\fB\-\-jobs\fR)
+\fB\-\-jobs\fR or 1)
 .TP
 \fB\-\-jobs\-checkout\fR=\fI\,JOBS\/\fR
 number of local checkout jobs to run in parallel
-(defaults to \fB\-\-jobs\fR)
+(defaults to \fB\-\-jobs\fR or 8)
 .TP
 \fB\-f\fR, \fB\-\-force\-broken\fR
 obsolete option (to be deleted in the future)
@@ -105,6 +105,13 @@ delete refs that no longer exist on the remote
 .TP
 \fB\-\-no\-prune\fR
 do not delete refs that no longer exist on the remote
+.TP
+\fB\-\-auto\-gc\fR
+run garbage collection on all synced projects
+.TP
+\fB\-\-no\-auto\-gc\fR
+do not run garbage collection on any projects
+(default)
 .SS Logging options:
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR
@@ -112,6 +119,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .SS repo Version options:
 .TP
 \fB\-\-no\-repo\-verify\fR

man/repo-stage.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo stage" "Repo Manual"
+.TH REPO "1" "July 2022" "repo stage" "Repo Manual"
 .SH NAME
 repo \- repo stage - manual page for repo stage
 .SH SYNOPSIS
@@ -23,6 +23,19 @@ only show errors
 .TP
 \fB\-i\fR, \fB\-\-interactive\fR
 use interactive staging
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help stage` to view the detailed manual.
 .SH DETAILS

man/repo-start.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo start" "Repo Manual"
+.TH REPO "1" "July 2022" "repo start" "Repo Manual"
 .SH NAME
 repo \- repo start - manual page for repo start
 .SH SYNOPSIS
@@ -33,6 +33,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help start` to view the detailed manual.
 .SH DETAILS

man/repo-status.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo status" "Repo Manual"
+.TH REPO "1" "July 2022" "repo status" "Repo Manual"
 .SH NAME
 repo \- repo status - manual page for repo status
 .SH SYNOPSIS
@@ -28,6 +28,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help status` to view the detailed manual.
 .SH DETAILS

man/repo-sync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "November 2021" "repo sync" "Repo Manual"
+.TH REPO "1" "November 2022" "repo sync" "Repo Manual"
 .SH NAME
 repo \- repo sync - manual page for repo sync
 .SH SYNOPSIS
@@ -20,11 +20,11 @@ number of CPU cores)
 .TP
 \fB\-\-jobs\-network\fR=\fI\,JOBS\/\fR
 number of network jobs to run in parallel (defaults to
-\fB\-\-jobs\fR)
+\fB\-\-jobs\fR or 1)
 .TP
 \fB\-\-jobs\-checkout\fR=\fI\,JOBS\/\fR
 number of local checkout jobs to run in parallel
-(defaults to \fB\-\-jobs\fR)
+(defaults to \fB\-\-jobs\fR or 8)
 .TP
 \fB\-f\fR, \fB\-\-force\-broken\fR
 obsolete option (to be deleted in the future)
@@ -106,6 +106,13 @@ delete refs that no longer exist on the remote
 \fB\-\-no\-prune\fR
 do not delete refs that no longer exist on the remote
 .TP
+\fB\-\-auto\-gc\fR
+run garbage collection on all synced projects
+.TP
+\fB\-\-no\-auto\-gc\fR
+do not run garbage collection on any projects
+(default)
+.TP
 \fB\-s\fR, \fB\-\-smart\-sync\fR
 smart sync using manifest from the latest known good
 build
@@ -119,6 +126,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .SS repo Version options:
 .TP
 \fB\-\-no\-repo\-verify\fR
@@ -187,6 +207,9 @@ to a sha1 revision if the sha1 revision does not already exist locally.
 The \fB\-\-prune\fR option can be used to remove any refs that no longer exist on the
 remote.
 .PP
+The \fB\-\-auto\-gc\fR option can be used to trigger garbage collection on all projects.
+By default, repo does not run garbage collection.
+.PP
 SSH Connections
 .PP
 If at least one project remote URL uses an SSH connection (ssh://, git+ssh://,

man/repo-upload.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo upload" "Repo Manual"
+.TH REPO "1" "August 2022" "repo upload" "Repo Manual"
 .SH NAME
 repo \- repo upload - manual page for repo upload
 .SH SYNOPSIS
@@ -54,6 +54,9 @@ upload as a private change (deprecated; use \fB\-\-wip\fR)
 \fB\-w\fR, \fB\-\-wip\fR
 upload as a work\-in\-progress change
 .TP
+\fB\-r\fR, \fB\-\-ready\fR
+mark change as ready (clears work\-in\-progress setting)
+.TP
 \fB\-o\fR PUSH_OPTIONS, \fB\-\-push\-option\fR=\fI\,PUSH_OPTIONS\/\fR
 additional push options to transmit
 .TP
@@ -66,6 +69,12 @@ do everything except actually upload the CL
 \fB\-y\fR, \fB\-\-yes\fR
 answer yes to all safe prompts
 .TP
+\fB\-\-ignore\-untracked\-files\fR
+ignore untracked files in the working copy
+.TP
+\fB\-\-no\-ignore\-untracked\-files\fR
+always ask about untracked files in the working copy
+.TP
 \fB\-\-no\-cert\-checks\fR
 disable verifying ssl certs (unsafe)
 .SS Logging options:
@@ -75,6 +84,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .SS pre\-upload hooks:
 .TP
 \fB\-\-no\-verify\fR
@@ -105,6 +127,12 @@ respective list of users, and emails are sent to any new users. Users passed as
 \fB\-\-reviewers\fR must already be registered with the code review system, or the
 upload will fail.
 .PP
+While most normal Gerrit options have dedicated command line options, direct
+access to the Gerit options is available via \fB\-\-push\-options\fR. This is useful when
+Gerrit has newer functionality that repo upload doesn't yet support, or doesn't
+have plans to support. See the Push Options documentation for more details:
+https://gerrit\-review.googlesource.com/Documentation/user\-upload.html#push_options
+.PP
 Configuration
 .PP
 review.URL.autoupload:

man/repo-version.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo version" "Repo Manual"
+.TH REPO "1" "July 2022" "repo version" "Repo Manual"
 .SH NAME
 repo \- repo version - manual page for repo version
 .SH SYNOPSIS
@@ -20,5 +20,18 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help version` to view the detailed manual.

man/repo.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "November 2021" "repo" "Repo Manual"
+.TH REPO "1" "November 2022" "repo" "Repo Manual"
 .SH NAME
 repo \- repository management tool built on top of git
 .SH SYNOPSIS
@@ -25,6 +25,10 @@ control color usage: auto, always, never
 \fB\-\-trace\fR
 trace git command execution (REPO_TRACE=1)
 .TP
+\fB\-\-trace\-to\-stderr\fR
+trace outputs go to stderr in addition to
+\&.repo/TRACE_FILE
+.TP
 \fB\-\-trace\-python\fR
 trace python command execution
 .TP
@@ -43,6 +47,9 @@ filename of event log to append timeline to
 .TP
 \fB\-\-git\-trace2\-event\-log\fR=\fI\,GIT_TRACE2_EVENT_LOG\/\fR
 directory to write git trace2 event log to
+.TP
+\fB\-\-submanifest\-path\fR=\fI\,REL_PATH\/\fR
+submanifest path
 .SS "The complete list of recognized repo commands is:"
 .TP
 abandon

manifest_xml.py

@@ -123,7 +123,7 @@ class _Default(object):
   destBranchExpr = None
   upstreamExpr = None
   remote = None
-  sync_j = 1
+  sync_j = None
   sync_c = False
   sync_s = False
   sync_tags = True
@@ -214,6 +214,7 @@ class _XmlSubmanifest:
     revision: a string, the commitish.
     manifestName: a string, the submanifest file name.
     groups: a list of strings, the groups to add to all projects in the submanifest.
+    default_groups: a list of strings, the default groups to sync.
     path: a string, the relative path for the submanifest checkout.
     parent: an XmlManifest, the parent manifest.
     annotations: (derived) a list of annotations.
@@ -226,6 +227,7 @@ class _XmlSubmanifest:
                revision=None,
                manifestName=None,
                groups=None,
+               default_groups=None,
                path=None,
                parent=None):
     self.name = name
@@ -234,6 +236,7 @@ class _XmlSubmanifest:
     self.revision = revision
     self.manifestName = manifestName
     self.groups = groups
+    self.default_groups = default_groups
     self.path = path
     self.parent = parent
     self.annotations = []
@@ -250,7 +253,8 @@ class _XmlSubmanifest:
         os.path.join(parent.path_prefix, self.relpath), MANIFEST_FILE_NAME)
     rc = self.repo_client = RepoClient(
         parent.repodir, linkFile, parent_groups=','.join(groups) or '',
-        submanifest_path=self.relpath, outer_client=outer_client)
+        submanifest_path=self.relpath, outer_client=outer_client,
+        default_groups=default_groups)
 
     self.present = os.path.exists(manifestFile)
 
@@ -264,6 +268,7 @@ class _XmlSubmanifest:
         self.revision == other.revision and
         self.manifestName == other.manifestName and
         self.groups == other.groups and
+        self.default_groups == other.default_groups and
         self.path == other.path and
         sorted(self.annotations) == sorted(other.annotations))
 
@@ -279,11 +284,12 @@ class _XmlSubmanifest:
     if self.project:
       manifestUrl = remote.ToRemoteSpec(self.project).url
     else:
-      manifestUrl = mp.GetRemote(mp.remote.name).url
+      manifestUrl = mp.GetRemote().url
     manifestName = self.manifestName or 'default.xml'
     revision = self.revision or self.name
     path = self.path or revision.split('/')[-1]
     groups = self.groups or []
+    default_groups = self.default_groups or []
 
     return SubmanifestSpec(self.name, manifestUrl, manifestName, revision, path,
                            groups)
@@ -300,6 +306,10 @@ class _XmlSubmanifest:
       return ','.join(self.groups)
     return ''
 
+  def GetDefaultGroupsStr(self):
+    """Returns the `default-groups` given for this submanifest."""
+    return ','.join(self.default_groups or [])
+
   def AddAnnotation(self, name, value, keep):
     """Add annotations to the submanifest."""
     self.annotations.append(Annotation(name, value, keep))
@@ -327,7 +337,8 @@ class XmlManifest(object):
   """manages the repo configuration file"""
 
   def __init__(self, repodir, manifest_file, local_manifests=None,
-               outer_client=None, parent_groups='', submanifest_path=''):
+               outer_client=None, parent_groups='', submanifest_path='',
+               default_groups=None):
     """Initialize.
 
     Args:
@@ -337,16 +348,20 @@ class XmlManifest(object):
           be |repodir|/|MANIFEST_FILE_NAME|.
       local_manifests: Full path to the directory of local override manifests.
           This will usually be |repodir|/|LOCAL_MANIFESTS_DIR_NAME|.
-      outer_client: RepoClient of the outertree.
+      outer_client: RepoClient of the outer manifest.
       parent_groups: a string, the groups to apply to this projects.
       submanifest_path: The submanifest root relative to the repo root.
+      default_groups: a string, the default manifest groups to use.
     """
     # TODO(vapier): Move this out of this class.
     self.globalConfig = GitConfig.ForUser()
 
     self.repodir = os.path.abspath(repodir)
     self._CheckLocalPath(submanifest_path)
-    self.topdir = os.path.join(os.path.dirname(self.repodir), submanifest_path)
+    self.topdir = os.path.dirname(self.repodir)
+    if submanifest_path:
+      # This avoids a trailing os.path.sep when submanifest_path is empty.
+      self.topdir = os.path.join(self.topdir, submanifest_path)
     if manifest_file != os.path.abspath(manifest_file):
       raise ManifestParseError('manifest_file must be abspath')
     self.manifestFile = manifest_file
@@ -358,6 +373,7 @@ class XmlManifest(object):
     self.local_manifests = local_manifests
     self._load_local_manifests = True
     self.parent_groups = parent_groups
+    self.default_groups = default_groups
 
     if outer_client and self.isGitcClient:
       raise ManifestParseError('Multi-manifest is incompatible with `gitc-init`')
@@ -472,6 +488,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
       e.setAttribute('path', r.path)
     if r.groups:
       e.setAttribute('groups', r.GetGroupsStr())
+    if r.default_groups:
+      e.setAttribute('default-groups', r.GetDefaultGroupsStr())
 
     for a in r.annotations:
       if a.keep == 'true':
@@ -487,7 +505,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     """
     return [x for x in re.split(r'[,\s]+', field) if x]
 
-  def ToXml(self, peg_rev=False, peg_rev_upstream=True, peg_rev_dest_branch=True, groups=None):
+  def ToXml(self, peg_rev=False, peg_rev_upstream=True,
+            peg_rev_dest_branch=True, groups=None, omit_local=False):
     """Return the current manifest XML."""
     mp = self.manifestProject
 
@@ -532,7 +551,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     if d.upstreamExpr:
       have_default = True
       e.setAttribute('upstream', d.upstreamExpr)
-    if d.sync_j > 1:
+    if d.sync_j is not None:
       have_default = True
       e.setAttribute('sync-j', '%d' % d.sync_j)
     if d.sync_c:
@@ -568,6 +587,9 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
       if not p.MatchesGroups(groups):
         return
 
+      if omit_local and self.IsFromLocalManifest(p):
+        return
+
       name = p.name
       relpath = p.relpath
       if parent:
@@ -753,23 +775,29 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
   @property
   def is_multimanifest(self):
-    """Whether this is a multimanifest checkout"""
-    return bool(self.outer_client.submanifests)
+    """Whether this is a multimanifest checkout.
+
+    This is safe to use as long as the outermost manifest XML has been parsed.
+    """
+    return bool(self._outer_client._submanifests)
 
   @property
   def is_submanifest(self):
-    """Whether this manifest is a submanifest"""
+    """Whether this manifest is a submanifest.
+
+    This is safe to use as long as the outermost manifest XML has been parsed.
+    """
     return self._outer_client and self._outer_client != self
 
   @property
   def outer_client(self):
-    """The instance of the outermost manifest client"""
+    """The instance of the outermost manifest client."""
     self._Load()
     return self._outer_client
 
   @property
   def all_manifests(self):
-    """Generator yielding all (sub)manifests."""
+    """Generator yielding all (sub)manifests, in depth-first order."""
     self._Load()
     outer = self._outer_client
     yield outer
@@ -778,7 +806,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
   @property
   def all_children(self):
-    """Generator yielding all child submanifests."""
+    """Generator yielding all (present) child submanifests."""
     self._Load()
     for child in self._submanifests.values():
       if child.repo_client:
@@ -795,7 +823,14 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
   @property
   def all_paths(self):
-    """All project paths for all (sub)manifests.  See `paths`."""
+    """All project paths for all (sub)manifests.
+
+    See also `paths`.
+
+    Returns:
+      A dictionary of {path: Project()}.  `path` is relative to the outer
+      manifest.
+    """
     ret = {}
     for tree in self.all_manifests:
       prefix = tree.path_prefix
@@ -811,7 +846,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
   def paths(self):
     """Return all paths for this manifest.
 
-    Return:
+    Returns:
       A dictionary of {path: Project()}.  `path` is relative to this manifest.
     """
     self._Load()
@@ -825,11 +860,13 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
   @property
   def remotes(self):
+    """Return a list of remotes for this manifest."""
     self._Load()
     return self._remotes
 
   @property
   def default(self):
+    """Return default values for this manifest."""
     self._Load()
     return self._default
 
@@ -967,16 +1004,21 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                          worktree=os.path.join(subdir, 'manifests'))
     return mp
 
-  def GetDefaultGroupsStr(self):
-    """Returns the default group string for the platform."""
-    return 'default,platform-' + platform.system().lower()
+  def GetDefaultGroupsStr(self, with_platform=True):
+    """Returns the default group string to use.
+
+    Args:
+      with_platform: a boolean, whether to include the group for the
+                     underlying platform.
+    """
+    groups = ','.join(self.default_groups or ['default'])
+    if with_platform:
+      groups += f',platform-{platform.system().lower()}'
+    return groups
 
   def GetGroupsStr(self):
     """Returns the manifest group string that should be synced."""
-    groups = self.manifestProject.manifest_groups
-    if not groups:
-      groups = self.GetDefaultGroupsStr()
-    return groups
+    return self.manifestProject.manifest_groups or self.GetDefaultGroupsStr()
 
   def Unload(self):
     """Unload the manifest.
@@ -1067,8 +1109,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         if override:
           self.manifestFile = savedManifestFile
 
-      # Now that we have loaded this manifest, load any submanifest  manifests
-      # as well.  We need to do this after self._loaded is set to avoid looping.
+      # Now that we have loaded this manifest, load any submanifests as well.
+      # We need to do this after self._loaded is set to avoid looping.
       for name in self._submanifests:
         tree = self._submanifests[name]
         spec = tree.ToSubmanifestSpec()
@@ -1247,6 +1289,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
           remote = self._default.remote
         else:
           remote = self._get_remote(node)
+        dest_branch = node.getAttribute('dest-branch')
+        upstream = node.getAttribute('upstream')
 
         named_projects = self._projects[name]
         if dest_path and not path and len(named_projects) > 1:
@@ -1262,6 +1306,10 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
           if remote_name:
             p.remote = remote.ToRemoteSpec(name)
+          if dest_branch:
+            p.dest_branch = dest_branch
+          if upstream:
+            p.upstream = upstream
 
           if dest_path:
             del self._paths[p.relpath]
@@ -1346,7 +1394,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
   def _AddMetaProjectMirror(self, m):
     name = None
-    m_url = m.GetRemote(m.remote.name).url
+    m_url = m.GetRemote().url
     if m_url.endswith('/.git'):
       raise ManifestParseError('refusing to mirror %s' % m_url)
 
@@ -1423,8 +1471,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     d.destBranchExpr = node.getAttribute('dest-branch') or None
     d.upstreamExpr = node.getAttribute('upstream') or None
 
-    d.sync_j = XmlInt(node, 'sync-j', 1)
-    if d.sync_j <= 0:
+    d.sync_j = XmlInt(node, 'sync-j', None)
+    if d.sync_j is not None and d.sync_j <= 0:
       raise ManifestParseError('%s: sync-j must be greater than 0, not "%s"' %
                                (self.manifestFile, d.sync_j))
 
@@ -1491,6 +1539,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     if node.hasAttribute('groups'):
       groups = node.getAttribute('groups')
     groups = self._ParseList(groups)
+    default_groups = self._ParseList(node.getAttribute('default-groups'))
     path = node.getAttribute('path')
     if path == '':
       path = None
@@ -1511,7 +1560,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
             '<submanifest> invalid "path": %s: %s' % (path, msg))
 
     submanifest = _XmlSubmanifest(name, remote, project, revision, manifestName,
-                                  groups, path, self)
+                                  groups, default_groups, path, self)
 
     for n in node.childNodes:
       if n.nodeName == 'annotation':
@@ -1635,6 +1684,10 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
       name: a string, the name of the project.
       path: a string, the path of the project.
       remote: a string, the remote.name of the project.
+
+    Returns:
+      A tuple of (relpath, worktree, gitdir, objdir, use_git_worktrees) for the
+      project with |name| and |path|.
     """
     # The manifest entries might have trailing slashes.  Normalize them to avoid
     # unexpected filesystem behavior since we do string concatenation below.
@@ -1642,7 +1695,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     name = name.rstrip('/')
     remote = remote.rstrip('/')
     use_git_worktrees = False
-    use_remote_name = bool(self._outer_client._submanifests)
+    use_remote_name = self.is_multimanifest
     relpath = path
     if self.IsMirror:
       worktree = None
@@ -1658,7 +1711,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
       # We allow people to mix git worktrees & non-git worktrees for now.
       # This allows for in situ migration of repo clients.
       if os.path.exists(gitdir) or not self.UseGitWorktrees:
-        objdir = os.path.join(self.subdir, 'project-objects', namepath)
+        objdir = os.path.join(self.repodir, 'project-objects', namepath)
       else:
         use_git_worktrees = True
         gitdir = os.path.join(self.repodir, 'worktrees', namepath)
@@ -1672,6 +1725,9 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
       name: a string, the name of the project.
       all_manifests: a boolean, if True, then all manifests are searched. If
                      False, then only this manifest is searched.
+
+    Returns:
+      A list of Project instances with name |name|.
     """
     if all_manifests:
       return list(itertools.chain.from_iterable(
@@ -1890,11 +1946,14 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     fromKeys = sorted(fromProjects.keys())
     toKeys = sorted(toProjects.keys())
 
-    diff = {'added': [], 'removed': [], 'changed': [], 'unreachable': []}
+    diff = {'added': [], 'removed': [], 'missing': [], 'changed': [], 'unreachable': []}
 
     for proj in fromKeys:
       if proj not in toKeys:
         diff['removed'].append(fromProjects[proj])
+      elif not fromProjects[proj].Exists:
+        diff['missing'].append(toProjects[proj])
+        toKeys.remove(proj)
       else:
         fromProj = fromProjects[proj]
         toProj = toProjects[proj]
@@ -1932,6 +1991,16 @@ class RepoClient(XmlManifest):
   """Manages a repo client checkout."""
 
   def __init__(self, repodir, manifest_file=None, submanifest_path='', **kwargs):
+    """Initialize.
+
+    Args:
+      repodir: Path to the .repo/ dir for holding all internal checkout state.
+          It must be in the top directory of the repo client checkout.
+      manifest_file: Full path to the manifest file to parse.  This will usually
+          be |repodir|/|MANIFEST_FILE_NAME|.
+      submanifest_path: The submanifest root relative to the repo root.
+      **kwargs: Additional keyword arguments, passed to XmlManifest.
+    """
     self.isGitcClient = False
     submanifest_path = submanifest_path or ''
     if submanifest_path:

pager.py

@@ -56,8 +56,11 @@ def _PipePager(pager):
   global pager_process, old_stdout, old_stderr
   assert pager_process is None, "Only one active pager process at a time"
   # Create pager process, piping stdout/err into its stdin
-  pager_process = subprocess.Popen([pager], stdin=subprocess.PIPE, stdout=sys.stdout,
-                                   stderr=sys.stderr)
+  try:
+    pager_process = subprocess.Popen([pager], stdin=subprocess.PIPE, stdout=sys.stdout,
+                                     stderr=sys.stderr)
+  except FileNotFoundError:
+    sys.exit(f'fatal: cannot start pager "{pager}"')
   old_stdout = sys.stdout
   old_stderr = sys.stderr
   sys.stdout = pager_process.stdin

progress.py

@@ -15,7 +15,7 @@
 import os
 import sys
 from time import time
-from repo_trace import IsTrace
+from repo_trace import IsTraceToStderr
 
 _NOT_TTY = not os.isatty(2)
 
@@ -80,7 +80,7 @@ class Progress(object):
   def update(self, inc=1, msg=''):
     self._done += inc
 
-    if _NOT_TTY or IsTrace():
+    if _NOT_TTY or IsTraceToStderr():
       return
 
     if not self._show:
@@ -113,7 +113,7 @@ class Progress(object):
       sys.stderr.flush()
 
   def end(self):
-    if _NOT_TTY or IsTrace() or not self._show:
+    if _NOT_TTY or IsTraceToStderr() or not self._show:
       return
 
     duration = duration_str(time() - self._start)

project.py

@@ -26,28 +26,42 @@ import sys
 import tarfile
 import tempfile
 import time
+from typing import NamedTuple
 import urllib.parse
 
 from color import Coloring
+import fetch
 from git_command import GitCommand, git_require
 from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \
     ID_RE
+import git_superproject
 from git_trace2_event_log import EventLog
 from error import GitError, UploadError, DownloadError
 from error import ManifestInvalidRevisionError, ManifestInvalidPathError
 from error import NoManifestException, ManifestParseError
 import platform_utils
 import progress
-from repo_trace import IsTrace, Trace
+from repo_trace import Trace
 
 from git_refs import GitRefs, HEAD, R_HEADS, R_TAGS, R_PUB, R_M, R_WORKTREE_M
 
 
+class SyncNetworkHalfResult(NamedTuple):
+  """Sync_NetworkHalf return value."""
+  # True if successful.
+  success: bool
+  # Did we query the remote? False when optimized_fetch is True and we have the
+  # commit already present.
+  remote_fetched: bool
+
 # Maximum sleep time allowed during retries.
 MAXIMUM_RETRY_SLEEP_SEC = 3600.0
 # +-10% random jitter is added to each Fetches retry sleep duration.
 RETRY_JITTER_PERCENT = 0.1
 
+# Whether to use alternates.  Switching back and forth is *NOT* supported.
+# TODO(vapier): Remove knob once behavior is verified.
+_ALTERNATES = os.environ.get('REPO_USE_ALTERNATES') == '1'
 
 def _lwrite(path, content):
   lock = '%s.lock' % path
@@ -200,6 +214,7 @@ class ReviewableBranch(object):
                       private=False,
                       notify=None,
                       wip=False,
+                      ready=False,
                       dest_branch=None,
                       validate_certs=True,
                       push_options=None):
@@ -212,6 +227,7 @@ class ReviewableBranch(object):
                                  private=private,
                                  notify=notify,
                                  wip=wip,
+                                 ready=ready,
                                  dest_branch=dest_branch,
                                  validate_certs=validate_certs,
                                  push_options=push_options)
@@ -459,7 +475,13 @@ class RemoteSpec(object):
 
 class Project(object):
   # These objects can be shared between several working trees.
-  shareable_dirs = ['hooks', 'objects', 'rr-cache']
+  @property
+  def shareable_dirs(self):
+    """Return the shareable directories"""
+    if self.UseAlternates:
+      return ['hooks', 'rr-cache']
+    else:
+      return ['hooks', 'objects', 'rr-cache']
 
   def __init__(self,
                manifest,
@@ -589,6 +611,14 @@ class Project(object):
     self.bare_ref = GitRefs(self.gitdir)
     self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=self.objdir)
 
+  @property
+  def UseAlternates(self):
+    """Whether git alternates are in use.
+
+    This will be removed once migration to alternates is complete.
+    """
+    return _ALTERNATES or self.manifest.is_multimanifest
+
   @property
   def Derived(self):
     return self.is_derived
@@ -631,7 +661,7 @@ class Project(object):
       return True
     if self.work_git.DiffZ('diff-files'):
       return True
-    if consider_untracked and self.work_git.LsOthers():
+    if consider_untracked and self.UntrackedFiles():
       return True
     return False
 
@@ -665,9 +695,13 @@ class Project(object):
       self._userident_name = ''
       self._userident_email = ''
 
-  def GetRemote(self, name):
+  def GetRemote(self, name=None):
     """Get the configuration for a single remote.
+
+    Defaults to the current project's remote.
     """
+    if name is None:
+      name = self.remote.name
     return self.config.GetRemote(name)
 
   def GetBranch(self, name):
@@ -714,7 +748,8 @@ class Project(object):
        The special manifest group "default" will match any project that
        does not have the special project group "notdefault"
     """
-    expanded_manifest_groups = manifest_groups or ['default']
+    default_groups = self.manifest.default_groups or ['default']
+    expanded_manifest_groups = manifest_groups or default_groups
     expanded_project_groups = ['all'] + (self.groups or [])
     if 'notdefault' not in expanded_project_groups:
       expanded_project_groups += ['default']
@@ -759,33 +794,37 @@ class Project(object):
       if not get_all:
         return details
 
-    changes = self.work_git.LsOthers()
+    changes = self.UntrackedFiles()
     if changes:
       details.extend(changes)
 
     return details
 
+  def UntrackedFiles(self):
+    """Returns a list of strings, untracked files in the git tree."""
+    return self.work_git.LsOthers()
+
   def HasChanges(self):
     """Returns true if there are uncommitted changes.
     """
-    if self.UncommitedFiles(get_all=False):
-      return True
-    else:
-      return False
+    return bool(self.UncommitedFiles(get_all=False))
 
-  def PrintWorkTreeStatus(self, output_redir=None, quiet=False):
+  def PrintWorkTreeStatus(self, output_redir=None, quiet=False, local=False):
     """Prints the status of the repository to stdout.
 
     Args:
       output_redir: If specified, redirect the output to this object.
       quiet:  If True then only print the project name.  Do not print
               the modified files, branch name, etc.
+      local: a boolean, if True, the path is relative to the local
+             (sub)manifest.  If false, the path is relative to the
+             outermost manifest.
     """
     if not platform_utils.isdir(self.worktree):
       if output_redir is None:
         output_redir = sys.stdout
       print(file=output_redir)
-      print('project %s/' % self.relpath, file=output_redir)
+      print('project %s/' % self.RelPath(local), file=output_redir)
       print('  missing (run "repo sync")', file=output_redir)
       return
 
@@ -803,7 +842,7 @@ class Project(object):
     out = StatusColoring(self.config)
     if output_redir is not None:
       out.redirect(output_redir)
-    out.project('project %-40s', self.relpath + '/ ')
+    out.project('project %-40s', self.RelPath(local) + '/ ')
 
     if quiet:
       out.nl()
@@ -864,7 +903,8 @@ class Project(object):
 
     return 'DIRTY'
 
-  def PrintWorkTreeDiff(self, absolute_paths=False, output_redir=None):
+  def PrintWorkTreeDiff(self, absolute_paths=False, output_redir=None,
+                        local=False):
     """Prints the status of the repository to stdout.
     """
     out = DiffColoring(self.config)
@@ -875,8 +915,8 @@ class Project(object):
       cmd.append('--color')
     cmd.append(HEAD)
     if absolute_paths:
-      cmd.append('--src-prefix=a/%s/' % self.relpath)
-      cmd.append('--dst-prefix=b/%s/' % self.relpath)
+      cmd.append('--src-prefix=a/%s/' % self.RelPath(local))
+      cmd.append('--dst-prefix=b/%s/' % self.RelPath(local))
     cmd.append('--')
     try:
       p = GitCommand(self,
@@ -886,14 +926,14 @@ class Project(object):
       p.Wait()
     except GitError as e:
       out.nl()
-      out.project('project %s/' % self.relpath)
+      out.project('project %s/' % self.RelPath(local))
       out.nl()
       out.fail('%s', str(e))
       out.nl()
       return False
     if p.stdout:
       out.nl()
-      out.project('project %s/' % self.relpath)
+      out.project('project %s/' % self.RelPath(local))
       out.nl()
       out.write('%s', p.stdout)
     return p.Wait() == 0
@@ -978,6 +1018,7 @@ class Project(object):
                       private=False,
                       notify=None,
                       wip=False,
+                      ready=False,
                       dest_branch=None,
                       validate_certs=True,
                       push_options=None):
@@ -994,6 +1035,13 @@ class Project(object):
     if not branch.remote.review:
       raise GitError('remote %s has no review url' % branch.remote.name)
 
+    # Basic validity check on label syntax.
+    for label in labels:
+      if not re.match(r'^.+[+-][0-9]+$', label):
+        raise UploadError(
+            f'invalid label syntax "{label}": labels use forms like '
+            'CodeReview+1 or Verified-1')
+
     if dest_branch is None:
       dest_branch = self.dest_branch
     if dest_branch is None:
@@ -1029,6 +1077,7 @@ class Project(object):
     if auto_topic:
       opts += ['topic=' + branch.name]
     opts += ['t=%s' % p for p in hashtags]
+    # NB: No need to encode labels as they've been validated above.
     opts += ['l=%s' % p for p in labels]
 
     opts += ['r=%s' % p for p in people[0]]
@@ -1039,6 +1088,8 @@ class Project(object):
       opts += ['private']
     if wip:
       opts += ['wip']
+    if ready:
+      opts += ['ready']
     if opts:
       ref_spec = ref_spec + '%' + ','.join(opts)
     cmd.append(ref_spec)
@@ -1091,7 +1142,7 @@ class Project(object):
     if archive and not isinstance(self, MetaProject):
       if self.remote.url.startswith(('http://', 'https://')):
         _error("%s: Cannot fetch archives from http/https remotes.", self.name)
-        return False
+        return SyncNetworkHalfResult(False, False)
 
       name = self.relpath.replace('\\', '/')
       name = name.replace('/', '_')
@@ -1102,19 +1153,19 @@ class Project(object):
         self._FetchArchive(tarpath, cwd=topdir)
       except GitError as e:
         _error('%s', e)
-        return False
+        return SyncNetworkHalfResult(False, False)
 
       # From now on, we only need absolute tarpath
       tarpath = os.path.join(topdir, tarpath)
 
       if not self._ExtractArchive(tarpath, path=topdir):
-        return False
+        return SyncNetworkHalfResult(False, True)
       try:
         platform_utils.remove(tarpath)
       except OSError as e:
         _warn("Cannot remove archive %s: %s", tarpath, str(e))
       self._CopyAndLinkFiles()
-      return True
+      return SyncNetworkHalfResult(True, True)
 
     # If the shared object dir already exists, don't try to rebootstrap with a
     # clone bundle download.  We should have the majority of objects already.
@@ -1133,6 +1184,17 @@ class Project(object):
       self._UpdateHooks(quiet=quiet)
     self._InitRemote()
 
+    if self.UseAlternates:
+      # If gitdir/objects is a symlink, migrate it from the old layout.
+      gitdir_objects = os.path.join(self.gitdir, 'objects')
+      if platform_utils.islink(gitdir_objects):
+        platform_utils.remove(gitdir_objects, missing_ok=True)
+      gitdir_alt = os.path.join(self.gitdir, 'objects/info/alternates')
+      if not os.path.exists(gitdir_alt):
+        os.makedirs(os.path.dirname(gitdir_alt), exist_ok=True)
+        _lwrite(gitdir_alt, os.path.join(
+            os.path.relpath(self.objdir, gitdir_objects), 'objects') + '\n')
+
     if is_new:
       alt = os.path.join(self.objdir, 'objects/info/alternates')
       try:
@@ -1167,9 +1229,11 @@ class Project(object):
       depth = self.manifest.manifestProject.depth
 
     # See if we can skip the network fetch entirely.
+    remote_fetched = False
     if not (optimized_fetch and
             (ID_RE.match(self.revisionExpr) and
              self._CheckForImmutableRevision())):
+      remote_fetched = True
       if not self._RemoteFetch(
               initial=is_new,
               quiet=quiet, verbose=verbose, output_redir=output_redir,
@@ -1178,7 +1242,7 @@ class Project(object):
               submodules=submodules, force_sync=force_sync,
               ssh_proxy=ssh_proxy,
               clone_filter=clone_filter, retry_fetches=retry_fetches):
-        return False
+        return SyncNetworkHalfResult(False, remote_fetched)
 
     mp = self.manifest.manifestProject
     dissociate = mp.dissociate
@@ -1191,7 +1255,7 @@ class Project(object):
         if p.stdout and output_redir:
           output_redir.write(p.stdout)
         if p.Wait() != 0:
-          return False
+          return SyncNetworkHalfResult(False, remote_fetched)
         platform_utils.remove(alternates_file)
 
     if self.worktree:
@@ -1200,7 +1264,7 @@ class Project(object):
       self._InitMirrorHead()
       platform_utils.remove(os.path.join(self.gitdir, 'FETCH_HEAD'),
                             missing_ok=True)
-    return True
+    return SyncNetworkHalfResult(True, remote_fetched)
 
   def PostRepoUpgrade(self):
     self._InitHooks()
@@ -1233,7 +1297,7 @@ class Project(object):
     if self.revisionId:
       return self.revisionId
 
-    rem = self.GetRemote(self.remote.name)
+    rem = self.GetRemote()
     rev = rem.ToLocal(self.revisionExpr)
 
     if all_refs is not None and rev in all_refs:
@@ -1398,6 +1462,8 @@ class Project(object):
         cnt_mine += 1
 
     if not upstream_gain and cnt_mine == len(local_changes):
+      # The copy/linkfile config may have changed.
+      self._CopyAndLinkFiles()
       return
 
     if self.IsDirty(consider_untracked=False):
@@ -1425,7 +1491,7 @@ class Project(object):
                    "discarding %d commits removed from upstream",
                    len(local_changes) - cnt_mine)
 
-    branch.remote = self.GetRemote(self.remote.name)
+    branch.remote = self.GetRemote()
     if not ID_RE.match(self.revisionExpr):
       # in case of manifest sync the revisionExpr might be a SHA1
       branch.merge = self.revisionExpr
@@ -1483,7 +1549,7 @@ class Project(object):
   def DownloadPatchSet(self, change_id, patch_id):
     """Download a single patch set of a single change to FETCH_HEAD.
     """
-    remote = self.GetRemote(self.remote.name)
+    remote = self.GetRemote()
 
     cmd = ['fetch', remote.name]
     cmd.append('refs/changes/%2.2d/%d/%d'
@@ -1513,14 +1579,14 @@ class Project(object):
     if self.IsDirty():
       if force:
         print('warning: %s: Removing dirty project: uncommitted changes lost.' %
-              (self.relpath,), file=sys.stderr)
+              (self.RelPath(local=False),), file=sys.stderr)
       else:
         print('error: %s: Cannot remove project: uncommitted changes are '
-              'present.\n' % (self.relpath,), file=sys.stderr)
+              'present.\n' % (self.RelPath(local=False),), file=sys.stderr)
         return False
 
     if not quiet:
-      print('%s: Deleting obsolete checkout.' % (self.relpath,))
+      print('%s: Deleting obsolete checkout.' % (self.RelPath(local=False),))
 
     # Unlock and delink from the main worktree.  We don't use git's worktree
     # remove because it will recursively delete projects -- we handle that
@@ -1559,7 +1625,8 @@ class Project(object):
       if e.errno != errno.ENOENT:
         print('error: %s: %s' % (self.gitdir, e), file=sys.stderr)
         print('error: %s: Failed to delete obsolete checkout; remove manually, '
-              'then run `repo sync -l`.' % (self.relpath,), file=sys.stderr)
+              'then run `repo sync -l`.' % (self.RelPath(local=False),),
+              file=sys.stderr)
         return False
 
     # Delete everything under the worktree, except for directories that contain
@@ -1595,7 +1662,7 @@ class Project(object):
             print('error: %s: Failed to remove: %s' % (d, e), file=sys.stderr)
             failed = True
     if failed:
-      print('error: %s: Failed to delete obsolete checkout.' % (self.relpath,),
+      print('error: %s: Failed to delete obsolete checkout.' % (self.RelPath(local=False),),
             file=sys.stderr)
       print('       Remove manually, then run `repo sync -l`.', file=sys.stderr)
       return False
@@ -1624,13 +1691,10 @@ class Project(object):
 
     all_refs = self.bare_ref.all
     if R_HEADS + name in all_refs:
-      return GitCommand(self,
-                        ['checkout', name, '--'],
-                        capture_stdout=True,
-                        capture_stderr=True).Wait() == 0
+      return GitCommand(self, ['checkout', '-q', name, '--']).Wait() == 0
 
     branch = self.GetBranch(name)
-    branch.remote = self.GetRemote(self.remote.name)
+    branch.remote = self.GetRemote()
     branch.merge = branch_merge
     if not branch.merge.startswith('refs/') and not ID_RE.match(branch_merge):
       branch.merge = R_HEADS + branch_merge
@@ -1652,10 +1716,7 @@ class Project(object):
       branch.Save()
       return True
 
-    if GitCommand(self,
-                  ['checkout', '-b', branch.name, revid],
-                  capture_stdout=True,
-                  capture_stderr=True).Wait() == 0:
+    if GitCommand(self, ['checkout', '-q', '-b', branch.name, revid]).Wait() == 0:
       branch.Save()
       return True
     return False
@@ -1998,7 +2059,7 @@ class Project(object):
       self.bare_git.rev_list('-1', '--missing=allow-any',
                              '%s^0' % self.revisionExpr, '--')
       if self.upstream:
-        rev = self.GetRemote(self.remote.name).ToLocal(self.upstream)
+        rev = self.GetRemote().ToLocal(self.upstream)
         self.bare_git.rev_list('-1', '--missing=allow-any',
                                '%s^0' % rev, '--')
         self.bare_git.merge_base('--is-ancestor', self.revisionExpr, rev)
@@ -2010,7 +2071,7 @@ class Project(object):
   def _FetchArchive(self, tarpath, cwd=None):
     cmd = ['archive', '-v', '-o', tarpath]
     cmd.append('--remote=%s' % self.remote.url)
-    cmd.append('--prefix=%s/' % self.relpath)
+    cmd.append('--prefix=%s/' % self.RelPath(local=False))
     cmd.append(self.revisionExpr)
 
     command = GitCommand(self, cmd, cwd=cwd,
@@ -2156,6 +2217,8 @@ class Project(object):
     if prune:
       cmd.append('--prune')
 
+    # Always pass something for --recurse-submodules, git with GIT_DIR behaves
+    # incorrectly when not given `--recurse-submodules=no`. (b/218891912)
     cmd.append(f'--recurse-submodules={"on-demand" if submodules else "no"}')
 
     spec = []
@@ -2287,7 +2350,7 @@ class Project(object):
     if initial and (self.manifest.manifestProject.depth or self.clone_depth):
       return False
 
-    remote = self.GetRemote(self.remote.name)
+    remote = self.GetRemote()
     bundle_url = remote.url + '/clone.bundle'
     bundle_url = GitConfig.ForUser().UrlInsteadOf(bundle_url)
     if GetSchemeFromUrl(bundle_url) not in ('http', 'https',
@@ -2353,16 +2416,16 @@ class Project(object):
         srcUrl = 'http' + srcUrl[len('persistent-http'):]
       cmd += [srcUrl]
 
-      if IsTrace():
-        Trace('%s', ' '.join(cmd))
-      if verbose:
-        print('%s: Downloading bundle: %s' % (self.name, srcUrl))
-      stdout = None if verbose else subprocess.PIPE
-      stderr = None if verbose else subprocess.STDOUT
-      try:
-        proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr)
-      except OSError:
-        return False
+      proc = None
+      with Trace('Fetching bundle: %s', ' '.join(cmd)):
+        if verbose:
+          print('%s: Downloading bundle: %s' % (self.name, srcUrl))
+        stdout = None if verbose else subprocess.PIPE
+        stderr = None if verbose else subprocess.STDOUT
+        try:
+          proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr)
+        except OSError:
+          return False
 
       (output, _) = proc.communicate()
       curlret = proc.returncode
@@ -2592,7 +2655,7 @@ class Project(object):
         if not filecmp.cmp(stock_hook, dst, shallow=False):
           if not quiet:
             _warn("%s: Not replacing locally modified %s hook",
-                  self.relpath, name)
+                  self.RelPath(local=False), name)
         continue
       try:
         platform_utils.symlink(
@@ -2608,7 +2671,7 @@ class Project(object):
 
   def _InitRemote(self):
     if self.remote.url:
-      remote = self.GetRemote(self.remote.name)
+      remote = self.GetRemote()
       remote.url = self.remote.url
       remote.pushUrl = self.remote.pushUrl
       remote.review = self.remote.review
@@ -2621,6 +2684,7 @@ class Project(object):
       remote.Save()
 
   def _InitMRef(self):
+    """Initialize the pseudo m/<manifest branch> ref."""
     if self.manifest.branch:
       if self.use_git_worktrees:
         # Set up the m/ space to point to the worktree-specific ref space.
@@ -2650,6 +2714,16 @@ class Project(object):
     self._InitAnyMRef(HEAD, self.bare_git)
 
   def _InitAnyMRef(self, ref, active_git, detach=False):
+    """Initialize |ref| in |active_git| to the value in the manifest.
+
+    This points |ref| to the <project> setting in the manifest.
+
+    Args:
+      ref: The branch to update.
+      active_git: The git repository to make updates in.
+      detach: Whether to update target of symbolic refs, or overwrite the ref
+        directly (and thus make it non-symbolic).
+    """
     cur = self.bare_ref.symref(ref)
 
     if self.revisionId:
@@ -2658,7 +2732,7 @@ class Project(object):
         dst = self.revisionId + '^0'
         active_git.UpdateRef(ref, dst, message=msg, detach=True)
     else:
-      remote = self.GetRemote(self.remote.name)
+      remote = self.GetRemote()
       dst = remote.ToLocal(self.revisionExpr)
       if cur != dst:
         msg = 'manifest set to %s' % self.revisionExpr
@@ -2687,7 +2761,7 @@ class Project(object):
                          'work tree. If you\'re comfortable with the '
                          'possibility of losing the work tree\'s git metadata,'
                          ' use `repo sync --force-sync {0}` to '
-                         'proceed.'.format(self.relpath))
+                         'proceed.'.format(self.RelPath(local=False)))
 
   def _ReferenceGitDir(self, gitdir, dotgit, copy_all):
     """Update |dotgit| to reference |gitdir|, using symlinks where possible.
@@ -3167,7 +3241,7 @@ class _InfoMessage(object):
     self.text = text
 
   def Print(self, syncbuf):
-    syncbuf.out.info('%s/: %s', self.project.relpath, self.text)
+    syncbuf.out.info('%s/: %s', self.project.RelPath(local=False), self.text)
     syncbuf.out.nl()
 
 
@@ -3179,7 +3253,7 @@ class _Failure(object):
 
   def Print(self, syncbuf):
     syncbuf.out.fail('error: %s/: %s',
-                     self.project.relpath,
+                     self.project.RelPath(local=False),
                      str(self.why))
     syncbuf.out.nl()
 
@@ -3192,7 +3266,7 @@ class _Later(object):
 
   def Run(self, syncbuf):
     out = syncbuf.out
-    out.project('project %s/', self.project.relpath)
+    out.project('project %s/', self.project.RelPath(local=False))
     out.nl()
     try:
       self.action()
@@ -3431,7 +3505,7 @@ class ManifestProject(MetaProject):
   @property
   def partial_clone_exclude(self):
     """Partial clone exclude string"""
-    return self.config.GetBoolean('repo.partialcloneexclude')
+    return self.config.GetString('repo.partialcloneexclude')
 
   @property
   def manifest_platform(self):
@@ -3443,6 +3517,67 @@ class ManifestProject(MetaProject):
     """Return the name of the platform."""
     return platform.system().lower()
 
+  def SyncWithPossibleInit(self, submanifest, verbose=False,
+                           current_branch_only=False, tags='', git_event_log=None):
+    """Sync a manifestProject, possibly for the first time.
+
+    Call Sync() with arguments from the most recent `repo init`.  If this is a
+    new sub manifest, then inherit options from the parent's manifestProject.
+
+    This is used by subcmds.Sync() to do an initial download of new sub
+    manifests.
+
+    Args:
+      submanifest: an XmlSubmanifest, the submanifest to re-sync.
+      verbose: a boolean, whether to show all output, rather than only errors.
+      current_branch_only: a boolean, whether to only fetch the current manifest
+          branch from the server.
+      tags: a boolean, whether to fetch tags.
+      git_event_log: an EventLog, for git tracing.
+    """
+    # TODO(lamontjones): when refactoring sync (and init?) consider how to
+    # better get the init options that we should use for new submanifests that
+    # are added when syncing an existing workspace.
+    git_event_log = git_event_log or EventLog()
+    spec = submanifest.ToSubmanifestSpec()
+    # Use the init options from the existing manifestProject, or the parent if
+    # it doesn't exist.
+    #
+    # Today, we only support changing manifest_groups on the sub-manifest, with
+    # no supported-for-the-user way to change the other arguments from those
+    # specified by the outermost manifest.
+    #
+    # TODO(lamontjones): determine which of these should come from the outermost
+    # manifest and which should come from the parent manifest.
+    mp = self if self.Exists else submanifest.parent.manifestProject
+    return self.Sync(
+        manifest_url=spec.manifestUrl,
+        manifest_branch=spec.revision,
+        standalone_manifest=mp.standalone_manifest_url,
+        groups=mp.manifest_groups,
+        platform=mp.manifest_platform,
+        mirror=mp.mirror,
+        dissociate=mp.dissociate,
+        reference=mp.reference,
+        worktree=mp.use_worktree,
+        submodules=mp.submodules,
+        archive=mp.archive,
+        partial_clone=mp.partial_clone,
+        clone_filter=mp.clone_filter,
+        partial_clone_exclude=mp.partial_clone_exclude,
+        clone_bundle=mp.clone_bundle,
+        git_lfs=mp.git_lfs,
+        use_superproject=mp.use_superproject,
+        verbose=verbose,
+        current_branch_only=current_branch_only,
+        tags=tags,
+        depth=mp.depth,
+        git_event_log=git_event_log,
+        manifest_name=spec.manifestName,
+        this_manifest_only=True,
+        outer_manifest=False,
+    )
+
   def Sync(self, _kwargs_only=(), manifest_url='', manifest_branch=None,
            standalone_manifest=False, groups='', mirror=False, reference='',
            dissociate=False, worktree=False, submodules=False, archive=False,
@@ -3495,7 +3630,7 @@ class ManifestProject(MetaProject):
     """
     assert _kwargs_only == (), 'Sync only accepts keyword arguments.'
 
-    groups = groups or 'default'
+    groups = groups or self.manifest.GetDefaultGroupsStr(with_platform=False)
     platform = platform or 'auto'
     git_event_log = git_event_log or EventLog()
     if outer_manifest and self.manifest.is_submanifest:
@@ -3589,7 +3724,7 @@ class ManifestProject(MetaProject):
 
     # Set the remote URL before the remote branch as we might need it below.
     if manifest_url:
-      r = self.GetRemote(self.remote.name)
+      r = self.GetRemote()
       r.url = manifest_url
       r.ResetFetch()
       r.Save()
@@ -3709,46 +3844,45 @@ class ManifestProject(MetaProject):
     if use_superproject is not None:
       self.config.SetBoolean('repo.superproject', use_superproject)
 
-    if standalone_manifest:
-      if is_new:
-        manifest_name = 'default.xml'
-        manifest_data = fetch.fetch_file(manifest_url, verbose=verbose)
-        dest = os.path.join(self.worktree, manifest_name)
-        os.makedirs(os.path.dirname(dest), exist_ok=True)
-        with open(dest, 'wb') as f:
-          f.write(manifest_data)
-      return
+    if not standalone_manifest:
+      if not self.Sync_NetworkHalf(
+          is_new=is_new, quiet=not verbose, verbose=verbose,
+          clone_bundle=clone_bundle, current_branch_only=current_branch_only,
+          tags=tags, submodules=submodules, clone_filter=clone_filter,
+          partial_clone_exclude=self.manifest.PartialCloneExclude).success:
+        r = self.GetRemote()
+        print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
 
-    if not self.Sync_NetworkHalf(is_new=is_new, quiet=not verbose, verbose=verbose,
-                              clone_bundle=clone_bundle,
-                              current_branch_only=current_branch_only,
-                              tags=tags, submodules=submodules,
-                              clone_filter=clone_filter,
-                              partial_clone_exclude=self.manifest.PartialCloneExclude):
-      r = self.GetRemote(self.remote.name)
-      print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
-
-      # Better delete the manifest git dir if we created it; otherwise next
-      # time (when user fixes problems) we won't go through the "is_new" logic.
-      if is_new:
-        platform_utils.rmtree(self.gitdir)
-      return False
-
-    if manifest_branch:
-      self.MetaBranchSwitch(submodules=submodules)
-
-    syncbuf = SyncBuffer(self.config)
-    self.Sync_LocalHalf(syncbuf, submodules=submodules)
-    syncbuf.Finish()
-
-    if is_new or self.CurrentBranch is None:
-      if not self.StartBranch('default'):
-        print('fatal: cannot create default in manifest', file=sys.stderr)
+        # Better delete the manifest git dir if we created it; otherwise next
+        # time (when user fixes problems) we won't go through the "is_new" logic.
+        if is_new:
+          platform_utils.rmtree(self.gitdir)
         return False
 
-    if not manifest_name:
-      print('fatal: manifest name (-m) is required.', file=sys.stderr)
-      return False
+      if manifest_branch:
+        self.MetaBranchSwitch(submodules=submodules)
+
+      syncbuf = SyncBuffer(self.config)
+      self.Sync_LocalHalf(syncbuf, submodules=submodules)
+      syncbuf.Finish()
+
+      if is_new or self.CurrentBranch is None:
+        if not self.StartBranch('default'):
+          print('fatal: cannot create default in manifest', file=sys.stderr)
+          return False
+
+      if not manifest_name:
+        print('fatal: manifest name (-m) is required.', file=sys.stderr)
+        return False
+
+    elif is_new:
+      # This is a new standalone manifest.
+      manifest_name = 'default.xml'
+      manifest_data = fetch.fetch_file(manifest_url, verbose=verbose)
+      dest = os.path.join(self.worktree, manifest_name)
+      os.makedirs(os.path.dirname(dest), exist_ok=True)
+      with open(dest, 'wb') as f:
+        f.write(manifest_data)
 
     try:
       self.manifest.Link(manifest_name)
@@ -3789,10 +3923,10 @@ class ManifestProject(MetaProject):
             outer_manifest=False,
         )
 
-    # Lastly, clone the superproject(s).
-    if self.manifest.manifestProject.use_superproject:
-      sync_result = Superproject(
-          self.manifest, self.manifest.repodir, git_event_log, quiet=not verbose).Sync()
+    # Lastly, if the manifest has a <superproject> then have the superproject
+    # sync it (if it will be used).
+    if git_superproject.UseSuperproject(use_superproject, self.manifest):
+      sync_result = self.manifest.superproject.Sync(git_event_log)
       if not sync_result.success:
         print('warning: git update of superproject for '
               f'{self.manifest.path_prefix} failed, repo sync will not use '

release/update-manpages

@@ -59,27 +59,30 @@ def main(argv):
   version = RepoSourceVersion()
   cmdlist = [['help2man', '-N', '-n', f'repo {cmd} - manual page for repo {cmd}',
     '-S', f'repo {cmd}', '-m', 'Repo Manual', f'--version-string={version}',
-    '-o', MANDIR.joinpath(f'repo-{cmd}.1.tmp'), TOPDIR.joinpath('repo'),
+    '-o', MANDIR.joinpath(f'repo-{cmd}.1.tmp'), './repo',
     '-h', f'help {cmd}'] for cmd in subcmds.all_commands]
   cmdlist.append(['help2man', '-N', '-n', 'repository management tool built on top of git',
     '-S', 'repo', '-m', 'Repo Manual', f'--version-string={version}',
-    '-o', MANDIR.joinpath('repo.1.tmp'), TOPDIR.joinpath('repo'),
+    '-o', MANDIR.joinpath('repo.1.tmp'), './repo',
     '-h', '--help-all'])
 
   with tempfile.TemporaryDirectory() as tempdir:
-    repo_dir = Path(tempdir) / '.repo'
+    tempdir = Path(tempdir)
+    repo_dir = tempdir / '.repo'
     repo_dir.mkdir()
     (repo_dir / 'repo').symlink_to(TOPDIR)
 
+    # Create a repo wrapper using the active Python executable.  We can't pass
+    # this directly to help2man as it's too simple, so insert it via shebang.
+    data = (TOPDIR / 'repo').read_text(encoding='utf-8')
+    tempbin = tempdir / 'repo'
+    tempbin.write_text(f'#!{sys.executable}\n' + data, encoding='utf-8')
+    tempbin.chmod(0o755)
+
     # Run all cmd in parallel, and wait for them to finish.
     with multiprocessing.Pool() as pool:
       pool.map(partial(worker, cwd=tempdir, check=True), cmdlist)
 
-  regex = (
-      (r'(It was generated by help2man) [0-9.]+', '\g<1>.'),
-      (r'^\.IP\n(.*:)\n', '.SS \g<1>\n'),
-      (r'^\.PP\nDescription', '.SH DETAILS'),
-  )
   for tmp_path in MANDIR.glob('*.1.tmp'):
     path = tmp_path.parent / tmp_path.stem
     old_data = path.read_text() if path.exists() else ''
@@ -87,8 +90,7 @@ def main(argv):
     data = tmp_path.read_text()
     tmp_path.unlink()
 
-    for pattern, replacement in regex:
-      data = re.sub(pattern, replacement, data, flags=re.M)
+    data = replace_regex(data)
 
     # If the only thing that changed was the date, don't refresh.  This avoids
     # a lot of noise when only one file actually updates.
@@ -98,5 +100,25 @@ def main(argv):
       path.write_text(data)
 
 
+def replace_regex(data):
+  """Replace semantically null regexes in the data.
+
+  Args:
+    data: manpage text.
+
+  Returns:
+    Updated manpage text.
+  """
+  regex = (
+      (r'(It was generated by help2man) [0-9.]+', '\g<1>.'),
+      (r'^\033\[[0-9;]*m([^\033]*)\033\[m', '\g<1>'),
+      (r'^\.IP\n(.*:)\n', '.SS \g<1>\n'),
+      (r'^\.PP\nDescription', '.SH DETAILS'),
+  )
+  for pattern, replacement in regex:
+    data = re.sub(pattern, replacement, data, flags=re.M)
+  return data
+
+
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))

repo

@@ -149,7 +149,7 @@ if not REPO_REV:
 BUG_URL = 'https://bugs.chromium.org/p/gerrit/issues/entry?template=Repo+tool+issue'
 
 # increment this whenever we make important changes to this script
-VERSION = (2, 21)
+VERSION = (2, 30)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (2, 3)
@@ -316,6 +316,10 @@ def InitParser(parser, gitc_init=False):
                    help='download the manifest as a static file '
                         'rather then create a git checkout of '
                         'the manifest repo')
+  group.add_option('--manifest-depth', type='int', default=0, metavar='DEPTH',
+                   help='create a shallow clone of the manifest repo with '
+                        'given depth (0 for full clone); see git clone '
+                        '(default: %default)')
 
   # Options that only affect manifest project, and not any of the projects
   # specified in the manifest itself.
@@ -325,9 +329,9 @@ def InitParser(parser, gitc_init=False):
   # want -c, so try to satisfy both as best we can.
   if not gitc_init:
     cbr_opts += ['-c']
-  group.add_option(*cbr_opts,
+  group.add_option(*cbr_opts, default=True,
                    dest='current_branch_only', action='store_true',
-                   help='fetch only current manifest branch from server')
+                   help='fetch only current manifest branch from server (default)')
   group.add_option('--no-current-branch',
                    dest='current_branch_only', action='store_false',
                    help='fetch all manifest branches from server')
@@ -612,15 +616,20 @@ def _Init(args, gitc_init=False):
   try:
     if not opt.quiet:
       print('Downloading Repo source from', url)
-    dst = os.path.abspath(os.path.join(repodir, S_repo))
+    dst_final = os.path.abspath(os.path.join(repodir, S_repo))
+    dst = dst_final + '.tmp'
+    shutil.rmtree(dst, ignore_errors=True)
     _Clone(url, dst, opt.clone_bundle, opt.quiet, opt.verbose)
 
     remote_ref, rev = check_repo_rev(dst, rev, opt.repo_verify, quiet=opt.quiet)
     _Checkout(dst, remote_ref, rev, opt.quiet)
 
     if not os.path.isfile(os.path.join(dst, 'repo')):
-      print("warning: '%s' does not look like a git-repo repository, is "
-            "REPO_URL set correctly?" % url, file=sys.stderr)
+      print("fatal: '%s' does not look like a git-repo repository, is "
+            "--repo-url set correctly?" % url, file=sys.stderr)
+      raise CloneFailure()
+
+    os.rename(dst, dst_final)
 
   except CloneFailure:
     print('fatal: double check your --repo-rev setting.', file=sys.stderr)
@@ -1317,6 +1326,7 @@ def main(orig_args):
         print("fatal: cloning the git-repo repository failed, will remove "
               "'%s' " % path, file=sys.stderr)
         shutil.rmtree(path, ignore_errors=True)
+        shutil.rmtree(path + '.tmp', ignore_errors=True)
         sys.exit(1)
       repo_main, rel_repo_dir = _FindRepo()
     else:

repo_trace.py

@@ -15,26 +15,133 @@
 """Logic for tracing repo interactions.
 
 Activated via `repo --trace ...` or `REPO_TRACE=1 repo ...`.
+
+Temporary: Tracing is always on. Set `REPO_TRACE=0` to turn off.
+To also include trace outputs in stderr do `repo --trace_to_stderr ...`
 """
 
 import sys
 import os
+import time
+from contextlib import ContextDecorator
+
+import platform_utils
 
 # Env var to implicitly turn on tracing.
 REPO_TRACE = 'REPO_TRACE'
 
-_TRACE = os.environ.get(REPO_TRACE) == '1'
+# Temporarily set tracing to always on unless user expicitly sets to 0.
+_TRACE = os.environ.get(REPO_TRACE) != '0'
+_TRACE_TO_STDERR = False
+_TRACE_FILE = None
+_TRACE_FILE_NAME = 'TRACE_FILE'
+_MAX_SIZE = 70  # in mb
+_NEW_COMMAND_SEP = '+++++++++++++++NEW COMMAND+++++++++++++++++++'
+
+
+def IsTraceToStderr():
+  return _TRACE_TO_STDERR
 
 
 def IsTrace():
   return _TRACE
 
 
+def SetTraceToStderr():
+  global _TRACE_TO_STDERR
+  _TRACE_TO_STDERR = True
+
+
 def SetTrace():
   global _TRACE
   _TRACE = True
 
 
-def Trace(fmt, *args):
-  if IsTrace():
-    print(fmt % args, file=sys.stderr)
+def _SetTraceFile(quiet):
+  global _TRACE_FILE
+  _TRACE_FILE = _GetTraceFile(quiet)
+
+
+class Trace(ContextDecorator):
+
+  def _time(self):
+    """Generate nanoseconds of time in a py3.6 safe way"""
+    return int(time.time() * 1e+9)
+
+  def __init__(self, fmt, *args, first_trace=False, quiet=True):
+    """Initialize the object.
+
+    Args:
+      fmt: The format string for the trace.
+      *args: Arguments to pass to formatting.
+      first_trace: Whether this is the first trace of a `repo` invocation.
+      quiet: Whether to suppress notification of trace file location.
+    """
+    if not IsTrace():
+      return
+    self._trace_msg = fmt % args
+
+    if not _TRACE_FILE:
+      _SetTraceFile(quiet)
+
+    if first_trace:
+      _ClearOldTraces()
+      self._trace_msg = f'{_NEW_COMMAND_SEP} {self._trace_msg}'
+
+  def __enter__(self):
+    if not IsTrace():
+      return self
+
+    print_msg = f'PID: {os.getpid()} START: {self._time()} :{self._trace_msg}\n'
+
+    with open(_TRACE_FILE, 'a') as f:
+      print(print_msg, file=f)
+
+    if _TRACE_TO_STDERR:
+      print(print_msg, file=sys.stderr)
+
+    return self
+
+  def __exit__(self, *exc):
+    if not IsTrace():
+      return False
+
+    print_msg = f'PID: {os.getpid()} END: {self._time()} :{self._trace_msg}\n'
+
+    with open(_TRACE_FILE, 'a') as f:
+      print(print_msg, file=f)
+
+    if _TRACE_TO_STDERR:
+      print(print_msg, file=sys.stderr)
+
+    return False
+
+
+def _GetTraceFile(quiet):
+  """Get the trace file or create one."""
+  # TODO: refactor to pass repodir to Trace.
+  repo_dir = os.path.dirname(os.path.dirname(__file__))
+  trace_file = os.path.join(repo_dir, _TRACE_FILE_NAME)
+  if not quiet:
+    print(f'Trace outputs in {trace_file}', file=sys.stderr)
+  return trace_file
+
+
+def _ClearOldTraces():
+  """Clear the oldest commands if trace file is too big.
+
+  Note: If the trace file contains output from two `repo`
+        commands that were running at the same time, this
+        will not work precisely.
+  """
+  if os.path.isfile(_TRACE_FILE):
+    while os.path.getsize(_TRACE_FILE) / (1024 * 1024) > _MAX_SIZE:
+      temp_file = _TRACE_FILE + '.tmp'
+      with open(_TRACE_FILE, 'r', errors='ignore') as fin:
+        with open(temp_file, 'w') as tf:
+          trace_lines = fin.readlines()
+          for i, l in enumerate(trace_lines):
+            if 'END:' in l and _NEW_COMMAND_SEP in l:
+              tf.writelines(trace_lines[i + 1:])
+              break
+      platform_utils.rename(temp_file, _TRACE_FILE)

run_tests

@@ -20,6 +20,7 @@ import os
 import shutil
 import subprocess
 import sys
+import repo_trace
 
 
 def find_pytest():

ssh.py

@@ -182,28 +182,29 @@ class ProxyManager:
     # be important because we can't tell that that 'git@myhost.com' is the same
     # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file.
     check_command = command_base + ['-O', 'check']
-    try:
-      Trace(': %s', ' '.join(check_command))
-      check_process = subprocess.Popen(check_command,
-                                       stdout=subprocess.PIPE,
-                                       stderr=subprocess.PIPE)
-      check_process.communicate()  # read output, but ignore it...
-      isnt_running = check_process.wait()
+    with Trace('Call to ssh (check call): %s', ' '.join(check_command)):
+      try:
+        check_process = subprocess.Popen(check_command,
+                                        stdout=subprocess.PIPE,
+                                        stderr=subprocess.PIPE)
+        check_process.communicate()  # read output, but ignore it...
+        isnt_running = check_process.wait()
 
-      if not isnt_running:
-        # Our double-check found that the master _was_ infact running.  Add to
-        # the list of keys.
-        self._master_keys[key] = True
-        return True
-    except Exception:
-      # Ignore excpetions.  We we will fall back to the normal command and print
-      # to the log there.
-      pass
+        if not isnt_running:
+          # Our double-check found that the master _was_ infact running.  Add to
+          # the list of keys.
+          self._master_keys[key] = True
+          return True
+      except Exception:
+        # Ignore excpetions.  We we will fall back to the normal command and
+        # print to the log there.
+        pass
 
     command = command_base[:1] + ['-M', '-N'] + command_base[1:]
+    p = None
     try:
-      Trace(': %s', ' '.join(command))
-      p = subprocess.Popen(command)
+      with Trace('Call to ssh: %s', ' '.join(command)):
+        p = subprocess.Popen(command)
     except Exception as e:
       self._master_broken.value = True
       print('\nwarn: cannot enable ssh control master for %s:%s\n%s'

subcmds/branches.py

@@ -155,11 +155,11 @@ is shown, then the branch appears in all projects.
         if i.IsSplitCurrent or (in_cnt <= project_cnt - in_cnt):
           in_type = 'in'
           for b in i.projects:
-            relpath = b.project.relpath
+            relpath = _RelPath(b.project)
             if not i.IsSplitCurrent or b.current:
-              paths.append(_RelPath(b.project))
+              paths.append(relpath)
             else:
-              non_cur_paths.append(_RelPath(b.project))
+              non_cur_paths.append(relpath)
         else:
           fmt = out.notinproject
           in_type = 'not in'

subcmds/cherry_pick.py

@@ -60,8 +60,10 @@ change id will be added.
                    capture_stderr=True)
     status = p.Wait()
 
-    print(p.stdout, file=sys.stdout)
-    print(p.stderr, file=sys.stderr)
+    if p.stdout:
+      print(p.stdout.strip(), file=sys.stdout)
+    if p.stderr:
+      print(p.stderr.strip(), file=sys.stderr)
 
     if status == 0:
       # The cherry-pick was applied correctly. We just need to edit the

subcmds/diff.py

@@ -35,18 +35,21 @@ to the Unix 'patch' command.
                  dest='absolute', action='store_true',
                  help='paths are relative to the repository root')
 
-  def _ExecuteOne(self, absolute, project):
+  def _ExecuteOne(self, absolute, local, project):
     """Obtains the diff for a specific project.
 
     Args:
       absolute: Paths are relative to the root.
+      local: a boolean, if True, the path is relative to the local
+             (sub)manifest.  If false, the path is relative to the
+             outermost manifest.
       project: Project to get status of.
 
     Returns:
       The status of the project.
     """
     buf = io.StringIO()
-    ret = project.PrintWorkTreeDiff(absolute, output_redir=buf)
+    ret = project.PrintWorkTreeDiff(absolute, output_redir=buf, local=local)
     return (ret, buf.getvalue())
 
   def Execute(self, opt, args):
@@ -63,7 +66,7 @@ to the Unix 'patch' command.
 
     return self.ExecuteInParallel(
         opt.jobs,
-        functools.partial(self._ExecuteOne, opt.absolute),
+        functools.partial(self._ExecuteOne, opt.absolute, opt.this_manifest_only),
         all_projects,
         callback=_ProcessResults,
         ordered=True)

subcmds/diffmanifests.py

@@ -77,33 +77,35 @@ synced and their revisions won't be found.
                  metavar='<FORMAT>',
                  help='print the log using a custom git pretty format string')
 
-  def _printRawDiff(self, diff, pretty_format=None):
+  def _printRawDiff(self, diff, pretty_format=None, local=False):
+    _RelPath = lambda p: p.RelPath(local=local)
     for project in diff['added']:
-      self.printText("A %s %s" % (project.relpath, project.revisionExpr))
+      self.printText("A %s %s" % (_RelPath(project), project.revisionExpr))
       self.out.nl()
 
     for project in diff['removed']:
-      self.printText("R %s %s" % (project.relpath, project.revisionExpr))
+      self.printText("R %s %s" % (_RelPath(project), project.revisionExpr))
       self.out.nl()
 
     for project, otherProject in diff['changed']:
-      self.printText("C %s %s %s" % (project.relpath, project.revisionExpr,
+      self.printText("C %s %s %s" % (_RelPath(project), project.revisionExpr,
                                      otherProject.revisionExpr))
       self.out.nl()
       self._printLogs(project, otherProject, raw=True, color=False, pretty_format=pretty_format)
 
     for project, otherProject in diff['unreachable']:
-      self.printText("U %s %s %s" % (project.relpath, project.revisionExpr,
+      self.printText("U %s %s %s" % (_RelPath(project), project.revisionExpr,
                                      otherProject.revisionExpr))
       self.out.nl()
 
-  def _printDiff(self, diff, color=True, pretty_format=None):
+  def _printDiff(self, diff, color=True, pretty_format=None, local=False):
+    _RelPath = lambda p: p.RelPath(local=local)
     if diff['added']:
       self.out.nl()
       self.printText('added projects : \n')
       self.out.nl()
       for project in diff['added']:
-        self.printProject('\t%s' % (project.relpath))
+        self.printProject('\t%s' % (_RelPath(project)))
         self.printText(' at revision ')
         self.printRevision(project.revisionExpr)
         self.out.nl()
@@ -113,7 +115,17 @@ synced and their revisions won't be found.
       self.printText('removed projects : \n')
       self.out.nl()
       for project in diff['removed']:
-        self.printProject('\t%s' % (project.relpath))
+        self.printProject('\t%s' % (_RelPath(project)))
+        self.printText(' at revision ')
+        self.printRevision(project.revisionExpr)
+        self.out.nl()
+
+    if diff['missing']:
+      self.out.nl()
+      self.printText('missing projects : \n')
+      self.out.nl()
+      for project in diff['missing']:
+        self.printProject('\t%s' % (_RelPath(project)))
         self.printText(' at revision ')
         self.printRevision(project.revisionExpr)
         self.out.nl()
@@ -123,7 +135,7 @@ synced and their revisions won't be found.
       self.printText('changed projects : \n')
       self.out.nl()
       for project, otherProject in diff['changed']:
-        self.printProject('\t%s' % (project.relpath))
+        self.printProject('\t%s' % (_RelPath(project)))
         self.printText(' changed from ')
         self.printRevision(project.revisionExpr)
         self.printText(' to ')
@@ -138,7 +150,7 @@ synced and their revisions won't be found.
       self.printText('projects with unreachable revisions : \n')
       self.out.nl()
       for project, otherProject in diff['unreachable']:
-        self.printProject('\t%s ' % (project.relpath))
+        self.printProject('\t%s ' % (_RelPath(project)))
         self.printRevision(project.revisionExpr)
         self.printText(' or ')
         self.printRevision(otherProject.revisionExpr)
@@ -204,6 +216,8 @@ synced and their revisions won't be found.
 
     diff = manifest1.projectsDiff(manifest2)
     if opt.raw:
-      self._printRawDiff(diff, pretty_format=opt.pretty_format)
+      self._printRawDiff(diff, pretty_format=opt.pretty_format,
+                         local=opt.this_manifest_only)
     else:
-      self._printDiff(diff, color=opt.color, pretty_format=opt.pretty_format)
+      self._printDiff(diff, color=opt.color, pretty_format=opt.pretty_format,
+                      local=opt.this_manifest_only)

subcmds/gitc_init.py

@@ -68,7 +68,8 @@ use for this GITC client.
         sys.exit(1)
       manifest_file = opt.manifest_file
 
-    manifest = GitcManifest(self.repodir, gitc_client)
+    manifest = GitcManifest(self.repodir, os.path.join(self.client_dir,
+                                                       '.manifest'))
     manifest.Override(manifest_file)
     gitc_utils.generate_gitc_manifest(None, manifest)
     print('Please run `cd %s` to view your GITC client.' %

subcmds/info.py

@@ -65,8 +65,7 @@ class Info(PagedCommand):
       self.manifest = self.manifest.outer_client
     manifestConfig = self.manifest.manifestProject.config
     mergeBranch = manifestConfig.GetBranch("default").merge
-    manifestGroups = (manifestConfig.GetString('manifest.groups')
-                      or 'all,-notdefault')
+    manifestGroups = self.manifest.GetGroupsStr()
 
     self.heading("Manifest branch: ")
     if self.manifest.default.revisionExpr:

subcmds/init.py

@@ -24,8 +24,6 @@ from error import ManifestParseError
 from project import SyncBuffer
 from git_config import GitConfig
 from git_command import git_require, MIN_GIT_VERSION_SOFT, MIN_GIT_VERSION_HARD
-import fetch
-import platform_utils
 from wrapper import Wrapper
 
 
@@ -91,11 +89,10 @@ to update the working directory files.
   def _Options(self, p, gitc_init=False):
     Wrapper().InitParser(p, gitc_init=gitc_init)
     m = p.add_option_group('Multi-manifest')
-    m.add_option('--outer-manifest', action='store_true',
+    m.add_option('--outer-manifest', action='store_true', default=True,
                  help='operate starting at the outermost manifest')
     m.add_option('--no-outer-manifest', dest='outer_manifest',
-                 action='store_false', default=None,
-                 help='do not operate on outer manifests')
+                 action='store_false', help='do not operate on outer manifests')
     m.add_option('--this-manifest-only', action='store_true', default=None,
                  help='only operate on this (sub)manifest')
     m.add_option('--no-this-manifest-only', '--all-manifests',
@@ -112,6 +109,10 @@ to update the working directory files.
     Args:
       opt: options from optparse.
     """
+    # Normally this value is set when instantiating the project, but the
+    # manifest project is special and is created when instantiating the
+    # manifest which happens before we parse options.
+    self.manifest.manifestProject.clone_depth = opt.manifest_depth
     if not self.manifest.manifestProject.Sync(
         manifest_url=opt.manifest_url,
         manifest_branch=opt.manifest_branch,
@@ -147,7 +148,7 @@ to update the working directory files.
       return value
     return a
 
-  def _ShouldConfigureUser(self, opt):
+  def _ShouldConfigureUser(self, opt, existing_checkout):
     gc = self.client.globalConfig
     mp = self.manifest.manifestProject
 
@@ -159,7 +160,7 @@ to update the working directory files.
       mp.config.SetString('user.name', gc.GetString('user.name'))
       mp.config.SetString('user.email', gc.GetString('user.email'))
 
-    if not opt.quiet:
+    if not opt.quiet and not existing_checkout or opt.verbose:
       print()
       print('Your identity is: %s <%s>' % (mp.config.GetString('user.name'),
                                            mp.config.GetString('user.email')))
@@ -244,7 +245,7 @@ to update the working directory files.
     if current_dir != self.manifest.topdir:
       print('If this is not the directory in which you want to initialize '
             'repo, please run:')
-      print('   rm -r %s/.repo' % self.manifest.topdir)
+      print('   rm -r %s' % os.path.join(self.manifest.topdir, '.repo'))
       print('and try again.')
 
   def ValidateOptions(self, opt, args):
@@ -314,10 +315,17 @@ to update the working directory files.
       # Older versions of git supported worktree, but had dangerous gc bugs.
       git_require((2, 15, 0), fail=True, msg='git gc worktree corruption')
 
+    # Provide a short notice that we're reinitializing an existing checkout.
+    # Sometimes developers might not realize that they're in one, or that
+    # repo doesn't do nested checkouts.
+    existing_checkout = self.manifest.manifestProject.Exists
+    if not opt.quiet and existing_checkout:
+      print('repo: reusing existing repo client checkout in', self.manifest.topdir)
+
     self._SyncManifest(opt)
 
     if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
-      if opt.config_name or self._ShouldConfigureUser(opt):
+      if opt.config_name or self._ShouldConfigureUser(opt, existing_checkout):
         self._ConfigureUser(opt)
       self._ConfigureColor()
 

subcmds/selfupdate.py

@@ -51,7 +51,7 @@ need to be performed by an end-user.
       _PostRepoUpgrade(self.manifest)
 
     else:
-      if not rp.Sync_NetworkHalf():
+      if not rp.Sync_NetworkHalf().success:
         print("error: can't update repo", file=sys.stderr)
         sys.exit(1)
 

subcmds/stage.py

@@ -75,6 +75,7 @@ The '%prog' command stages files to prepare the next commit.
       out.nl()
 
       out.prompt('project> ')
+      out.flush()
       try:
         a = sys.stdin.readline()
       except KeyboardInterrupt:

subcmds/status.py

@@ -83,7 +83,7 @@ the following meanings:
                  dest='orphans', action='store_true',
                  help="include objects in working directory outside of repo projects")
 
-  def _StatusHelper(self, quiet, project):
+  def _StatusHelper(self, quiet, local, project):
     """Obtains the status for a specific project.
 
     Obtains the status for a project, redirecting the output to
@@ -91,13 +91,17 @@ the following meanings:
 
     Args:
       quiet: Where to output the status.
+      local: a boolean, if True, the path is relative to the local
+             (sub)manifest.  If false, the path is relative to the
+             outermost manifest.
       project: Project to get status of.
 
     Returns:
       The status of the project.
     """
     buf = io.StringIO()
-    ret = project.PrintWorkTreeStatus(quiet=quiet, output_redir=buf)
+    ret = project.PrintWorkTreeStatus(quiet=quiet, output_redir=buf,
+                                      local=local)
     return (ret, buf.getvalue())
 
   def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring):
@@ -130,7 +134,7 @@ the following meanings:
 
     counter = self.ExecuteInParallel(
         opt.jobs,
-        functools.partial(self._StatusHelper, opt.quiet),
+        functools.partial(self._StatusHelper, opt.quiet, opt.this_manifest_only),
         all_projects,
         callback=_ProcessResults,
         ordered=True)

subcmds/sync.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
 import functools
 import http.cookiejar as cookielib
 import io
@@ -24,9 +25,11 @@ import socket
 import sys
 import tempfile
 import time
+from typing import NamedTuple, List, Set
 import urllib.error
 import urllib.parse
 import urllib.request
+import xml.parsers.expat
 import xmlrpc.client
 
 try:
@@ -51,22 +54,79 @@ import git_superproject
 import gitc_utils
 from project import Project
 from project import RemoteSpec
-from command import Command, MirrorSafeCommand, WORKER_BATCH_SIZE
+from command import Command, DEFAULT_LOCAL_JOBS, MirrorSafeCommand, WORKER_BATCH_SIZE
 from error import RepoChangedException, GitError, ManifestParseError
 import platform_utils
 from project import SyncBuffer
 from progress import Progress
+from repo_trace import Trace
 import ssh
 from wrapper import Wrapper
 from manifest_xml import GitcManifest
 
 _ONE_DAY_S = 24 * 60 * 60
 
+# Env var to implicitly turn auto-gc back on.  This was added to allow a user to
+# revert a change in default behavior in v2.29.9.  Remove after 2023-04-01.
+_REPO_AUTO_GC = 'REPO_AUTO_GC'
+_AUTO_GC = os.environ.get(_REPO_AUTO_GC) == '1'
+
+
+class _FetchOneResult(NamedTuple):
+  """_FetchOne return value.
+
+  Attributes:
+    success (bool): True if successful.
+    project (Project): The fetched project.
+    start (float): The starting time.time().
+    finish (float): The ending time.time().
+    remote_fetched (bool): True if the remote was actually queried.
+  """
+  success: bool
+  project: Project
+  start: float
+  finish: float
+  remote_fetched: bool
+
+
+class _FetchResult(NamedTuple):
+  """_Fetch return value.
+
+  Attributes:
+    success (bool): True if successful.
+    projects (Set[str]): The names of the git directories of fetched projects.
+  """
+  success: bool
+  projects: Set[str]
+
+
+class _FetchMainResult(NamedTuple):
+  """_FetchMain return value.
+
+  Attributes:
+    all_projects (List[Project]): The fetched projects.
+  """
+  all_projects: List[Project]
+
+
+class _CheckoutOneResult(NamedTuple):
+  """_CheckoutOne return value.
+
+  Attributes:
+    success (bool): True if successful.
+    project (Project): The project.
+    start (float): The starting time.time().
+    finish (float): The ending time.time().
+  """
+  success: bool
+  project: Project
+  start: float
+  finish: float
+
 
 class Sync(Command, MirrorSafeCommand):
-  jobs = 1
   COMMON = True
-  MULTI_MANIFEST_SUPPORT = False
+  MULTI_MANIFEST_SUPPORT = True
   helpSummary = "Update working tree to the latest revision"
   helpUsage = """
 %prog [<project>...]
@@ -140,6 +200,9 @@ exist locally.
 The --prune option can be used to remove any refs that no longer
 exist on the remote.
 
+The --auto-gc option can be used to trigger garbage collection on all
+projects. By default, repo does not run garbage collection.
+
 # SSH Connections
 
 If at least one project remote URL uses an SSH connection (ssh://,
@@ -167,21 +230,16 @@ If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
 later is required to fix a server side protocol bug.
 
 """
-  PARALLEL_JOBS = 1
-
-  def _CommonOptions(self, p):
-    if self.outer_client and self.outer_client.manifest:
-      try:
-        self.PARALLEL_JOBS = self.outer_client.manifest.default.sync_j
-      except ManifestParseError:
-        pass
-    super()._CommonOptions(p)
+  # A value of 0 means we want parallel jobs, but we'll determine the default
+  # value later on.
+  PARALLEL_JOBS = 0
 
   def _Options(self, p, show_smart=True):
     p.add_option('--jobs-network', default=None, type=int, metavar='JOBS',
-                 help='number of network jobs to run in parallel (defaults to --jobs)')
+                 help='number of network jobs to run in parallel (defaults to --jobs or 1)')
     p.add_option('--jobs-checkout', default=None, type=int, metavar='JOBS',
-                 help='number of local checkout jobs to run in parallel (defaults to --jobs)')
+                 help='number of local checkout jobs to run in parallel (defaults to --jobs or '
+                      f'{DEFAULT_LOCAL_JOBS})')
 
     p.add_option('-f', '--force-broken',
                  dest='force_broken', action='store_true',
@@ -254,6 +312,10 @@ later is required to fix a server side protocol bug.
                  help='delete refs that no longer exist on the remote (default)')
     p.add_option('--no-prune', dest='prune', action='store_false',
                  help='do not delete refs that no longer exist on the remote')
+    p.add_option('--auto-gc', action='store_true', default=None,
+                 help='run garbage collection on all synced projects')
+    p.add_option('--no-auto-gc', dest='auto_gc', action='store_false',
+                 help='do not run garbage collection on any projects (default)')
     if show_smart:
       p.add_option('-s', '--smart-sync',
                    dest='smart_sync', action='store_true',
@@ -295,52 +357,92 @@ later is required to fix a server side protocol bug.
     """
     return git_superproject.UseSuperproject(opt.use_superproject, manifest) or opt.current_branch_only
 
-  def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests, superproject_logging_data, manifest):
-    """Update revisionId of every project with the SHA from superproject.
+  def _UpdateProjectsRevisionId(self, opt, args, superproject_logging_data,
+                                manifest):
+    """Update revisionId of projects with the commit hash from the superproject.
 
-    This function updates each project's revisionId with SHA from superproject.
-    It writes the updated manifest into a file and reloads the manifest from it.
+    This function updates each project's revisionId with the commit hash from
+    the superproject.  It writes the updated manifest into a file and reloads
+    the manifest from it.  When appropriate, sub manifests are also processed.
 
     Args:
       opt: Program options returned from optparse.  See _Options().
       args: Arguments to pass to GetProjects. See the GetProjects
           docstring for details.
-      load_local_manifests: Whether to load local manifests.
-      superproject_logging_data: A dictionary of superproject data that is to be logged.
+      superproject_logging_data: A dictionary of superproject data to log.
       manifest: The manifest to use.
-
-    Returns:
-      Returns path to the overriding manifest file instead of None.
     """
-    superproject = self.manifest.superproject
-    superproject.SetQuiet(opt.quiet)
-    print_messages = git_superproject.PrintMessages(opt.use_superproject,
-                                                    self.manifest)
-    superproject.SetPrintMessages(print_messages)
-    if opt.local_only:
-      manifest_path = superproject.manifest_path
+    have_superproject = manifest.superproject or any(
+        m.superproject for m in manifest.all_children)
+    if not have_superproject:
+      return
+
+    if opt.local_only and manifest.superproject:
+      manifest_path = manifest.superproject.manifest_path
       if manifest_path:
-        self._ReloadManifest(manifest_path, manifest, load_local_manifests)
-      return manifest_path
+        self._ReloadManifest(manifest_path, manifest)
+      return
 
     all_projects = self.GetProjects(args,
                                     missing_ok=True,
-                                    submodules_ok=opt.fetch_submodules)
-    update_result = superproject.UpdateProjectsRevisionId(
-        all_projects, git_event_log=self.git_event_log)
-    manifest_path = update_result.manifest_path
-    superproject_logging_data['updatedrevisionid'] = bool(manifest_path)
-    if manifest_path:
-      self._ReloadManifest(manifest_path, manifest, load_local_manifests)
+                                    submodules_ok=opt.fetch_submodules,
+                                    manifest=manifest,
+                                    all_manifests=not opt.this_manifest_only)
+
+    per_manifest = collections.defaultdict(list)
+    manifest_paths = {}
+    if opt.this_manifest_only:
+      per_manifest[manifest.path_prefix] = all_projects
     else:
-      if print_messages:
-        print('warning: Update of revisionId from superproject has failed, '
-              'repo sync will not use superproject to fetch the source. ',
-              'Please resync with the --no-use-superproject option to avoid this repo warning.',
-              file=sys.stderr)
-      if update_result.fatal and opt.use_superproject is not None:
-        sys.exit(1)
-    return manifest_path
+      for p in all_projects:
+        per_manifest[p.manifest.path_prefix].append(p)
+
+    superproject_logging_data = {}
+    need_unload = False
+    for m in self.ManifestList(opt):
+      if not m.path_prefix in per_manifest:
+        continue
+      use_super = git_superproject.UseSuperproject(opt.use_superproject, m)
+      if superproject_logging_data:
+        superproject_logging_data['multimanifest'] = True
+      superproject_logging_data.update(
+          superproject=use_super,
+          haslocalmanifests=bool(m.HasLocalManifests),
+          hassuperprojecttag=bool(m.superproject),
+      )
+      if use_super and (m.IsMirror or m.IsArchive):
+        # Don't use superproject, because we have no working tree.
+        use_super = False
+        superproject_logging_data['superproject'] = False
+        superproject_logging_data['noworktree'] = True
+        if opt.use_superproject is not False:
+          print(f'{m.path_prefix}: not using superproject because there is no '
+                'working tree.')
+
+      if not use_super:
+        continue
+      m.superproject.SetQuiet(opt.quiet)
+      print_messages = git_superproject.PrintMessages(opt.use_superproject, m)
+      m.superproject.SetPrintMessages(print_messages)
+      update_result = m.superproject.UpdateProjectsRevisionId(
+          per_manifest[m.path_prefix], git_event_log=self.git_event_log)
+      manifest_path = update_result.manifest_path
+      superproject_logging_data['updatedrevisionid'] = bool(manifest_path)
+      if manifest_path:
+        m.SetManifestOverride(manifest_path)
+        need_unload = True
+      else:
+        if print_messages:
+          print(f'{m.path_prefix}: warning: Update of revisionId from '
+                'superproject has failed, repo sync will not use superproject '
+                'to fetch the source. ',
+                'Please resync with the --no-use-superproject option to avoid '
+                'this repo warning.',
+                file=sys.stderr)
+        if update_result.fatal and opt.use_superproject is not None:
+          sys.exit(1)
+    if need_unload:
+      m.outer_client.manifest.Unload()
 
   def _FetchProjectList(self, opt, projects):
     """Main function of the fetch worker.
@@ -368,9 +470,10 @@ later is required to fix a server side protocol bug.
     """
     start = time.time()
     success = False
+    remote_fetched = False
     buf = io.StringIO()
     try:
-      success = project.Sync_NetworkHalf(
+      sync_result = project.Sync_NetworkHalf(
           quiet=opt.quiet,
           verbose=opt.verbose,
           output_redir=buf,
@@ -384,6 +487,8 @@ later is required to fix a server side protocol bug.
           ssh_proxy=self.ssh_proxy,
           clone_filter=project.manifest.CloneFilter,
           partial_clone_exclude=project.manifest.PartialCloneExclude)
+      success = sync_result.success
+      remote_fetched = sync_result.remote_fetched
 
       output = buf.getvalue()
       if (opt.verbose or not success) and output:
@@ -401,7 +506,7 @@ later is required to fix a server side protocol bug.
       raise
 
     finish = time.time()
-    return (success, project, start, finish)
+    return _FetchOneResult(success, project, start, finish, remote_fetched)
 
   @classmethod
   def _FetchInitChild(cls, ssh_proxy):
@@ -410,8 +515,9 @@ later is required to fix a server side protocol bug.
   def _Fetch(self, projects, opt, err_event, ssh_proxy):
     ret = True
 
-    jobs = opt.jobs_network if opt.jobs_network else self.jobs
+    jobs = opt.jobs_network
     fetched = set()
+    remote_fetched = set()
     pm = Progress('Fetching', len(projects), delay=False, quiet=opt.quiet)
 
     objdir_project_map = dict()
@@ -422,10 +528,16 @@ later is required to fix a server side protocol bug.
     def _ProcessResults(results_sets):
       ret = True
       for results in results_sets:
-        for (success, project, start, finish) in results:
+        for result in results:
+          success = result.success
+          project = result.project
+          start = result.start
+          finish = result.finish
           self._fetch_times.Set(project, finish - start)
           self.event_log.AddSync(project, event_log.TASK_SYNC_NETWORK,
                                  start, finish, success)
+          if result.remote_fetched:
+            remote_fetched.add(project)
           # Check for any errors before running any more tasks.
           # ...we'll let existing jobs finish, though.
           if not success:
@@ -483,10 +595,10 @@ later is required to fix a server side protocol bug.
     if not self.outer_client.manifest.IsArchive:
       self._GCProjects(projects, opt, err_event)
 
-    return (ret, fetched)
+    return _FetchResult(ret, fetched)
 
-  def _FetchMain(self, opt, args, all_projects, err_event, manifest_name,
-                 load_local_manifests, ssh_proxy, manifest):
+  def _FetchMain(self, opt, args, all_projects, err_event,
+                 ssh_proxy, manifest):
     """The main network fetch loop.
 
     Args:
@@ -494,8 +606,6 @@ later is required to fix a server side protocol bug.
       args: Command line args used to filter out projects.
       all_projects: List of all projects that should be fetched.
       err_event: Whether an error was hit while processing.
-      manifest_name: Manifest file to be reloaded.
-      load_local_manifests: Whether to load local manifests.
       ssh_proxy: SSH manager for clients & masters.
       manifest: The manifest to use.
 
@@ -511,7 +621,9 @@ later is required to fix a server side protocol bug.
     to_fetch.extend(all_projects)
     to_fetch.sort(key=self._fetch_times.Get, reverse=True)
 
-    success, fetched = self._Fetch(to_fetch, opt, err_event, ssh_proxy)
+    result = self._Fetch(to_fetch, opt, err_event, ssh_proxy)
+    success = result.success
+    fetched = result.projects
     if not success:
       err_event.set()
 
@@ -521,15 +633,17 @@ later is required to fix a server side protocol bug.
       if err_event.is_set():
         print('\nerror: Exited sync due to fetch errors.\n', file=sys.stderr)
         sys.exit(1)
-      return
+      return _FetchMainResult([])
 
     # Iteratively fetch missing and/or nested unregistered submodules
     previously_missing_set = set()
     while True:
-      self._ReloadManifest(manifest_name, self.manifest, load_local_manifests)
+      self._ReloadManifest(None, manifest)
       all_projects = self.GetProjects(args,
                                       missing_ok=True,
-                                      submodules_ok=opt.fetch_submodules)
+                                      submodules_ok=opt.fetch_submodules,
+                                      manifest=manifest,
+                                      all_manifests=not opt.this_manifest_only)
       missing = []
       for project in all_projects:
         if project.gitdir not in fetched:
@@ -542,12 +656,14 @@ later is required to fix a server side protocol bug.
       if previously_missing_set == missing_set:
         break
       previously_missing_set = missing_set
-      success, new_fetched = self._Fetch(missing, opt, err_event, ssh_proxy)
+      result = self._Fetch(missing, opt, err_event, ssh_proxy)
+      success = result.success
+      new_fetched = result.projects
       if not success:
         err_event.set()
       fetched.update(new_fetched)
 
-    return all_projects
+    return _FetchMainResult(all_projects)
 
   def _CheckoutOne(self, detach_head, force_sync, project):
     """Checkout work tree for one project
@@ -579,7 +695,7 @@ later is required to fix a server side protocol bug.
     if not success:
       print('error: Cannot checkout %s' % (project.name), file=sys.stderr)
     finish = time.time()
-    return (success, project, start, finish)
+    return _CheckoutOneResult(success, project, start, finish)
 
   def _Checkout(self, all_projects, opt, err_results):
     """Checkout projects listed in all_projects
@@ -594,14 +710,18 @@ later is required to fix a server side protocol bug.
 
     def _ProcessResults(pool, pm, results):
       ret = True
-      for (success, project, start, finish) in results:
+      for result in results:
+        success = result.success
+        project = result.project
+        start = result.start
+        finish = result.finish
         self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL,
                                start, finish, success)
         # Check for any errors before running any more tasks.
         # ...we'll let existing jobs finish, though.
         if not success:
           ret = False
-          err_results.append(project.relpath)
+          err_results.append(project.RelPath(local=opt.this_manifest_only))
           if opt.fail_fast:
             if pool:
               pool.close()
@@ -610,33 +730,105 @@ later is required to fix a server side protocol bug.
       return ret
 
     return self.ExecuteInParallel(
-        opt.jobs_checkout if opt.jobs_checkout else self.jobs,
+        opt.jobs_checkout,
         functools.partial(self._CheckoutOne, opt.detach_head, opt.force_sync),
         all_projects,
         callback=_ProcessResults,
         output=Progress('Checking out', len(all_projects), quiet=opt.quiet)) and not err_results
 
-  def _GCProjects(self, projects, opt, err_event):
-    pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet)
-    pm.update(inc=0, msg='prescan')
+  @staticmethod
+  def _GetPreciousObjectsState(project: Project, opt):
+    """Get the preciousObjects state for the project.
 
-    tidy_dirs = {}
-    for project in projects:
-      # Make sure pruning never kicks in with shared projects.
-      if (not project.use_git_worktrees and
-              len(project.manifest.GetProjectsWithName(project.name)) > 1):
+    Args:
+      project (Project): the project to examine, and possibly correct.
+      opt (optparse.Values): options given to sync.
+
+    Returns:
+      Expected state of extensions.preciousObjects:
+        False: Should be disabled. (not present)
+        True: Should be enabled.
+    """
+    if project.use_git_worktrees:
+      return False
+    projects = project.manifest.GetProjectsWithName(project.name,
+                                                    all_manifests=True)
+    if len(projects) == 1:
+      return False
+    relpath = project.RelPath(local=opt.this_manifest_only)
+    if len(projects) > 1:
+      # Objects are potentially shared with another project.
+      # See the logic in Project.Sync_NetworkHalf regarding UseAlternates.
+      # - When False, shared projects share (via symlink)
+      #   .repo/project-objects/{PROJECT_NAME}.git as the one-and-only objects
+      #   directory.  All objects are precious, since there is no project with a
+      #   complete set of refs.
+      # - When True, shared projects share (via info/alternates)
+      #   .repo/project-objects/{PROJECT_NAME}.git as an alternate object store,
+      #   which is written only on the first clone of the project, and is not
+      #   written subsequently.  (When Sync_NetworkHalf sees that it exists, it
+      #   makes sure that the alternates file points there, and uses a
+      #   project-local .git/objects directory for all syncs going forward.
+      # We do not support switching between the options.  The environment
+      # variable is present for testing and migration only.
+      return not project.UseAlternates
+    print(f'\r{relpath}: project not found in manifest.', file=sys.stderr)
+    return False
+
+  def _SetPreciousObjectsState(self, project: Project, opt):
+    """Correct the preciousObjects state for the project.
+
+    Args:
+      project: the project to examine, and possibly correct.
+      opt: options given to sync.
+    """
+    expected = self._GetPreciousObjectsState(project, opt)
+    actual = project.config.GetBoolean('extensions.preciousObjects') or False
+    relpath = project.RelPath(local=opt.this_manifest_only)
+
+    if expected != actual:
+      # If this is unexpected, log it and repair.
+      Trace(f'{relpath} expected preciousObjects={expected}, got {actual}')
+      if expected:
         if not opt.quiet:
           print('\r%s: Shared project %s found, disabling pruning.' %
-                (project.relpath, project.name))
+                (relpath, project.name))
         if git_require((2, 7, 0)):
           project.EnableRepositoryExtension('preciousObjects')
         else:
           # This isn't perfect, but it's the best we can do with old git.
-          print('\r%s: WARNING: shared projects are unreliable when using old '
-                'versions of git; please upgrade to git-2.7.0+.'
-                % (project.relpath,),
+          print('\r%s: WARNING: shared projects are unreliable when using '
+                'old versions of git; please upgrade to git-2.7.0+.'
+                % (relpath,),
                 file=sys.stderr)
           project.config.SetString('gc.pruneExpire', 'never')
+      else:
+        if not opt.quiet:
+          print(f'\r{relpath}: not shared, disabling pruning.')
+        project.config.SetString('extensions.preciousObjects', None)
+        project.config.SetString('gc.pruneExpire', None)
+
+  def _GCProjects(self, projects, opt, err_event):
+    """Perform garbage collection.
+
+    If We are skipping garbage collection (opt.auto_gc not set), we still want
+    to potentially mark objects precious, so that `git gc` does not discard
+    shared objects.
+    """
+    if not opt.auto_gc:
+      # Just repair preciousObjects state, and return.
+      for project in projects:
+        self._SetPreciousObjectsState(project, opt)
+      return
+
+    pm = Progress('Garbage collecting', len(projects), delay=False,
+                  quiet=opt.quiet)
+    pm.update(inc=0, msg='prescan')
+
+    tidy_dirs = {}
+    for project in projects:
+      self._SetPreciousObjectsState(project, opt)
+
       project.config.SetString('gc.autoDetach', 'false')
       # Only call git gc once per objdir, but call pack-refs for the remainder.
       if project.objdir not in tidy_dirs:
@@ -650,12 +842,12 @@ later is required to fix a server side protocol bug.
             project.bare_git,
         )
 
-    cpu_count = os.cpu_count()
-    jobs = min(self.jobs, cpu_count)
+    jobs = opt.jobs
 
     if jobs < 2:
       for (run_gc, bare_git) in tidy_dirs.values():
         pm.update(msg=bare_git._project.name)
+
         if run_gc:
           bare_git.gc('--auto')
         else:
@@ -663,6 +855,7 @@ later is required to fix a server side protocol bug.
       pm.end()
       return
 
+    cpu_count = os.cpu_count()
     config = {'pack.threads': cpu_count // jobs if cpu_count > jobs else 1}
 
     threads = set()
@@ -698,7 +891,7 @@ later is required to fix a server side protocol bug.
       t.join()
     pm.end()
 
-  def _ReloadManifest(self, manifest_name, manifest, load_local_manifests=True):
+  def _ReloadManifest(self, manifest_name, manifest):
     """Reload the manfiest from the file specified by the |manifest_name|.
 
     It unloads the manifest if |manifest_name| is None.
@@ -706,17 +899,29 @@ later is required to fix a server side protocol bug.
     Args:
       manifest_name: Manifest file to be reloaded.
       manifest: The manifest to use.
-      load_local_manifests: Whether to load local manifests.
     """
     if manifest_name:
       # Override calls Unload already
-      manifest.Override(manifest_name, load_local_manifests=load_local_manifests)
+      manifest.Override(manifest_name)
     else:
       manifest.Unload()
 
   def UpdateProjectList(self, opt, manifest):
+    """Update the cached projects list for |manifest|
+
+    In a multi-manifest checkout, each manifest has its own project.list.
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      manifest: The manifest to use.
+
+    Returns:
+      0: success
+      1: failure
+    """
     new_project_paths = []
-    for project in self.GetProjects(None, missing_ok=True):
+    for project in self.GetProjects(None, missing_ok=True, manifest=manifest,
+                                    all_manifests=False):
       if project.relpath:
         new_project_paths.append(project.relpath)
     file_name = 'project.list'
@@ -766,7 +971,8 @@ later is required to fix a server side protocol bug.
     new_paths = {}
     new_linkfile_paths = []
     new_copyfile_paths = []
-    for project in self.GetProjects(None, missing_ok=True):
+    for project in self.GetProjects(None, missing_ok=True,
+                                    manifest=manifest, all_manifests=False):
       new_linkfile_paths.extend(x.dest for x in project.linkfiles)
       new_copyfile_paths.extend(x.dest for x in project.copyfiles)
 
@@ -897,8 +1103,40 @@ later is required to fix a server side protocol bug.
 
     return manifest_name
 
+  def _UpdateAllManifestProjects(self, opt, mp, manifest_name):
+    """Fetch & update the local manifest project.
+
+    After syncing the manifest project, if the manifest has any sub manifests,
+    those are recursively processed.
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      mp: the manifestProject to query.
+      manifest_name: Manifest file to be reloaded.
+    """
+    if not mp.standalone_manifest_url:
+      self._UpdateManifestProject(opt, mp, manifest_name)
+
+    if mp.manifest.submanifests:
+      for submanifest in mp.manifest.submanifests.values():
+        child = submanifest.repo_client.manifest
+        child.manifestProject.SyncWithPossibleInit(
+            submanifest,
+            current_branch_only=self._GetCurrentBranchOnly(opt, child),
+            verbose=opt.verbose,
+            tags=opt.tags,
+            git_event_log=self.git_event_log,
+        )
+        self._UpdateAllManifestProjects(opt, child.manifestProject, None)
+
   def _UpdateManifestProject(self, opt, mp, manifest_name):
-    """Fetch & update the local manifest project."""
+    """Fetch & update the local manifest project.
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      mp: the manifestProject to query.
+      manifest_name: Manifest file to be reloaded.
+    """
     if not opt.local_only:
       start = time.time()
       success = mp.Sync_NetworkHalf(quiet=opt.quiet, verbose=opt.verbose,
@@ -924,8 +1162,6 @@ later is required to fix a server side protocol bug.
       if not clean:
         sys.exit(1)
       self._ReloadManifest(manifest_name, mp.manifest)
-      if opt.jobs is None:
-        self.jobs = mp.manifest.default.sync_j
 
   def ValidateOptions(self, opt, args):
     if opt.force_broken:
@@ -948,18 +1184,54 @@ later is required to fix a server side protocol bug.
     if opt.prune is None:
       opt.prune = True
 
-    if self.outer_client.manifest.is_multimanifest and not opt.this_manifest_only and args:
-      self.OptionParser.error('partial syncs must use --this-manifest-only')
+    if opt.auto_gc is None and _AUTO_GC:
+      print(f"Will run `git gc --auto` because {_REPO_AUTO_GC} is set.",
+            f'{_REPO_AUTO_GC} is deprecated and will be removed in a future',
+            'release.  Use `--auto-gc` instead.', file=sys.stderr)
+      opt.auto_gc = True
+
+  def _ValidateOptionsWithManifest(self, opt, mp):
+    """Like ValidateOptions, but after we've updated the manifest.
+
+    Needed to handle sync-xxx option defaults in the manifest.
+
+    Args:
+      opt: The options to process.
+      mp: The manifest project to pull defaults from.
+    """
+    if not opt.jobs:
+      # If the user hasn't made a choice, use the manifest value.
+      opt.jobs = mp.manifest.default.sync_j
+    if opt.jobs:
+      # If --jobs has a non-default value, propagate it as the default for
+      # --jobs-xxx flags too.
+      if not opt.jobs_network:
+        opt.jobs_network = opt.jobs
+      if not opt.jobs_checkout:
+        opt.jobs_checkout = opt.jobs
+    else:
+      # Neither user nor manifest have made a choice, so setup defaults.
+      if not opt.jobs_network:
+        opt.jobs_network = 1
+      if not opt.jobs_checkout:
+        opt.jobs_checkout = DEFAULT_LOCAL_JOBS
+      opt.jobs = os.cpu_count()
+
+    # Try to stay under user rlimit settings.
+    #
+    # Since each worker requires at 3 file descriptors to run `git fetch`, use
+    # that to scale down the number of jobs.  Unfortunately there isn't an easy
+    # way to determine this reliably as systems change, but it was last measured
+    # by hand in 2011.
+    soft_limit, _ = _rlimit_nofile()
+    jobs_soft_limit = max(1, (soft_limit - 5) // 3)
+    opt.jobs = min(opt.jobs, jobs_soft_limit)
+    opt.jobs_network = min(opt.jobs_network, jobs_soft_limit)
+    opt.jobs_checkout = min(opt.jobs_checkout, jobs_soft_limit)
 
   def Execute(self, opt, args):
-    if opt.jobs:
-      self.jobs = opt.jobs
-    if self.jobs > 1:
-      soft_limit, _ = _rlimit_nofile()
-      self.jobs = min(self.jobs, (soft_limit - 5) // 3)
-
     manifest = self.outer_manifest
-    if opt.this_manifest_only or not opt.outer_manifest:
+    if not opt.outer_manifest:
       manifest = self.manifest
 
     if opt.manifest_name:
@@ -994,39 +1266,29 @@ later is required to fix a server side protocol bug.
               'receive updates; run `repo init --repo-rev=stable` to fix.',
               file=sys.stderr)
 
-    mp = manifest.manifestProject
-    is_standalone_manifest = bool(mp.standalone_manifest_url)
-    if not is_standalone_manifest:
-      mp.PreSync()
+    for m in self.ManifestList(opt):
+      if not m.manifestProject.standalone_manifest_url:
+        m.manifestProject.PreSync()
 
     if opt.repo_upgraded:
       _PostRepoUpgrade(manifest, quiet=opt.quiet)
 
-    if not opt.mp_update:
+    mp = manifest.manifestProject
+    if opt.mp_update:
+      self._UpdateAllManifestProjects(opt, mp, manifest_name)
+    else:
       print('Skipping update of local manifest project.')
-    elif not is_standalone_manifest:
-      self._UpdateManifestProject(opt, mp, manifest_name)
 
-    load_local_manifests = not manifest.HasLocalManifests
-    use_superproject = git_superproject.UseSuperproject(opt.use_superproject, manifest)
-    if use_superproject and (manifest.IsMirror or manifest.IsArchive):
-      # Don't use superproject, because we have no working tree.
-      use_superproject = False
-      if opt.use_superproject is not None:
-        print('Defaulting to no-use-superproject because there is no working tree.')
-    superproject_logging_data = {
-        'superproject': use_superproject,
-        'haslocalmanifests': bool(manifest.HasLocalManifests),
-        'hassuperprojecttag': bool(manifest.superproject),
-    }
-    if use_superproject:
-      manifest_name = self._UpdateProjectsRevisionId(
-          opt, args, load_local_manifests, superproject_logging_data,
-          manifest) or opt.manifest_name
+    # Now that the manifests are up-to-date, setup options whose defaults might
+    # be in the manifest.
+    self._ValidateOptionsWithManifest(opt, mp)
+
+    superproject_logging_data = {}
+    self._UpdateProjectsRevisionId(opt, args, superproject_logging_data,
+                                   manifest)
 
     if self.gitc_manifest:
-      gitc_manifest_projects = self.GetProjects(args,
-                                                missing_ok=True)
+      gitc_manifest_projects = self.GetProjects(args, missing_ok=True)
       gitc_projects = []
       opened_projects = []
       for project in gitc_manifest_projects:
@@ -1059,12 +1321,16 @@ later is required to fix a server side protocol bug.
               for path in opened_projects]
       if not args:
         return
+
     all_projects = self.GetProjects(args,
                                     missing_ok=True,
-                                    submodules_ok=opt.fetch_submodules)
+                                    submodules_ok=opt.fetch_submodules,
+                                    manifest=manifest,
+                                    all_manifests=not opt.this_manifest_only)
 
     err_network_sync = False
     err_update_projects = False
+    err_update_linkfiles = False
 
     self._fetch_times = _FetchTimes(manifest)
     if not opt.local_only:
@@ -1072,9 +1338,9 @@ later is required to fix a server side protocol bug.
         with ssh.ProxyManager(manager) as ssh_proxy:
           # Initialize the socket dir once in the parent.
           ssh_proxy.sock()
-          all_projects = self._FetchMain(opt, args, all_projects, err_event,
-                                         manifest_name, load_local_manifests,
-                                         ssh_proxy, manifest)
+          result = self._FetchMain(opt, args, all_projects, err_event,
+                                   ssh_proxy, manifest)
+          all_projects = result.all_projects
 
       if opt.network_only:
         return
@@ -1090,23 +1356,24 @@ later is required to fix a server side protocol bug.
                 file=sys.stderr)
           sys.exit(1)
 
-    if manifest.IsMirror or manifest.IsArchive:
-      # bail out now, we have no working tree
-      return
+    for m in self.ManifestList(opt):
+      if m.IsMirror or m.IsArchive:
+        # bail out now, we have no working tree
+        continue
 
-    if self.UpdateProjectList(opt, manifest):
-      err_event.set()
-      err_update_projects = True
-      if opt.fail_fast:
-        print('\nerror: Local checkouts *not* updated.', file=sys.stderr)
-        sys.exit(1)
+      if self.UpdateProjectList(opt, m):
+        err_event.set()
+        err_update_projects = True
+        if opt.fail_fast:
+          print('\nerror: Local checkouts *not* updated.', file=sys.stderr)
+          sys.exit(1)
 
-    err_update_linkfiles = not self.UpdateCopyLinkfileList(manifest)
-    if err_update_linkfiles:
-      err_event.set()
-      if opt.fail_fast:
-        print('\nerror: Local update copyfile or linkfile failed.', file=sys.stderr)
-        sys.exit(1)
+      err_update_linkfiles = not self.UpdateCopyLinkfileList(m)
+      if err_update_linkfiles:
+        err_event.set()
+        if opt.fail_fast:
+          print('\nerror: Local update copyfile or linkfile failed.', file=sys.stderr)
+          sys.exit(1)
 
     err_results = []
     # NB: We don't exit here because this is the last step.
@@ -1114,10 +1381,14 @@ later is required to fix a server side protocol bug.
     if err_checkout:
       err_event.set()
 
-    # If there's a notice that's supposed to print at the end of the sync, print
-    # it now...
-    if manifest.notice:
-      print(manifest.notice)
+    printed_notices = set()
+    # If there's a notice that's supposed to print at the end of the sync,
+    # print it now...  But avoid printing duplicate messages, and preserve
+    # order.
+    for m in sorted(self.ManifestList(opt), key=lambda x: x.path_prefix):
+      if m.notice and m.notice not in printed_notices:
+        print(m.notice)
+        printed_notices.add(m.notice)
 
     # If we saw an error, exit with code 1 so that other scripts can check.
     if err_event.is_set():
@@ -1332,11 +1603,16 @@ class PersistentTransport(xmlrpc.client.Transport):
           raise
 
       p, u = xmlrpc.client.getparser()
-      while 1:
-        data = response.read(1024)
-        if not data:
-          break
+      # Response should be fairly small, so read it all at once.
+      # This way we can show it to the user in case of error (e.g. HTML).
+      data = response.read()
+      try:
         p.feed(data)
+      except xml.parsers.expat.ExpatError as e:
+        raise IOError(
+            f'Parsing the manifest failed: {e}\n'
+            f'Please report this to your manifest server admin.\n'
+            f'Here is the full response:\n{data.decode("utf-8")}')
       p.close()
       return u.close()
 

subcmds/upload.py

@@ -78,6 +78,13 @@ added to the respective list of users, and emails are sent to any
 new users.  Users passed as --reviewers must already be registered
 with the code review system, or the upload will fail.
 
+While most normal Gerrit options have dedicated command line options,
+direct access to the Gerit options is available via --push-options.
+This is useful when Gerrit has newer functionality that %prog doesn't
+yet support, or doesn't have plans to support.  See the Push Options
+documentation for more details:
+https://gerrit-review.googlesource.com/Documentation/user-upload.html#push_options
+
 # Configuration
 
 review.URL.autoupload:
@@ -190,6 +197,9 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     p.add_option('-w', '--wip',
                  action='store_true', dest='wip', default=False,
                  help='upload as a work-in-progress change')
+    p.add_option('-r', '--ready',
+                 action='store_true', default=False,
+                 help='mark change as ready (clears work-in-progress setting)')
     p.add_option('-o', '--push-option',
                  type='string', action='append', dest='push_options',
                  default=[],
@@ -204,6 +214,12 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     p.add_option('-y', '--yes',
                  default=False, action='store_true',
                  help='answer yes to all safe prompts')
+    p.add_option('--ignore-untracked-files',
+                 action='store_true', default=False,
+                 help='ignore untracked files in the working copy')
+    p.add_option('--no-ignore-untracked-files',
+                 dest='ignore_untracked_files', action='store_false',
+                 help='always ask about untracked files in the working copy')
     p.add_option('--no-cert-checks',
                  dest='validate_certs', action='store_false', default=True,
                  help='disable verifying ssl certs (unsafe)')
@@ -246,7 +262,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
         answer = sys.stdin.readline().strip().lower()
         answer = answer in ('y', 'yes', '1', 'true', 't')
 
-    if answer:
+    if not opt.yes and answer:
       if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD:
         answer = _ConfirmManyUploads()
 
@@ -262,8 +278,9 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     script = []
     script.append('# Uncomment the branches to upload:')
     for project, avail in pending:
+      project_path = project.RelPath(local=opt.this_manifest_only)
       script.append('#')
-      script.append('# project %s/:' % project.RelPath(local=opt.this_manifest_only))
+      script.append(f'# project {project_path}/:')
 
       b = {}
       for branch in avail:
@@ -286,8 +303,8 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
           script.append('#         %s' % commit)
         b[name] = branch
 
-      projects[project.RelPath(local=opt.this_manifest_only)] = project
-      branches[project.name] = b
+      projects[project_path] = project
+      branches[project_path] = b
     script.append('')
 
     script = Editor.EditString("\n".join(script)).split("\n")
@@ -312,21 +329,23 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
         name = m.group(1)
         if not project:
           _die('project for branch %s not in script', name)
-        branch = branches[project.name].get(name)
+        project_path = project.RelPath(local=opt.this_manifest_only)
+        branch = branches[project_path].get(name)
         if not branch:
-          _die('branch %s not in %s', name, project.RelPath(local=opt.this_manifest_only))
+          _die('branch %s not in %s', name, project_path)
         todo.append(branch)
     if not todo:
       _die("nothing uncommented for upload")
 
-    many_commits = False
-    for branch in todo:
-      if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD:
-        many_commits = True
-        break
-    if many_commits:
-      if not _ConfirmManyUploads(multiple_branches=True):
-        _die("upload aborted by user")
+    if not opt.yes:
+      many_commits = False
+      for branch in todo:
+        if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD:
+          many_commits = True
+          break
+      if many_commits:
+        if not _ConfirmManyUploads(multiple_branches=True):
+          _die("upload aborted by user")
 
     self._UploadAndReport(opt, todo, people)
 
@@ -370,6 +389,10 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
 
         # Check if there are local changes that may have been forgotten
         changes = branch.project.UncommitedFiles()
+        if opt.ignore_untracked_files:
+          untracked = set(branch.project.UntrackedFiles())
+          changes = [x for x in changes if x not in untracked]
+
         if changes:
           key = 'review.%s.autoupload' % branch.project.remote.review
           answer = branch.project.config.GetBoolean(key)
@@ -421,12 +444,6 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
         labels = set(_ExpandCommaList(branch.project.config.GetString(key)))
         for label in opt.labels:
           labels.update(_ExpandCommaList(label))
-        # Basic sanity check on label syntax.
-        for label in labels:
-          if not re.match(r'^.+[+-][0-9]+$', label):
-            print('repo: error: invalid label syntax "%s": labels use forms '
-                  'like CodeReview+1 or Verified-1' % (label,), file=sys.stderr)
-            sys.exit(1)
 
         # Handle e-mail notifications.
         if opt.notify is False:
@@ -461,6 +478,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
                                private=opt.private,
                                notify=notify,
                                wip=opt.wip,
+                               ready=opt.ready,
                                dest_branch=destination,
                                validate_certs=opt.validate_certs,
                                push_options=opt.push_options)

subcmds/version.py

@@ -33,7 +33,7 @@ class Version(Command, MirrorSafeCommand):
 
   def Execute(self, opt, args):
     rp = self.manifest.repoProject
-    rem = rp.GetRemote(rp.remote.name)
+    rem = rp.GetRemote()
     branch = rp.GetBranch('default')
 
     # These might not be the same.  Report them both.

tests/test_git_command.py

@@ -15,6 +15,7 @@
 """Unittests for the git_command.py module."""
 
 import re
+import os
 import unittest
 
 try:
@@ -26,6 +27,38 @@ import git_command
 import wrapper
 
 
+class GitCommandTest(unittest.TestCase):
+  """Tests the GitCommand class (via git_command.git)."""
+
+  def setUp(self):
+
+    def realpath_mock(val):
+      return val
+
+    mock.patch.object(os.path, 'realpath', side_effect=realpath_mock).start()
+
+  def tearDown(self):
+    mock.patch.stopall()
+
+  def test_alternative_setting_when_matching(self):
+    r = git_command._build_env(
+      objdir = os.path.join('zap', 'objects'),
+      gitdir = 'zap'
+    )
+
+    self.assertIsNone(r.get('GIT_ALTERNATE_OBJECT_DIRECTORIES'))
+    self.assertEqual(r.get('GIT_OBJECT_DIRECTORY'), os.path.join('zap', 'objects'))
+
+  def test_alternative_setting_when_different(self):
+    r = git_command._build_env(
+      objdir = os.path.join('wow', 'objects'),
+      gitdir = 'zap'
+    )
+
+    self.assertEqual(r.get('GIT_ALTERNATE_OBJECT_DIRECTORIES'), os.path.join('zap', 'objects'))
+    self.assertEqual(r.get('GIT_OBJECT_DIRECTORY'), os.path.join('wow', 'objects'))
+
+
 class GitCallUnitTest(unittest.TestCase):
   """Tests the _GitCall class (via git_command.git)."""
 
@@ -84,7 +117,8 @@ class GitRequireTests(unittest.TestCase):
   """Test the git_require helper."""
 
   def setUp(self):
-    ver = wrapper.GitVersion(1, 2, 3, 4)
+    self.wrapper = wrapper.Wrapper()
+    ver = self.wrapper.GitVersion(1, 2, 3, 4)
     mock.patch.object(git_command.git, 'version_tuple', return_value=ver).start()
 
   def tearDown(self):

tests/test_git_config.py

@@ -19,6 +19,7 @@ import tempfile
 import unittest
 
 import git_config
+import repo_trace
 
 
 def fixture(*paths):
@@ -33,9 +34,16 @@ class GitConfigReadOnlyTests(unittest.TestCase):
   def setUp(self):
     """Create a GitConfig object using the test.gitconfig fixture.
     """
+
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    repo_trace._TRACE_FILE = os.path.join(self.tempdirobj.name, 'TRACE_FILE_from_test')
+
     config_fixture = fixture('test.gitconfig')
     self.config = git_config.GitConfig(config_fixture)
 
+  def tearDown(self):
+    self.tempdirobj.cleanup()
+
   def test_GetString_with_empty_config_values(self):
     """
     Test config entries with no value.
@@ -109,9 +117,15 @@ class GitConfigReadWriteTests(unittest.TestCase):
   """Read/write tests of the GitConfig class."""
 
   def setUp(self):
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    repo_trace._TRACE_FILE = os.path.join(self.tempdirobj.name, 'TRACE_FILE_from_test')
+
     self.tmpfile = tempfile.NamedTemporaryFile()
     self.config = self.get_config()
 
+  def tearDown(self):
+    self.tempdirobj.cleanup()
+
   def get_config(self):
     """Get a new GitConfig instance."""
     return git_config.GitConfig(self.tmpfile.name)
@@ -186,7 +200,3 @@ class GitConfigReadWriteTests(unittest.TestCase):
     for key, value in TESTS:
       self.assertEqual(sync_data[f'{git_config.SYNC_STATE_PREFIX}{key}'], value)
     self.assertTrue(sync_data[f'{git_config.SYNC_STATE_PREFIX}main.synctime'])
-
-
-if __name__ == '__main__':
-  unittest.main()

tests/test_git_superproject.py

@@ -24,7 +24,7 @@ from unittest import mock
 import git_superproject
 import git_trace2_event_log
 import manifest_xml
-import platform_utils
+import repo_trace
 from test_manifest_xml import sort_attributes
 
 
@@ -38,7 +38,9 @@ class SuperprojectTestCase(unittest.TestCase):
 
   def setUp(self):
     """Set up superproject every time."""
-    self.tempdir = tempfile.mkdtemp(prefix='repo_tests')
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    self.tempdir = self.tempdirobj.name
+    repo_trace._TRACE_FILE = os.path.join(self.tempdir, 'TRACE_FILE_from_test')
     self.repodir = os.path.join(self.tempdir, '.repo')
     self.manifest_file = os.path.join(
         self.repodir, manifest_xml.MANIFEST_FILE_NAME)
@@ -75,7 +77,7 @@ class SuperprojectTestCase(unittest.TestCase):
 
   def tearDown(self):
     """Tear down superproject every time."""
-    platform_utils.rmtree(self.tempdir)
+    self.tempdirobj.cleanup()
 
   def getXmlManifest(self, data):
     """Helper to initialize a manifest for testing."""
@@ -312,9 +314,6 @@ class SuperprojectTestCase(unittest.TestCase):
               '<project groups="notdefault,platform-' + self.platform + '" '
               'name="platform/art" path="art" '
               'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea" upstream="refs/heads/main"/>'
-              '<project clone-depth="1" groups="' + local_group + '" '
-              'name="platform/vendor/x" path="vendor/x" remote="goog" '
-              'revision="master-with-vendor"/>'
               '<superproject name="superproject"/>'
               '</manifest>')
 
@@ -367,7 +366,3 @@ class SuperprojectTestCase(unittest.TestCase):
               'revision="52d3c9f7c107839ece2319d077de0cd922aa9d8f"/>'
               '<superproject name="superproject"/>'
               '</manifest>')
-
-
-if __name__ == '__main__':
-  unittest.main()

tests/test_git_trace2_event_log.py

@@ -369,7 +369,7 @@ class EventLogTestCase(unittest.TestCase):
         server_thread.start()
 
         with server_ready:
-          server_ready.wait()
+          server_ready.wait(timeout=120)
 
         self._event_log_module.StartEvent()
         path = self._event_log_module.Write(path=f'af_unix:{socket_path}')
@@ -385,7 +385,3 @@ class EventLogTestCase(unittest.TestCase):
     # Check for 'start' event specific fields.
     self.assertIn('argv', start_event)
     self.assertIsInstance(start_event['argv'], list)
-
-
-if __name__ == '__main__':
-  unittest.main()

tests/test_manifest_xml.py

@@ -17,13 +17,13 @@
 import os
 import platform
 import re
-import shutil
 import tempfile
 import unittest
 import xml.dom.minidom
 
 import error
 import manifest_xml
+import repo_trace
 
 
 # Invalid paths that we don't want in the filesystem.
@@ -92,7 +92,9 @@ class ManifestParseTestCase(unittest.TestCase):
   """TestCase for parsing manifests."""
 
   def setUp(self):
-    self.tempdir = tempfile.mkdtemp(prefix='repo_tests')
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    self.tempdir = self.tempdirobj.name
+    repo_trace._TRACE_FILE = os.path.join(self.tempdir, 'TRACE_FILE_from_test')
     self.repodir = os.path.join(self.tempdir, '.repo')
     self.manifest_dir = os.path.join(self.repodir, 'manifests')
     self.manifest_file = os.path.join(
@@ -111,11 +113,11 @@ class ManifestParseTestCase(unittest.TestCase):
 """)
 
   def tearDown(self):
-    shutil.rmtree(self.tempdir, ignore_errors=True)
+    self.tempdirobj.cleanup()
 
   def getXmlManifest(self, data):
     """Helper to initialize a manifest for testing."""
-    with open(self.manifest_file, 'w') as fp:
+    with open(self.manifest_file, 'w', encoding="utf-8") as fp:
       fp.write(data)
     return manifest_xml.XmlManifest(self.repodir, self.manifest_file)
 
@@ -252,6 +254,37 @@ class XmlManifestTests(ManifestParseTestCase):
         '<manifest></manifest>')
     self.assertEqual(manifest.ToDict(), {})
 
+  def test_toxml_omit_local(self):
+    """Does not include local_manifests projects when omit_local=True."""
+    manifest = self.getXmlManifest(
+        '<?xml version="1.0" encoding="UTF-8"?><manifest>'
+        '<remote name="a" fetch=".."/><default remote="a" revision="r"/>'
+        '<project name="p" groups="local::me"/>'
+        '<project name="q"/>'
+        '<project name="r" groups="keep"/>'
+        '</manifest>')
+    self.assertEqual(
+        sort_attributes(manifest.ToXml(omit_local=True).toxml()),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch=".." name="a"/><default remote="a" revision="r"/>'
+        '<project name="q"/><project groups="keep" name="r"/></manifest>')
+
+  def test_toxml_with_local(self):
+    """Does include local_manifests projects when omit_local=False."""
+    manifest = self.getXmlManifest(
+        '<?xml version="1.0" encoding="UTF-8"?><manifest>'
+        '<remote name="a" fetch=".."/><default remote="a" revision="r"/>'
+        '<project name="p" groups="local::me"/>'
+        '<project name="q"/>'
+        '<project name="r" groups="keep"/>'
+        '</manifest>')
+    self.assertEqual(
+        sort_attributes(manifest.ToXml(omit_local=False).toxml()),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch=".." name="a"/><default remote="a" revision="r"/>'
+        '<project groups="local::me" name="p"/>'
+        '<project name="q"/><project groups="keep" name="r"/></manifest>')
+
   def test_repo_hooks(self):
     """Check repo-hooks settings."""
     manifest = self.getXmlManifest("""
@@ -395,7 +428,7 @@ class IncludeElementTests(ManifestParseTestCase):
     def parse(name):
       name = self.encodeXmlAttr(name)
       # Setup target of the include.
-      with open(os.path.join(self.manifest_dir, 'target.xml'), 'w') as fp:
+      with open(os.path.join(self.manifest_dir, 'target.xml'), 'w', encoding="utf-8") as fp:
         fp.write(f'<manifest><include name="{name}"/></manifest>')
 
       manifest = self.getXmlManifest("""
@@ -486,22 +519,22 @@ class ProjectElementTests(ManifestParseTestCase):
 """)
 
     manifest = parse('a/path/', 'foo')
-    self.assertEqual(manifest.projects[0].gitdir,
-                     os.path.join(self.tempdir, '.repo/projects/foo.git'))
-    self.assertEqual(manifest.projects[0].objdir,
-                     os.path.join(self.tempdir, '.repo/project-objects/a/path.git'))
+    self.assertEqual(os.path.normpath(manifest.projects[0].gitdir),
+                     os.path.join(self.tempdir, '.repo', 'projects', 'foo.git'))
+    self.assertEqual(os.path.normpath(manifest.projects[0].objdir),
+                     os.path.join(self.tempdir, '.repo', 'project-objects', 'a', 'path.git'))
 
     manifest = parse('a/path', 'foo/')
-    self.assertEqual(manifest.projects[0].gitdir,
-                     os.path.join(self.tempdir, '.repo/projects/foo.git'))
-    self.assertEqual(manifest.projects[0].objdir,
-                     os.path.join(self.tempdir, '.repo/project-objects/a/path.git'))
+    self.assertEqual(os.path.normpath(manifest.projects[0].gitdir),
+                     os.path.join(self.tempdir, '.repo', 'projects', 'foo.git'))
+    self.assertEqual(os.path.normpath(manifest.projects[0].objdir),
+                     os.path.join(self.tempdir, '.repo', 'project-objects', 'a', 'path.git'))
 
     manifest = parse('a/path', 'foo//////')
-    self.assertEqual(manifest.projects[0].gitdir,
-                     os.path.join(self.tempdir, '.repo/projects/foo.git'))
-    self.assertEqual(manifest.projects[0].objdir,
-                     os.path.join(self.tempdir, '.repo/project-objects/a/path.git'))
+    self.assertEqual(os.path.normpath(manifest.projects[0].gitdir),
+                     os.path.join(self.tempdir, '.repo', 'projects', 'foo.git'))
+    self.assertEqual(os.path.normpath(manifest.projects[0].objdir),
+                     os.path.join(self.tempdir, '.repo', 'project-objects', 'a', 'path.git'))
 
   def test_toplevel_path(self):
     """Check handling of path=. specially."""
@@ -518,8 +551,8 @@ class ProjectElementTests(ManifestParseTestCase):
 
     for path in ('.', './', './/', './//'):
       manifest = parse('server/path', path)
-      self.assertEqual(manifest.projects[0].gitdir,
-                       os.path.join(self.tempdir, '.repo/projects/..git'))
+      self.assertEqual(os.path.normpath(manifest.projects[0].gitdir),
+                       os.path.join(self.tempdir, '.repo', 'projects', '..git'))
 
   def test_bad_path_name_checks(self):
     """Check handling of bad path & name attributes."""
@@ -545,7 +578,7 @@ class ProjectElementTests(ManifestParseTestCase):
       parse('', 'ok')
 
     for path in INVALID_FS_PATHS:
-      if not path or path.endswith('/'):
+      if not path or path.endswith('/') or path.endswith(os.path.sep):
         continue
 
       with self.assertRaises(error.ManifestInvalidPathError):
@@ -843,3 +876,27 @@ class ExtendProjectElementTests(ManifestParseTestCase):
     else:
       self.assertEqual(manifest.projects[0].relpath, 'bar')
       self.assertEqual(manifest.projects[1].relpath, 'y')
+
+  def test_extend_project_dest_branch(self):
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" dest-branch="foo" />
+  <project name="myproject" />
+  <extend-project name="myproject" dest-branch="bar" />
+</manifest>
+""")
+    self.assertEqual(len(manifest.projects), 1)
+    self.assertEqual(manifest.projects[0].dest_branch, 'bar')
+
+  def test_extend_project_upstream(self):
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="myproject" />
+  <extend-project name="myproject" upstream="bar" />
+</manifest>
+""")
+    self.assertEqual(len(manifest.projects), 1)
+    self.assertEqual(manifest.projects[0].upstream, 'bar')

tests/test_project.py

@@ -17,26 +17,23 @@
 import contextlib
 import os
 from pathlib import Path
-import shutil
 import subprocess
 import tempfile
 import unittest
 
 import error
+import manifest_xml
 import git_command
 import git_config
 import platform_utils
 import project
+import repo_trace
 
 
 @contextlib.contextmanager
 def TempGitTree():
   """Create a new empty git checkout for testing."""
-  # TODO(vapier): Convert this to tempfile.TemporaryDirectory once we drop
-  # Python 2 support entirely.
-  try:
-    tempdir = tempfile.mkdtemp(prefix='repo-tests')
-
+  with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
     # Tests need to assume, that main is default branch at init,
     # which is not supported in config until 2.28.
     cmd = ['git', 'init']
@@ -50,8 +47,6 @@ def TempGitTree():
       cmd += ['--template', templatedir]
     subprocess.check_call(cmd, cwd=tempdir)
     yield tempdir
-  finally:
-    platform_utils.rmtree(tempdir)
 
 
 class FakeProject(object):
@@ -71,6 +66,13 @@ class FakeProject(object):
 class ReviewableBranchTests(unittest.TestCase):
   """Check ReviewableBranch behavior."""
 
+  def setUp(self):
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    repo_trace._TRACE_FILE = os.path.join(self.tempdirobj.name, 'TRACE_FILE_from_test')
+
+  def tearDown(self):
+    self.tempdirobj.cleanup()
+
   def test_smoke(self):
     """A quick run through everything."""
     with TempGitTree() as tempdir:
@@ -124,14 +126,15 @@ class CopyLinkTestCase(unittest.TestCase):
   """
 
   def setUp(self):
-    self.tempdir = tempfile.mkdtemp(prefix='repo_tests')
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    self.tempdir = self.tempdirobj.name
     self.topdir = os.path.join(self.tempdir, 'checkout')
     self.worktree = os.path.join(self.topdir, 'git-project')
     os.makedirs(self.topdir)
     os.makedirs(self.worktree)
 
   def tearDown(self):
-    shutil.rmtree(self.tempdir, ignore_errors=True)
+    self.tempdirobj.cleanup()
 
   @staticmethod
   def touch(path):
@@ -382,7 +385,7 @@ class MigrateWorkTreeTests(unittest.TestCase):
 
       # Make sure the dir was transformed into a symlink.
       self.assertTrue(dotgit.is_symlink())
-      self.assertEqual(os.readlink(dotgit), '../../.repo/projects/src/test.git')
+      self.assertEqual(os.readlink(dotgit), os.path.normpath('../../.repo/projects/src/test.git'))
 
       # Make sure files were moved over.
       gitdir = tempdir / '.repo/projects/src/test.git'
@@ -409,3 +412,81 @@ class MigrateWorkTreeTests(unittest.TestCase):
         self.assertTrue((dotgit / name).is_file())
       for name in self._SYMLINKS:
         self.assertTrue((dotgit / name).is_symlink())
+
+
+class ManifestPropertiesFetchedCorrectly(unittest.TestCase):
+  """Ensure properties are fetched properly."""
+
+  def setUpManifest(self, tempdir):
+    repo_trace._TRACE_FILE = os.path.join(tempdir, 'TRACE_FILE_from_test')
+
+    repodir = os.path.join(tempdir, '.repo')
+    manifest_dir = os.path.join(repodir, 'manifests')
+    manifest_file = os.path.join(
+        repodir, manifest_xml.MANIFEST_FILE_NAME)
+    local_manifest_dir = os.path.join(
+        repodir, manifest_xml.LOCAL_MANIFESTS_DIR_NAME)
+    os.mkdir(repodir)
+    os.mkdir(manifest_dir)
+    manifest = manifest_xml.XmlManifest(repodir, manifest_file)
+
+    return project.ManifestProject(
+        manifest, 'test/manifest', os.path.join(tempdir, '.git'), tempdir)
+
+  def test_manifest_config_properties(self):
+    """Test we are fetching the manifest config properties correctly."""
+
+    with TempGitTree() as tempdir:
+      fakeproj = self.setUpManifest(tempdir)
+
+      # Set property using the expected Set method, then ensure
+      # the porperty functions are using the correct Get methods.
+      fakeproj.config.SetString(
+          'manifest.standalone', 'https://chicken/manifest.git')
+      self.assertEqual(
+          fakeproj.standalone_manifest_url, 'https://chicken/manifest.git')
+
+      fakeproj.config.SetString('manifest.groups', 'test-group, admin-group')
+      self.assertEqual(fakeproj.manifest_groups, 'test-group, admin-group')
+
+      fakeproj.config.SetString('repo.reference', 'mirror/ref')
+      self.assertEqual(fakeproj.reference, 'mirror/ref')
+
+      fakeproj.config.SetBoolean('repo.dissociate', False)
+      self.assertFalse(fakeproj.dissociate)
+
+      fakeproj.config.SetBoolean('repo.archive', False)
+      self.assertFalse(fakeproj.archive)
+
+      fakeproj.config.SetBoolean('repo.mirror', False)
+      self.assertFalse(fakeproj.mirror)
+
+      fakeproj.config.SetBoolean('repo.worktree', False)
+      self.assertFalse(fakeproj.use_worktree)
+
+      fakeproj.config.SetBoolean('repo.clonebundle', False)
+      self.assertFalse(fakeproj.clone_bundle)
+
+      fakeproj.config.SetBoolean('repo.submodules', False)
+      self.assertFalse(fakeproj.submodules)
+
+      fakeproj.config.SetBoolean('repo.git-lfs', False)
+      self.assertFalse(fakeproj.git_lfs)
+
+      fakeproj.config.SetBoolean('repo.superproject', False)
+      self.assertFalse(fakeproj.use_superproject)
+
+      fakeproj.config.SetBoolean('repo.partialclone', False)
+      self.assertFalse(fakeproj.partial_clone)
+
+      fakeproj.config.SetString('repo.depth', '48')
+      self.assertEqual(fakeproj.depth, '48')
+
+      fakeproj.config.SetString('repo.clonefilter', 'blob:limit=10M')
+      self.assertEqual(fakeproj.clone_filter, 'blob:limit=10M')
+
+      fakeproj.config.SetString('repo.partialcloneexclude', 'third_party/big_repo')
+      self.assertEqual(fakeproj.partial_clone_exclude, 'third_party/big_repo')
+
+      fakeproj.config.SetString('manifest.platform', 'auto')
+      self.assertEqual(fakeproj.manifest_platform, 'auto')

tests/test_subcmds_sync.py

@@ -11,27 +11,26 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 """Unittests for the subcmds/sync.py module."""
 
+import os
+import unittest
 from unittest import mock
 
 import pytest
 
+import command
 from subcmds import sync
 
 
-@pytest.mark.parametrize(
-  'use_superproject, cli_args, result',
-  [
+@pytest.mark.parametrize('use_superproject, cli_args, result', [
     (True, ['--current-branch'], True),
     (True, ['--no-current-branch'], True),
     (True, [], True),
     (False, ['--current-branch'], True),
     (False, ['--no-current-branch'], False),
     (False, [], None),
-  ]
-)
+])
 def test_get_current_branch_only(use_superproject, cli_args, result):
   """Test Sync._GetCurrentBranchOnly logic.
 
@@ -41,5 +40,94 @@ def test_get_current_branch_only(use_superproject, cli_args, result):
   cmd = sync.Sync()
   opts, _ = cmd.OptionParser.parse_args(cli_args)
 
-  with mock.patch('git_superproject.UseSuperproject', return_value=use_superproject):
+  with mock.patch('git_superproject.UseSuperproject',
+                  return_value=use_superproject):
     assert cmd._GetCurrentBranchOnly(opts, cmd.manifest) == result
+
+
+# Used to patch os.cpu_count() for reliable results.
+OS_CPU_COUNT = 24
+
+@pytest.mark.parametrize('argv, jobs_manifest, jobs, jobs_net, jobs_check', [
+  # No user or manifest settings.
+  ([], None, OS_CPU_COUNT, 1, command.DEFAULT_LOCAL_JOBS),
+  # No user settings, so manifest settings control.
+  ([], 3, 3, 3, 3),
+  # User settings, but no manifest.
+  (['--jobs=4'], None, 4, 4, 4),
+  (['--jobs=4', '--jobs-network=5'], None, 4, 5, 4),
+  (['--jobs=4', '--jobs-checkout=6'], None, 4, 4, 6),
+  (['--jobs=4', '--jobs-network=5', '--jobs-checkout=6'], None, 4, 5, 6),
+  (['--jobs-network=5'], None, OS_CPU_COUNT, 5, command.DEFAULT_LOCAL_JOBS),
+  (['--jobs-checkout=6'], None, OS_CPU_COUNT, 1, 6),
+  (['--jobs-network=5', '--jobs-checkout=6'], None, OS_CPU_COUNT, 5, 6),
+  # User settings with manifest settings.
+  (['--jobs=4'], 3, 4, 4, 4),
+  (['--jobs=4', '--jobs-network=5'], 3, 4, 5, 4),
+  (['--jobs=4', '--jobs-checkout=6'], 3, 4, 4, 6),
+  (['--jobs=4', '--jobs-network=5', '--jobs-checkout=6'], 3, 4, 5, 6),
+  (['--jobs-network=5'], 3, 3, 5, 3),
+  (['--jobs-checkout=6'], 3, 3, 3, 6),
+  (['--jobs-network=5', '--jobs-checkout=6'], 3, 3, 5, 6),
+  # Settings that exceed rlimits get capped.
+  (['--jobs=1000000'], None, 83, 83, 83),
+  ([], 1000000, 83, 83, 83),
+])
+def test_cli_jobs(argv, jobs_manifest, jobs, jobs_net, jobs_check):
+  """Tests --jobs option behavior."""
+  mp = mock.MagicMock()
+  mp.manifest.default.sync_j = jobs_manifest
+
+  cmd = sync.Sync()
+  opts, args = cmd.OptionParser.parse_args(argv)
+  cmd.ValidateOptions(opts, args)
+
+  with mock.patch.object(sync, '_rlimit_nofile', return_value=(256, 256)):
+    with mock.patch.object(os, 'cpu_count', return_value=OS_CPU_COUNT):
+      cmd._ValidateOptionsWithManifest(opts, mp)
+      assert opts.jobs == jobs
+      assert opts.jobs_network == jobs_net
+      assert opts.jobs_checkout == jobs_check
+
+
+class GetPreciousObjectsState(unittest.TestCase):
+  """Tests for _GetPreciousObjectsState."""
+
+  def setUp(self):
+    """Common setup."""
+    self.cmd = sync.Sync()
+    self.project = p = mock.MagicMock(use_git_worktrees=False,
+                                      UseAlternates=False)
+    p.manifest.GetProjectsWithName.return_value = [p]
+
+    self.opt = mock.Mock(spec_set=['this_manifest_only'])
+    self.opt.this_manifest_only = False
+
+  def test_worktrees(self):
+    """False for worktrees."""
+    self.project.use_git_worktrees = True
+    self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt))
+
+  def test_not_shared(self):
+    """Singleton project."""
+    self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt))
+
+  def test_shared(self):
+    """Shared project."""
+    self.project.manifest.GetProjectsWithName.return_value = [
+        self.project, self.project
+    ]
+    self.assertTrue(self.cmd._GetPreciousObjectsState(self.project, self.opt))
+
+  def test_shared_with_alternates(self):
+    """Shared project, with alternates."""
+    self.project.manifest.GetProjectsWithName.return_value = [
+        self.project, self.project
+    ]
+    self.project.UseAlternates = True
+    self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt))
+
+  def test_not_found(self):
+    """Project not found in manifest."""
+    self.project.manifest.GetProjectsWithName.return_value = []
+    self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt))

tests/test_update_manpages.py

@@ -0,0 +1,27 @@
+# Copyright 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for the update_manpages module."""
+
+import unittest
+import tests.update_manpages as um
+
+
+class UpdateManpagesTest(unittest.TestCase):
+  """Tests the update-manpages code."""
+
+  def test_replace_regex(self):
+    """Check that replace_regex works."""
+    data = '\n\033[1mSummary\033[m\n'
+    self.assertEqual(um.replace_regex(data),'\nSummary\n')

tests/test_wrapper.py

@@ -14,11 +14,9 @@
 
 """Unittests for the wrapper.py module."""
 
-import contextlib
 from io import StringIO
 import os
 import re
-import shutil
 import sys
 import tempfile
 import unittest
@@ -26,22 +24,9 @@ from unittest import mock
 
 import git_command
 import main
-import platform_utils
 import wrapper
 
 
-@contextlib.contextmanager
-def TemporaryDirectory():
-  """Create a new empty git checkout for testing."""
-  # TODO(vapier): Convert this to tempfile.TemporaryDirectory once we drop
-  # Python 2 support entirely.
-  try:
-    tempdir = tempfile.mkdtemp(prefix='repo-tests')
-    yield tempdir
-  finally:
-    platform_utils.rmtree(tempdir)
-
-
 def fixture(*paths):
   """Return a path relative to tests/fixtures.
   """
@@ -74,12 +59,12 @@ class RepoWrapperUnitTest(RepoWrapperTestCase):
   def test_python_constraints(self):
     """The launcher should never require newer than main.py."""
     self.assertGreaterEqual(main.MIN_PYTHON_VERSION_HARD,
-                            wrapper.MIN_PYTHON_VERSION_HARD)
+                            self.wrapper.MIN_PYTHON_VERSION_HARD)
     self.assertGreaterEqual(main.MIN_PYTHON_VERSION_SOFT,
-                            wrapper.MIN_PYTHON_VERSION_SOFT)
+                            self.wrapper.MIN_PYTHON_VERSION_SOFT)
     # Make sure the versions are themselves in sync.
-    self.assertGreaterEqual(wrapper.MIN_PYTHON_VERSION_SOFT,
-                            wrapper.MIN_PYTHON_VERSION_HARD)
+    self.assertGreaterEqual(self.wrapper.MIN_PYTHON_VERSION_SOFT,
+                            self.wrapper.MIN_PYTHON_VERSION_HARD)
 
   def test_init_parser(self):
     """Make sure 'init' GetParser works."""
@@ -174,7 +159,9 @@ class RunCommand(RepoWrapperTestCase):
   def test_capture(self):
     """Check capture_output handling."""
     ret = self.wrapper.run_command(['echo', 'hi'], capture_output=True)
-    self.assertEqual(ret.stdout, 'hi\n')
+    # echo command appends OS specific linesep, but on Windows + Git Bash
+    # we get UNIX ending, so we allow both.
+    self.assertIn(ret.stdout, ['hi' + os.linesep, 'hi\n'])
 
   def test_check(self):
     """Check check handling."""
@@ -336,19 +323,19 @@ class NeedSetupGnuPG(RepoWrapperTestCase):
 
   def test_missing_dir(self):
     """The ~/.repoconfig tree doesn't exist yet."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = os.path.join(tempdir, 'foo')
       self.assertTrue(self.wrapper.NeedSetupGnuPG())
 
   def test_missing_keyring(self):
     """The keyring-version file doesn't exist yet."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       self.assertTrue(self.wrapper.NeedSetupGnuPG())
 
   def test_empty_keyring(self):
     """The keyring-version file exists, but is empty."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       with open(os.path.join(tempdir, 'keyring-version'), 'w'):
         pass
@@ -356,7 +343,7 @@ class NeedSetupGnuPG(RepoWrapperTestCase):
 
   def test_old_keyring(self):
     """The keyring-version file exists, but it's old."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       with open(os.path.join(tempdir, 'keyring-version'), 'w') as fp:
         fp.write('1.0\n')
@@ -364,7 +351,7 @@ class NeedSetupGnuPG(RepoWrapperTestCase):
 
   def test_new_keyring(self):
     """The keyring-version file exists, and is up-to-date."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       with open(os.path.join(tempdir, 'keyring-version'), 'w') as fp:
         fp.write('1000.0\n')
@@ -376,7 +363,7 @@ class SetupGnuPG(RepoWrapperTestCase):
 
   def test_full(self):
     """Make sure it works completely."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       self.wrapper.gpg_dir = os.path.join(self.wrapper.home_dot_repo, 'gnupg')
       self.assertTrue(self.wrapper.SetupGnuPG(True))
@@ -426,7 +413,8 @@ class GitCheckoutTestCase(RepoWrapperTestCase):
   @classmethod
   def setUpClass(cls):
     # Create a repo to operate on, but do it once per-class.
-    cls.GIT_DIR = tempfile.mkdtemp(prefix='repo-rev-tests')
+    cls.tempdirobj = tempfile.TemporaryDirectory(prefix='repo-rev-tests')
+    cls.GIT_DIR = cls.tempdirobj.name
     run_git = wrapper.Wrapper().run_git
 
     remote = os.path.join(cls.GIT_DIR, 'remote')
@@ -455,10 +443,10 @@ class GitCheckoutTestCase(RepoWrapperTestCase):
 
   @classmethod
   def tearDownClass(cls):
-    if not cls.GIT_DIR:
+    if not cls.tempdirobj:
       return
 
-    shutil.rmtree(cls.GIT_DIR)
+    cls.tempdirobj.cleanup()
 
 
 class ResolveRepoRev(GitCheckoutTestCase):
@@ -470,7 +458,7 @@ class ResolveRepoRev(GitCheckoutTestCase):
     self.assertEqual('refs/heads/stable', rrev)
     self.assertEqual(self.REV_LIST[1], lrev)
 
-    with self.assertRaises(wrapper.CloneFailure):
+    with self.assertRaises(self.wrapper.CloneFailure):
       self.wrapper.resolve_repo_rev(self.GIT_DIR, 'refs/heads/unknown')
 
   def test_explicit_tag(self):
@@ -479,7 +467,7 @@ class ResolveRepoRev(GitCheckoutTestCase):
     self.assertEqual('refs/tags/v1.0', rrev)
     self.assertEqual(self.REV_LIST[1], lrev)
 
-    with self.assertRaises(wrapper.CloneFailure):
+    with self.assertRaises(self.wrapper.CloneFailure):
       self.wrapper.resolve_repo_rev(self.GIT_DIR, 'refs/tags/unknown')
 
   def test_branch_name(self):
@@ -514,7 +502,7 @@ class ResolveRepoRev(GitCheckoutTestCase):
 
   def test_unknown(self):
     """Check unknown ref/commit argument."""
-    with self.assertRaises(wrapper.CloneFailure):
+    with self.assertRaises(self.wrapper.CloneFailure):
       self.wrapper.resolve_repo_rev(self.GIT_DIR, 'boooooooya')
 
 
@@ -565,7 +553,3 @@ class CheckRepoRev(GitCheckoutTestCase):
       rrev, lrev = self.wrapper.check_repo_rev(self.GIT_DIR, 'stable', repo_verify=False)
     self.assertEqual('refs/heads/stable', rrev)
     self.assertEqual(self.REV_LIST[1], lrev)
-
-
-if __name__ == '__main__':
-  unittest.main()

tests/update_manpages.py

@@ -0,0 +1 @@
+../release/update-manpages

tox.ini

@@ -15,7 +15,7 @@
 # https://tox.readthedocs.io/
 
 [tox]
-envlist = py36, py37, py38, py39
+envlist = py36, py37, py38, py39, py310
 
 [gh-actions]
 python =
@@ -23,11 +23,17 @@ python =
     3.7: py37
     3.8: py38
     3.9: py39
+    3.10: py310
 
 [testenv]
-deps = pytest
-commands = {envpython} run_tests
+deps =
+    pytest
+    pytest-timeout
+commands = {envpython} run_tests {posargs}
 setenv =
     GIT_AUTHOR_NAME = Repo test author
     GIT_COMMITTER_NAME = Repo test committer
     EMAIL = repo@gerrit.nodomain
+
+[pytest]
+timeout = 300

wrapper.py

@@ -12,12 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-try:
-  from importlib.machinery import SourceFileLoader
-  _loader = lambda *args: SourceFileLoader(*args).load_module()
-except ImportError:
-  import imp
-  _loader = lambda *args: imp.load_source(*args)
+import importlib.machinery
+import importlib.util
 import os
 
 
@@ -31,5 +27,9 @@ _wrapper_module = None
 def Wrapper():
   global _wrapper_module
   if not _wrapper_module:
-    _wrapper_module = _loader('wrapper', WrapperPath())
+    modname = 'wrapper'
+    loader = importlib.machinery.SourceFileLoader(modname, WrapperPath())
+    spec = importlib.util.spec_from_loader(modname, loader)
+    _wrapper_module = importlib.util.module_from_spec(spec)
+    loader.exec_module(_wrapper_module)
   return _wrapper_module