upload: Skip upload if merge branch doesn't match project revision and

dest_branch.

- This still prevents the case mentioned here:
https://gerrit-review.googlesource.com/c/50300
while also supporting dest_branch.
- Update _GetMergeBranch to get merge branches for any branch, not just
the one we happen to run `repo upload` in. (e.g. for uploading multiple
branches)

Bug: b/27955930
Change-Id: Ia8ee1d6a83a783c984bb2eb308bb11b3a721a95d
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/360794
Commit-Queue: Joanna Wang <jojwang@google.com>
Reviewed-by: Mike Frysinger <vapier@google.com>
Tested-by: Joanna Wang <jojwang@google.com>

.github/workflows/flake8-postsubmit.yml

@@ -0,0 +1,23 @@
+# GitHub actions workflow.
+# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/workflow-syntax-for-github-actions
+# https://github.com/marketplace/actions/python-flake8
+
+name: Flake8
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  lint:
+    name: Python Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Run flake8
+        uses: julianwachholz/flake8-action@v2
+        with:
+          checkName: "Python Lint"

.github/workflows/test-ci.yml

@@ -14,18 +14,18 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10']
     runs-on: ${{ matrix.os }}
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install tox tox-gh-actions
+        python -m pip install tox tox-gh-actions
     - name: Test with tox
       run: tox

.gitreview

@@ -0,0 +1,5 @@
+[gerrit]
+host=gerrit-review.googlesource.com
+scheme=https
+project=git-repo.git
+defaultbranch=main

README.md

@@ -8,7 +8,7 @@ that you can put anywhere in your path.
 
 * Homepage: <https://gerrit.googlesource.com/git-repo/>
 * Mailing list: [repo-discuss on Google Groups][repo-discuss]
-* Bug reports: <https://bugs.chromium.org/p/gerrit/issues/list?q=component:repo>
+* Bug reports: <https://bugs.chromium.org/p/gerrit/issues/list?q=component:Applications%3Erepo>
 * Source: <https://gerrit.googlesource.com/git-repo/>
 * Overview: <https://source.android.com/source/developing.html>
 * Docs: <https://source.android.com/source/using-repo.html>
@@ -51,5 +51,5 @@ $ chmod a+rx ~/.bin/repo
 
 
 [new-bug]: https://bugs.chromium.org/p/gerrit/issues/entry?template=Repo+tool+issue
-[issue tracker]: https://bugs.chromium.org/p/gerrit/issues/list?q=component:repo
+[issue tracker]: https://bugs.chromium.org/p/gerrit/issues/list?q=component:Applications%3Erepo
 [repo-discuss]: https://groups.google.com/forum/#!forum/repo-discuss

SUBMITTING_PATCHES.md

@@ -3,7 +3,7 @@
 # Short Version
 
  - Make small logical changes.
- - Provide a meaningful commit message.
+ - [Provide a meaningful commit message][commit-message-style].
  - Check for coding errors and style nits with flake8.
  - Make sure all code is under the Apache License, 2.0.
  - Publish your changes for review.
@@ -26,10 +26,11 @@ yourself with the following relevant bits.
 
 ## Make separate commits for logically separate changes.
 
-Unless your patch is really trivial, you should not be sending
-out a patch that was generated between your working tree and your
-commit head.  Instead, always make a commit with complete commit
-message and generate a series of patches from your repository.
+Unless your patch is really trivial, you should not be sending out a patch that
+was generated between your working tree and your commit head.
+Instead, always make a commit with a complete
+[commit message][commit-message-style] and generate a series of patches from
+your repository.
 It is a good discipline.
 
 Describe the technical detail of the change(s).
@@ -171,3 +172,6 @@ After you receive a Code-Review+2 from the maintainer, select the Verified
 button on the gerrit page for the change. This verifies that you have tested
 your changes and notifies the maintainer that they are ready to be submitted.
 The maintainer will then submit your changes to the repository.
+
+
+[commit-message-style]: https://chris.beams.io/posts/git-commit/

command.py

@@ -61,13 +61,21 @@ class Command(object):
   # it is the number of parallel jobs to default to.
   PARALLEL_JOBS = None
 
+  # Whether this command supports Multi-manifest.  If False, then main.py will
+  # iterate over the manifests and invoke the command once per (sub)manifest.
+  # This is only checked after calling ValidateOptions, so that partially
+  # migrated subcommands can set it to False.
+  MULTI_MANIFEST_SUPPORT = True
+
   def __init__(self, repodir=None, client=None, manifest=None, gitc_manifest=None,
-               git_event_log=None):
+               git_event_log=None, outer_client=None, outer_manifest=None):
     self.repodir = repodir
     self.client = client
+    self.outer_client = outer_client or client
     self.manifest = manifest
     self.gitc_manifest = gitc_manifest
     self.git_event_log = git_event_log
+    self.outer_manifest = outer_manifest
 
     # Cache for the OptionParser property.
     self._optparse = None
@@ -135,6 +143,17 @@ class Command(object):
           type=int, default=self.PARALLEL_JOBS,
           help=f'number of jobs to run in parallel (default: {default})')
 
+    m = p.add_option_group('Multi-manifest options')
+    m.add_option('--outer-manifest', action='store_true', default=None,
+                 help='operate starting at the outermost manifest')
+    m.add_option('--no-outer-manifest', dest='outer_manifest',
+                 action='store_false', help='do not operate on outer manifests')
+    m.add_option('--this-manifest-only', action='store_true', default=None,
+                 help='only operate on this (sub)manifest')
+    m.add_option('--no-this-manifest-only', '--all-manifests',
+                 dest='this_manifest_only', action='store_false',
+                 help='operate on this manifest and its submanifests')
+
   def _Options(self, p):
     """Initialize the option parser with subcommand-specific options."""
 
@@ -166,6 +185,10 @@ class Command(object):
     """Validate common options."""
     opt.quiet = opt.output_mode is False
     opt.verbose = opt.output_mode is True
+    if opt.outer_manifest is None:
+      # By default, treat multi-manifest instances as a single manifest from
+      # the user's perspective.
+      opt.outer_manifest = True
 
   def ValidateOptions(self, opt, args):
     """Validate the user options & arguments before executing.
@@ -252,15 +275,30 @@ class Command(object):
     return project
 
   def GetProjects(self, args, manifest=None, groups='', missing_ok=False,
-                  submodules_ok=False):
+                  submodules_ok=False, all_manifests=False):
     """A list of projects that match the arguments.
-    """
-    if not manifest:
-      manifest = self.manifest
-    all_projects_list = manifest.projects
-    result = []
 
-    mp = manifest.manifestProject
+    Args:
+      args: a list of (case-insensitive) strings, projects to search for.
+      manifest: an XmlManifest, the manifest to use, or None for default.
+      groups: a string, the manifest groups in use.
+      missing_ok: a boolean, whether to allow missing projects.
+      submodules_ok: a boolean, whether to allow submodules.
+      all_manifests: a boolean, if True then all manifests and submanifests are
+                     used.  If False, then only the local (sub)manifest is used.
+
+    Returns:
+      A list of matching Project instances.
+    """
+    if all_manifests:
+      if not manifest:
+        manifest = self.manifest.outer_client
+      all_projects_list = manifest.all_projects
+    else:
+      if not manifest:
+        manifest = self.manifest
+      all_projects_list = manifest.projects
+    result = []
 
     if not groups:
       groups = manifest.GetGroupsStr()
@@ -282,12 +320,20 @@ class Command(object):
       for arg in args:
         # We have to filter by manifest groups in case the requested project is
         # checked out multiple times or differently based on them.
-        projects = [project for project in manifest.GetProjectsWithName(arg)
+        projects = [project
+                    for project in manifest.GetProjectsWithName(
+                        arg, all_manifests=all_manifests)
                     if project.MatchesGroups(groups)]
 
         if not projects:
           path = os.path.abspath(arg).replace('\\', '/')
-          project = self._GetProjectByPath(manifest, path)
+          tree = manifest
+          if all_manifests:
+            # Look for the deepest matching submanifest.
+            for tree in reversed(list(manifest.all_manifests)):
+              if path.startswith(tree.topdir):
+                break
+          project = self._GetProjectByPath(tree, path)
 
           # If it's not a derived project, update path->project mapping and
           # search again, as arg might actually point to a derived subproject.
@@ -308,7 +354,8 @@ class Command(object):
 
         for project in projects:
           if not missing_ok and not project.Exists:
-            raise NoSuchProjectError('%s (%s)' % (arg, project.relpath))
+            raise NoSuchProjectError('%s (%s)' % (
+                arg, project.RelPath(local=not all_manifests)))
           if not project.MatchesGroups(groups):
             raise InvalidProjectGroupsError(arg)
 
@@ -319,12 +366,22 @@ class Command(object):
     result.sort(key=_getpath)
     return result
 
-  def FindProjects(self, args, inverse=False):
+  def FindProjects(self, args, inverse=False, all_manifests=False):
+    """Find projects from command line arguments.
+
+    Args:
+      args: a list of (case-insensitive) strings, projects to search for.
+      inverse: a boolean, if True, then projects not matching any |args| are
+               returned.
+      all_manifests: a boolean, if True then all manifests and submanifests are
+                     used.  If False, then only the local (sub)manifest is used.
+    """
     result = []
     patterns = [re.compile(r'%s' % a, re.IGNORECASE) for a in args]
-    for project in self.GetProjects(''):
+    for project in self.GetProjects('', all_manifests=all_manifests):
+      paths = [project.name, project.RelPath(local=not all_manifests)]
       for pattern in patterns:
-        match = pattern.search(project.name) or pattern.search(project.relpath)
+        match = any(pattern.search(x) for x in paths)
         if not inverse and match:
           result.append(project)
           break
@@ -333,9 +390,24 @@ class Command(object):
       else:
         if inverse:
           result.append(project)
-    result.sort(key=lambda project: project.relpath)
+    result.sort(key=lambda project: (project.manifest.path_prefix,
+                                     project.relpath))
     return result
 
+  def ManifestList(self, opt):
+    """Yields all of the manifests to traverse.
+
+    Args:
+      opt: The command options.
+    """
+    top = self.outer_manifest
+    if not opt.outer_manifest or opt.this_manifest_only:
+      top = self.manifest
+    yield top
+    if not opt.this_manifest_only:
+      for child in top.all_children:
+        yield child
+
 
 class InteractiveCommand(Command):
   """Command which requires user interaction on the tty and

docs/internal-fs-layout.md

@@ -50,6 +50,10 @@ For example, if you want to change the manifest branch, you can simply run
 For more documentation on the manifest format, including the local_manifests
 support, see the [manifest-format.md] file.
 
+*   `submanifests/{submanifest.path}/`: The path prefix to the manifest state of
+    a submanifest included in a multi-manifest checkout.  The outermost manifest
+    manifest state is found adjacent to `submanifests/`.
+
 *   `manifests/`: A git checkout of the manifest project.  Its `.git/` state
     points to the `manifest.git` bare checkout (see below).  It tracks the git
     branch specified at `repo init` time via `--manifest-branch`.
@@ -163,6 +167,7 @@ User controlled settings are initialized when running `repo init`.
 | repo.clonefilter         | `--clone-filter`          | Filter setting when using [partial git clones] |
 | repo.depth               | `--depth`                 | Create shallow checkouts when cloning |
 | repo.dissociate          | `--dissociate`            | Dissociate from any reference/mirrors after initial clone |
+| repo.git-lfs             | `--git-lfs`               | Enable [Git LFS] support |
 | repo.mirror              | `--mirror`                | Checkout is a repo mirror |
 | repo.partialclone        | `--partial-clone`         | Create [partial git clones] |
 | repo.partialcloneexclude | `--partial-clone-exclude` | Comma-delimited list of project names (not paths) to exclude while using [partial git clones] |
@@ -217,27 +222,30 @@ The `[remote]` settings are automatically populated/updated from the manifest.
 
 The `[branch]` settings are updated by `repo start` and `git branch`.
 
-| Setting                       | Subcommands   | Use/Meaning |
-|-------------------------------|---------------|-------------|
-| review.\<url\>.autocopy       | upload        | Automatically add to `--cc=<value>` |
-| review.\<url\>.autoreviewer   | upload        | Automatically add to `--reviewers=<value>` |
-| review.\<url\>.autoupload     | upload        | Automatically answer "yes" or "no" to all prompts |
-| review.\<url\>.uploadhashtags | upload        | Automatically add to `--hashtag=<value>` |
-| review.\<url\>.uploadlabels   | upload        | Automatically add to `--label=<value>` |
-| review.\<url\>.uploadnotify   | upload        | [Notify setting][upload-notify] to use |
-| review.\<url\>.uploadtopic    | upload        | Default [topic] to use |
-| review.\<url\>.username       | upload        | Override username with `ssh://` review URIs |
-| remote.\<remote\>.fetch       | sync          | Set of refs to fetch |
-| remote.\<remote\>.projectname | \<network\>   | The name of the project as it exists in Gerrit review |
-| remote.\<remote\>.pushurl     | upload        | The base URI for pushing CLs |
-| remote.\<remote\>.review      | upload        | The URI of the Gerrit review server |
-| remote.\<remote\>.url         | sync & upload | The URI of the git project to fetch |
-| branch.\<branch\>.merge       | sync & upload | The branch to merge & upload & track |
-| branch.\<branch\>.remote      | sync & upload | The remote to track |
+| Setting                               | Subcommands   | Use/Meaning |
+|---------------------------------------|---------------|-------------|
+| review.\<url\>.autocopy               | upload        | Automatically add to `--cc=<value>` |
+| review.\<url\>.autoreviewer           | upload        | Automatically add to `--reviewers=<value>` |
+| review.\<url\>.autoupload             | upload        | Automatically answer "yes" or "no" to all prompts |
+| review.\<url\>.uploadhashtags         | upload        | Automatically add to `--hashtag=<value>` |
+| review.\<url\>.uploadlabels           | upload        | Automatically add to `--label=<value>` |
+| review.\<url\>.uploadnotify           | upload        | [Notify setting][upload-notify] to use |
+| review.\<url\>.uploadtopic            | upload        | Default [topic] to use |
+| review.\<url\>.uploadwarningthreshold | upload        | Warn when attempting to upload more than this many CLs |
+| review.\<url\>.username               | upload        | Override username with `ssh://` review URIs |
+| remote.\<remote\>.fetch               | sync          | Set of refs to fetch |
+| remote.\<remote\>.projectname         | \<network\>   | The name of the project as it exists in Gerrit review |
+| remote.\<remote\>.pushurl             | upload        | The base URI for pushing CLs |
+| remote.\<remote\>.review              | upload        | The URI of the Gerrit review server |
+| remote.\<remote\>.url                 | sync & upload | The URI of the git project to fetch |
+| branch.\<branch\>.merge               | sync & upload | The branch to merge & upload & track |
+| branch.\<branch\>.remote              | sync & upload | The remote to track |
 
 ## ~/ dotconfig layout
 
-Repo will create & maintain a few files in the user's home directory.
+Repo will create & maintain a few files under the `.repoconfig/` directory.
+This is placed in the user's home directory by default but can be changed by
+setting `REPO_CONFIG_DIR`.
 
 *   `.repoconfig/`: Repo's per-user directory for all random config files/state.
 *   `.repoconfig/config`: Per-user settings using [git-config] file format.
@@ -254,6 +262,7 @@ Repo will create & maintain a few files in the user's home directory.
 
 
 [git-config]: https://git-scm.com/docs/git-config
+[Git LFS]: https://git-lfs.github.com/
 [git worktree]: https://git-scm.com/docs/git-worktree
 [gitsubmodules]: https://git-scm.com/docs/gitsubmodules
 [manifest-format.md]: ./manifest-format.md

docs/manifest-format.md

@@ -26,6 +26,7 @@ following DTD:
                       remote*,
                       default?,
                       manifest-server?,
+                      submanifest*?,
                       remove-project*,
                       project*,
                       extend-project*,
@@ -57,6 +58,16 @@ following DTD:
   <!ELEMENT manifest-server EMPTY>
   <!ATTLIST manifest-server url CDATA #REQUIRED>
 
+  <!ELEMENT submanifest EMPTY>
+  <!ATTLIST submanifest name           ID #REQUIRED>
+  <!ATTLIST submanifest remote         IDREF #IMPLIED>
+  <!ATTLIST submanifest project        CDATA #IMPLIED>
+  <!ATTLIST submanifest manifest-name  CDATA #IMPLIED>
+  <!ATTLIST submanifest revision       CDATA #IMPLIED>
+  <!ATTLIST submanifest path           CDATA #IMPLIED>
+  <!ATTLIST submanifest groups         CDATA #IMPLIED>
+  <!ATTLIST submanifest default-groups CDATA #IMPLIED>
+
   <!ELEMENT project (annotation*,
                      project*,
                      copyfile*,
@@ -94,6 +105,8 @@ following DTD:
   <!ATTLIST extend-project groups CDATA #IMPLIED>
   <!ATTLIST extend-project revision CDATA #IMPLIED>
   <!ATTLIST extend-project remote CDATA #IMPLIED>
+  <!ATTLIST extend-project dest-branch CDATA #IMPLIED>
+  <!ATTLIST extend-project upstream CDATA #IMPLIED>
 
   <!ELEMENT remove-project EMPTY>
   <!ATTLIST remove-project name  CDATA #REQUIRED>
@@ -236,6 +249,66 @@ the specified tag. This is used by repo sync when the --smart-tag option
 is given.
 
 
+### Element submanifest
+
+One or more submanifest elements may be specified.  Each element describes a
+single manifest to be checked out as a child.
+
+Attribute `name`: A unique name (within the current (sub)manifest) for this
+submanifest. It acts as a default for `revision` below.  The same name can be
+used for submanifests with different parent (sub)manifests.
+
+Attribute `remote`: Name of a previously defined remote element.
+If not supplied the remote given by the default element is used.
+
+Attribute `project`: The manifest project name.  The project's name is appended
+onto its remote's fetch URL to generate the actual URL to configure the Git
+remote with.  The URL gets formed as:
+
+    ${remote_fetch}/${project_name}.git
+
+where ${remote_fetch} is the remote's fetch attribute and
+${project_name} is the project's name attribute.  The suffix ".git"
+is always appended as repo assumes the upstream is a forest of
+bare Git repositories.  If the project has a parent element, its
+name will be prefixed by the parent's.
+
+The project name must match the name Gerrit knows, if Gerrit is
+being used for code reviews.
+
+`project` must not be empty, and may not be an absolute path or use "." or ".."
+path components.  It is always interpreted relative to the remote's fetch
+settings, so if a different base path is needed, declare a different remote
+with the new settings needed.
+
+If not supplied the remote and project for this manifest will be used: `remote`
+cannot be supplied.
+
+Projects from a submanifest and its submanifests are added to the
+submanifest::path:<path_prefix> group.
+
+Attribute `manifest-name`: The manifest filename in the manifest project.  If
+not supplied, `default.xml` is used.
+
+Attribute `revision`: Name of a Git branch (e.g. "main" or "refs/heads/main"),
+tag (e.g. "refs/tags/stable"), or a commit hash.  If not supplied, `name` is
+used.
+
+Attribute `path`: An optional path relative to the top directory
+of the repo client where the submanifest repo client top directory
+should be placed.  If not supplied, `revision` is used.
+
+`path` may not be an absolute path or use "." or ".." path components.
+
+Attribute `groups`: List of additional groups to which all projects
+in the included submanifest belong. This appends and recurses, meaning
+all projects in submanifests carry all parent submanifest groups.
+Same syntax as the corresponding element of `project`.
+
+Attribute `default-groups`: The list of manifest groups to sync if no
+`--groups=` parameter was specified at init.  When that list is empty, use this
+list instead of "default" as the list of groups to sync.
+
 ### Element project
 
 One or more project elements may be specified.  Each element
@@ -352,6 +425,12 @@ project.  Same syntax as the corresponding element of `project`.
 Attribute `remote`: If specified, overrides the remote of the original
 project.  Same syntax as the corresponding element of `project`.
 
+Attribute `dest-branch`: If specified, overrides the dest-branch of the original
+project.  Same syntax as the corresponding element of `project`.
+
+Attribute `upstream`: If specified, overrides the upstream of the original
+project.  Same syntax as the corresponding element of `project`.
+
 ### Element annotation
 
 Zero or more annotation elements may be specified as children of a
@@ -471,7 +550,7 @@ These restrictions are not enforced for [Local Manifests].
 
 Attribute `groups`: List of additional groups to which all projects
 in the included manifest belong. This appends and recurses, meaning
-all projects in sub-manifests carry all parent include groups.
+all projects in included manifests carry all parent include groups.
 Same syntax as the corresponding element of `project`.
 
 ## Local Manifests {#local-manifests}

docs/release-process.md

@@ -143,23 +143,14 @@ internal processes for accessing the restricted keys.
 ***
 
 ```sh
-# Set the gpg key directory.
-$ export GNUPGHOME=~/.gnupg/repo/
-
-# Verify the listed key is “Repo Maintainer”.
-$ gpg -K
-
-# Pick whatever branch or commit you want to tag.
-$ r=main
-
 # Pick the new version.
-$ t=1.12.10
+$ t=v2.30
 
-# Create the signed tag.
-$ git tag -s v$t -u "Repo Maintainer <repo@android.kernel.org>" -m "repo $t" $r
+# Create a new signed tag with the current HEAD.
+$ ./release/sign-tag.py $t
 
 # Verify the signed tag.
-$ git show v$t
+$ git show $t
 ```
 
 ### Push the new release
@@ -168,11 +159,11 @@ Once you're ready to make the release available to everyone, push it to the
 `stable` branch.
 
 Make sure you never push the tag itself to the stable branch!
-Only push the commit -- notice the use of `$t` and `$r` below.
+Only push the commit -- note the use of `^0` below.
 
 ```sh
-$ git push https://gerrit-review.googlesource.com/git-repo v$t
-$ git push https://gerrit-review.googlesource.com/git-repo $r:stable
+$ git push https://gerrit-review.googlesource.com/git-repo $t
+$ git push https://gerrit-review.googlesource.com/git-repo $t^0:stable
 ```
 
 If something goes horribly wrong, you can force push the previous version to the
@@ -195,7 +186,9 @@ You can create a short changelog using the command:
 ```sh
 # If you haven't pushed to the stable branch yet, you can use origin/stable.
 # If you have pushed, change origin/stable to the previous release tag.
-$ git log --format="%h (%aN) %s" --no-merges origin/stable..$r
+# This assumes "main" is the current tagged release.  If it's newer, change it
+# to the current release tag too.
+$ git log --format="%h (%aN) %s" --no-merges origin/stable..main
 ```
 
 ## Project References

git_command.py

@@ -16,6 +16,7 @@ import functools
 import os
 import sys
 import subprocess
+from typing import Any, Optional
 
 from error import GitError
 from git_refs import HEAD
@@ -157,7 +158,55 @@ def git_require(min_version, fail=False, msg=''):
   return False
 
 
+def _build_env(
+    _kwargs_only=(),
+    bare: Optional[bool] = False,
+    disable_editor: Optional[bool] = False,
+    ssh_proxy: Optional[Any] = None,
+    gitdir: Optional[str] = None,
+    objdir: Optional[str] = None
+):
+  """Constucts an env dict for command execution."""
+
+  assert _kwargs_only == (), '_build_env only accepts keyword arguments.'
+
+  env = GitCommand._GetBasicEnv()
+
+  if disable_editor:
+    env['GIT_EDITOR'] = ':'
+  if ssh_proxy:
+    env['REPO_SSH_SOCK'] = ssh_proxy.sock()
+    env['GIT_SSH'] = ssh_proxy.proxy
+    env['GIT_SSH_VARIANT'] = 'ssh'
+  if 'http_proxy' in env and 'darwin' == sys.platform:
+    s = "'http.proxy=%s'" % (env['http_proxy'],)
+    p = env.get('GIT_CONFIG_PARAMETERS')
+    if p is not None:
+      s = p + ' ' + s
+    env['GIT_CONFIG_PARAMETERS'] = s
+  if 'GIT_ALLOW_PROTOCOL' not in env:
+    env['GIT_ALLOW_PROTOCOL'] = (
+        'file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc')
+  env['GIT_HTTP_USER_AGENT'] = user_agent.git
+
+  if objdir:
+    # Set to the place we want to save the objects.
+    env['GIT_OBJECT_DIRECTORY'] = objdir
+
+    alt_objects = os.path.join(gitdir, 'objects') if gitdir else None
+    if alt_objects and os.path.realpath(alt_objects) != os.path.realpath(objdir):
+      # Allow git to search the original place in case of local or unique refs
+      # that git will attempt to resolve even if we aren't fetching them.
+      env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = alt_objects
+  if bare and gitdir is not None:
+      env[GIT_DIR] = gitdir
+
+  return env
+
+
 class GitCommand(object):
+  """Wrapper around a single git invocation."""
+
   def __init__(self,
                project,
                cmdv,
@@ -169,25 +218,8 @@ class GitCommand(object):
                disable_editor=False,
                ssh_proxy=None,
                cwd=None,
-               gitdir=None):
-    env = self._GetBasicEnv()
-
-    if disable_editor:
-      env['GIT_EDITOR'] = ':'
-    if ssh_proxy:
-      env['REPO_SSH_SOCK'] = ssh_proxy.sock()
-      env['GIT_SSH'] = ssh_proxy.proxy
-      env['GIT_SSH_VARIANT'] = 'ssh'
-    if 'http_proxy' in env and 'darwin' == sys.platform:
-      s = "'http.proxy=%s'" % (env['http_proxy'],)
-      p = env.get('GIT_CONFIG_PARAMETERS')
-      if p is not None:
-        s = p + ' ' + s
-      env['GIT_CONFIG_PARAMETERS'] = s
-    if 'GIT_ALLOW_PROTOCOL' not in env:
-      env['GIT_ALLOW_PROTOCOL'] = (
-          'file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc')
-    env['GIT_HTTP_USER_AGENT'] = user_agent.git
+               gitdir=None,
+               objdir=None):
 
     if project:
       if not cwd:
@@ -195,13 +227,23 @@ class GitCommand(object):
       if not gitdir:
         gitdir = project.gitdir
 
+    # Git on Windows wants its paths only using / for reliability.
+    if platform_utils.isWindows():
+      if objdir:
+        objdir = objdir.replace('\\', '/')
+      if gitdir:
+        gitdir = gitdir.replace('\\', '/')
+
+    env = _build_env(
+        disable_editor=disable_editor,
+        ssh_proxy=ssh_proxy,
+        objdir=objdir,
+        gitdir=gitdir,
+        bare=bare,
+    )
+
     command = [GIT]
     if bare:
-      if gitdir:
-        # Git on Windows wants its paths only using / for reliability.
-        if platform_utils.isWindows():
-          gitdir = gitdir.replace('\\', '/')
-        env[GIT_DIR] = gitdir
       cwd = None
     command.append(cmdv[0])
     # Need to use the --progress flag for fetch/clone so output will be
@@ -216,12 +258,11 @@ class GitCommand(object):
     stderr = (subprocess.STDOUT if merge_output else
               (subprocess.PIPE if capture_stderr else None))
 
+    dbg = ''
     if IsTrace():
       global LAST_CWD
       global LAST_GITDIR
 
-      dbg = ''
-
       if cwd and LAST_CWD != cwd:
         if LAST_GITDIR or LAST_CWD:
           dbg += '\n'
@@ -234,6 +275,12 @@ class GitCommand(object):
         dbg += ': export GIT_DIR=%s\n' % env[GIT_DIR]
         LAST_GITDIR = env[GIT_DIR]
 
+      if 'GIT_OBJECT_DIRECTORY' in env:
+        dbg += ': export GIT_OBJECT_DIRECTORY=%s\n' % env['GIT_OBJECT_DIRECTORY']
+      if 'GIT_ALTERNATE_OBJECT_DIRECTORIES' in env:
+        dbg += ': export GIT_ALTERNATE_OBJECT_DIRECTORIES=%s\n' % (
+            env['GIT_ALTERNATE_OBJECT_DIRECTORIES'])
+
       dbg += ': '
       dbg += ' '.join(command)
       if stdin == subprocess.PIPE:
@@ -244,36 +291,31 @@ class GitCommand(object):
         dbg += ' 2>|'
       elif stderr == subprocess.STDOUT:
         dbg += ' 2>&1'
-      Trace('%s', dbg)
 
-    try:
-      p = subprocess.Popen(command,
-                           cwd=cwd,
-                           env=env,
-                           encoding='utf-8',
-                           errors='backslashreplace',
-                           stdin=stdin,
-                           stdout=stdout,
-                           stderr=stderr)
-    except Exception as e:
-      raise GitError('%s: %s' % (command[1], e))
+    with Trace('git command %s %s with debug: %s', LAST_GITDIR, command, dbg):
+      try:
+        p = subprocess.Popen(command,
+                             cwd=cwd,
+                             env=env,
+                             encoding='utf-8',
+                             errors='backslashreplace',
+                             stdin=stdin,
+                             stdout=stdout,
+                             stderr=stderr)
+      except Exception as e:
+        raise GitError('%s: %s' % (command[1], e))
 
-    if ssh_proxy:
-      ssh_proxy.add_client(p)
-
-    self.process = p
-    if input:
-      if isinstance(input, str):
-        input = input.encode('utf-8')
-      p.stdin.write(input)
-      p.stdin.close()
-
-    try:
-      self.stdout, self.stderr = p.communicate()
-    finally:
       if ssh_proxy:
-        ssh_proxy.remove_client(p)
-    self.rc = p.wait()
+        ssh_proxy.add_client(p)
+
+      self.process = p
+
+      try:
+        self.stdout, self.stderr = p.communicate(input=input)
+      finally:
+        if ssh_proxy:
+          ssh_proxy.remove_client(p)
+      self.rc = p.wait()
 
   @staticmethod
   def _GetBasicEnv():

git_config.py

@@ -22,6 +22,7 @@ import re
 import ssl
 import subprocess
 import sys
+from typing import Union
 import urllib.error
 import urllib.request
 
@@ -68,8 +69,6 @@ def _key(name):
 class GitConfig(object):
   _ForUser = None
 
-  _USER_CONFIG = '~/.gitconfig'
-
   _ForSystem = None
   _SYSTEM_CONFIG = '/etc/gitconfig'
 
@@ -82,9 +81,13 @@ class GitConfig(object):
   @classmethod
   def ForUser(cls):
     if cls._ForUser is None:
-      cls._ForUser = cls(configfile=os.path.expanduser(cls._USER_CONFIG))
+      cls._ForUser = cls(configfile=cls._getUserConfig())
     return cls._ForUser
 
+  @staticmethod
+  def _getUserConfig():
+    return os.path.expanduser('~/.gitconfig')
+
   @classmethod
   def ForRepository(cls, gitdir, defaults=None):
     return cls(configfile=os.path.join(gitdir, 'config'),
@@ -117,7 +120,7 @@ class GitConfig(object):
       return self.defaults.Has(name, include_defaults=True)
     return False
 
-  def GetInt(self, name):
+  def GetInt(self, name: str) -> Union[int, None]:
     """Returns an integer from the configuration file.
 
     This follows the git config syntax.
@@ -126,7 +129,7 @@ class GitConfig(object):
       name: The key to lookup.
 
     Returns:
-      None if the value was not defined, or is not a boolean.
+      None if the value was not defined, or is not an int.
       Otherwise, the number itself.
     """
     v = self.GetString(name)
@@ -152,6 +155,9 @@ class GitConfig(object):
     try:
       return int(v, base=base) * mult
     except ValueError:
+      print(
+          f"warning: expected {name} to represent an integer, got {v} instead",
+          file=sys.stderr)
       return None
 
   def DumpConfigDict(self):
@@ -169,7 +175,7 @@ class GitConfig(object):
       config_dict[key] = self.GetString(key)
     return config_dict
 
-  def GetBoolean(self, name):
+  def GetBoolean(self, name: str) -> Union[str, None]:
     """Returns a boolean from the configuration file.
        None : The value was not defined, or is not a boolean.
        True : The value was set to true or yes.
@@ -183,6 +189,8 @@ class GitConfig(object):
       return True
     if v in ('false', 'no'):
       return False
+    print(f"warning: expected {name} to represent a boolean, got {v} instead",
+          file=sys.stderr)
     return None
 
   def SetBoolean(self, name, value):
@@ -191,7 +199,7 @@ class GitConfig(object):
       value = 'true' if value else 'false'
     self.SetString(name, value)
 
-  def GetString(self, name, all_keys=False):
+  def GetString(self, name: str, all_keys: bool = False) -> Union[str, None]:
     """Get the first value for a key, or None if it is not defined.
 
        This configuration file is used first, if the key is not
@@ -219,8 +227,8 @@ class GitConfig(object):
     """Set the value(s) for a key.
        Only this configuration file is modified.
 
-       The supplied value should be either a string,
-       or a list of strings (to store multiple values).
+       The supplied value should be either a string, or a list of strings (to
+       store multiple values), or None (to delete the key).
     """
     key = _key(name)
 
@@ -349,10 +357,10 @@ class GitConfig(object):
     except OSError:
       return None
     try:
-      Trace(': parsing %s', self.file)
-      with open(self._json) as fd:
-        return json.load(fd)
-    except (IOError, ValueErrorl):
+      with Trace(': parsing %s', self.file):
+        with open(self._json) as fd:
+          return json.load(fd)
+    except (IOError, ValueError):
       platform_utils.remove(self._json, missing_ok=True)
       return None
 
@@ -409,7 +417,10 @@ class GitConfig(object):
 class RepoConfig(GitConfig):
   """User settings for repo itself."""
 
-  _USER_CONFIG = '~/.repoconfig/config'
+  @staticmethod
+  def _getUserConfig():
+    repo_config_dir = os.getenv('REPO_CONFIG_DIR', os.path.expanduser('~'))
+    return os.path.join(repo_config_dir, '.repoconfig/config')
 
 
 class RefSpec(object):

git_refs.py

@@ -67,38 +67,37 @@ class GitRefs(object):
       self._LoadAll()
 
   def _NeedUpdate(self):
-    Trace(': scan refs %s', self._gitdir)
-
-    for name, mtime in self._mtime.items():
-      try:
-        if mtime != os.path.getmtime(os.path.join(self._gitdir, name)):
+    with Trace(': scan refs %s', self._gitdir):
+      for name, mtime in self._mtime.items():
+        try:
+          if mtime != os.path.getmtime(os.path.join(self._gitdir, name)):
+            return True
+        except OSError:
           return True
-      except OSError:
-        return True
-    return False
+      return False
 
   def _LoadAll(self):
-    Trace(': load refs %s', self._gitdir)
+    with Trace(': load refs %s', self._gitdir):
 
-    self._phyref = {}
-    self._symref = {}
-    self._mtime = {}
+      self._phyref = {}
+      self._symref = {}
+      self._mtime = {}
 
-    self._ReadPackedRefs()
-    self._ReadLoose('refs/')
-    self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD)
+      self._ReadPackedRefs()
+      self._ReadLoose('refs/')
+      self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD)
 
-    scan = self._symref
-    attempts = 0
-    while scan and attempts < 5:
-      scan_next = {}
-      for name, dest in scan.items():
-        if dest in self._phyref:
-          self._phyref[name] = self._phyref[dest]
-        else:
-          scan_next[name] = dest
-      scan = scan_next
-      attempts += 1
+      scan = self._symref
+      attempts = 0
+      while scan and attempts < 5:
+        scan_next = {}
+        for name, dest in scan.items():
+          if dest in self._phyref:
+            self._phyref[name] = self._phyref[dest]
+          else:
+            scan_next[name] = dest
+        scan = scan_next
+        attempts += 1
 
   def _ReadPackedRefs(self):
     path = os.path.join(self._gitdir, 'packed-refs')

git_superproject.py

@@ -18,7 +18,7 @@ For more information on superproject, check out:
 https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects
 
 Examples:
-  superproject = Superproject()
+  superproject = Superproject(manifest, name, remote, revision)
   UpdateProjectsResult = superproject.UpdateProjectsRevisionId(projects)
 """
 
@@ -31,8 +31,7 @@ from typing import NamedTuple
 
 from git_command import git_require, GitCommand
 from git_config import RepoConfig
-from git_refs import R_HEADS
-from manifest_xml import LOCAL_MANIFEST_GROUP_PREFIX
+from git_refs import GitRefs
 
 _SUPERPROJECT_GIT_NAME = 'superproject.git'
 _SUPERPROJECT_MANIFEST_NAME = 'superproject_override.xml'
@@ -72,41 +71,50 @@ class Superproject(object):
   lookup of commit ids for all projects. It contains _project_commit_ids which
   is a dictionary with project/commit id entries.
   """
-  def __init__(self, manifest, repodir, git_event_log,
-               superproject_dir='exp-superproject', quiet=False, print_messages=False):
+  def __init__(self, manifest, name, remote, revision,
+               superproject_dir='exp-superproject'):
     """Initializes superproject.
 
     Args:
       manifest: A Manifest object that is to be written to a file.
-      repodir: Path to the .repo/ dir for holding all internal checkout state.
-          It must be in the top directory of the repo client checkout.
-      git_event_log: A git trace2 event log to log events.
-      superproject_dir: Relative path under |repodir| to checkout superproject.
-      quiet:  If True then only print the progress messages.
-      print_messages: if True then print error/warning messages.
+      name: The unique name of the superproject
+      remote: The RemoteSpec for the remote.
+      revision: The name of the git branch to track.
+      superproject_dir: Relative path under |manifest.subdir| to checkout
+          superproject.
     """
     self._project_commit_ids = None
     self._manifest = manifest
-    self._git_event_log = git_event_log
-    self._quiet = quiet
-    self._print_messages = print_messages
-    self._branch = manifest.branch
-    self._repodir = os.path.abspath(repodir)
+    self.name = name
+    self.remote = remote
+    self.revision = self._branch = revision
+    self._repodir = manifest.repodir
     self._superproject_dir = superproject_dir
-    self._superproject_path = os.path.join(self._repodir, superproject_dir)
+    self._superproject_path = manifest.SubmanifestInfoDir(manifest.path_prefix,
+                                                          superproject_dir)
     self._manifest_path = os.path.join(self._superproject_path,
                                        _SUPERPROJECT_MANIFEST_NAME)
-    git_name = ''
-    if self._manifest.superproject:
-      remote = self._manifest.superproject['remote']
-      git_name = hashlib.md5(remote.name.encode('utf8')).hexdigest() + '-'
-      self._branch = self._manifest.superproject['revision']
-      self._remote_url = remote.url
-    else:
-      self._remote_url = None
+    git_name = hashlib.md5(remote.name.encode('utf8')).hexdigest() + '-'
+    self._remote_url = remote.url
     self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME
     self._work_git = os.path.join(self._superproject_path, self._work_git_name)
 
+    # The following are command arguemnts, rather than superproject attributes,
+    # and were included here originally.  They should eventually become
+    # arguments that are passed down from the public methods, instead of being
+    # treated as attributes.
+    self._git_event_log = None
+    self._quiet = False
+    self._print_messages = False
+
+  def SetQuiet(self, value):
+    """Set the _quiet attribute."""
+    self._quiet = value
+
+  def SetPrintMessages(self, value):
+    """Set the _print_messages attribute."""
+    self._print_messages = value
+
   @property
   def project_commit_ids(self):
     """Returns a dictionary of projects and their commit ids."""
@@ -117,23 +125,24 @@ class Superproject(object):
     """Returns the manifest path if the path exists or None."""
     return self._manifest_path if os.path.exists(self._manifest_path) else None
 
-  def _LogMessage(self, message):
+  def _LogMessage(self, fmt, *inputs):
     """Logs message to stderr and _git_event_log."""
+    message = f'{self._LogMessagePrefix()} {fmt.format(*inputs)}'
     if self._print_messages:
       print(message, file=sys.stderr)
-    self._git_event_log.ErrorEvent(message, f'{message}')
+    self._git_event_log.ErrorEvent(message, fmt)
 
   def _LogMessagePrefix(self):
     """Returns the prefix string to be logged in each log message"""
     return f'repo superproject branch: {self._branch} url: {self._remote_url}'
 
-  def _LogError(self, message):
+  def _LogError(self, fmt, *inputs):
     """Logs error message to stderr and _git_event_log."""
-    self._LogMessage(f'{self._LogMessagePrefix()} error: {message}')
+    self._LogMessage(f'error: {fmt}', *inputs)
 
-  def _LogWarning(self, message):
+  def _LogWarning(self, fmt, *inputs):
     """Logs warning message to stderr and _git_event_log."""
-    self._LogMessage(f'{self._LogMessagePrefix()} warning: {message}')
+    self._LogMessage(f'warning: {fmt}', *inputs)
 
   def _Init(self):
     """Sets up a local Git repository to get a copy of a superproject.
@@ -154,8 +163,8 @@ class Superproject(object):
                    capture_stderr=True)
     retval = p.Wait()
     if retval:
-      self._LogWarning(f'git init call failed, command: git {cmd}, '
-                       f'return code: {retval}, stderr: {p.stderr}')
+      self._LogWarning('git init call failed, command: git {}, '
+                       'return code: {}, stderr: {}', cmd, retval, p.stderr)
       return False
     return True
 
@@ -166,13 +175,23 @@ class Superproject(object):
       True if fetch is successful, or False.
     """
     if not os.path.exists(self._work_git):
-      self._LogWarning(f'git fetch missing directory: {self._work_git}')
+      self._LogWarning('git fetch missing directory: {}', self._work_git)
       return False
     if not git_require((2, 28, 0)):
       self._LogWarning('superproject requires a git version 2.28 or later')
       return False
     cmd = ['fetch', self._remote_url, '--depth', '1', '--force', '--no-tags',
            '--filter', 'blob:none']
+
+    # Check if there is a local ref that we can pass to --negotiation-tip.
+    # If this is the first fetch, it does not exist yet.
+    # We use --negotiation-tip to speed up the fetch. Superproject branches do
+    # not share commits. So this lets git know it only needs to send commits
+    # reachable from the specified local refs.
+    rev_commit = GitRefs(self._work_git).get(f'refs/heads/{self.revision}')
+    if rev_commit:
+      cmd.extend(['--negotiation-tip', rev_commit])
+
     if self._branch:
       cmd += [self._branch + ':' + self._branch]
     p = GitCommand(None,
@@ -182,8 +201,8 @@ class Superproject(object):
                    capture_stderr=True)
     retval = p.Wait()
     if retval:
-      self._LogWarning(f'git fetch call failed, command: git {cmd}, '
-                       f'return code: {retval}, stderr: {p.stderr}')
+      self._LogWarning('git fetch call failed, command: git {}, '
+                       'return code: {}, stderr: {}', cmd, retval, p.stderr)
       return False
     return True
 
@@ -196,7 +215,7 @@ class Superproject(object):
       data: data returned from 'git ls-tree ...' instead of None.
     """
     if not os.path.exists(self._work_git):
-      self._LogWarning(f'git ls-tree missing directory: {self._work_git}')
+      self._LogWarning('git ls-tree missing directory: {}', self._work_git)
       return None
     data = None
     branch = 'HEAD' if not self._branch else self._branch
@@ -211,27 +230,31 @@ class Superproject(object):
     if retval == 0:
       data = p.stdout
     else:
-      self._LogWarning(f'git ls-tree call failed, command: git {cmd}, '
-                       f'return code: {retval}, stderr: {p.stderr}')
+      self._LogWarning('git ls-tree call failed, command: git {}, '
+                       'return code: {}, stderr: {}', cmd, retval, p.stderr)
     return data
 
-  def Sync(self):
+  def Sync(self, git_event_log):
     """Gets a local copy of a superproject for the manifest.
 
+    Args:
+      git_event_log: an EventLog, for git tracing.
+
     Returns:
       SyncResult
     """
+    self._git_event_log = git_event_log
     if not self._manifest.superproject:
-      self._LogWarning(f'superproject tag is not defined in manifest: '
-                       f'{self._manifest.manifestFile}')
+      self._LogWarning('superproject tag is not defined in manifest: {}',
+                       self._manifest.manifestFile)
       return SyncResult(False, False)
 
-    print('NOTICE: --use-superproject is in beta; report any issues to the '
-          'address described in `repo version`', file=sys.stderr)
+    _PrintBetaNotice()
+
     should_exit = True
     if not self._remote_url:
-      self._LogWarning(f'superproject URL is not defined in manifest: '
-                       f'{self._manifest.manifestFile}')
+      self._LogWarning('superproject URL is not defined in manifest: {}',
+                       self._manifest.manifestFile)
       return SyncResult(False, should_exit)
 
     if not self._Init():
@@ -248,14 +271,14 @@ class Superproject(object):
     Returns:
       CommitIdsResult
     """
-    sync_result = self.Sync()
+    sync_result = self.Sync(self._git_event_log)
     if not sync_result.success:
       return CommitIdsResult(None, sync_result.fatal)
 
     data = self._LsTree()
     if not data:
-      self._LogWarning(f'git ls-tree failed to return data for manifest: '
-                       f'{self._manifest.manifestFile}')
+      self._LogWarning('git ls-tree failed to return data for manifest: {}',
+                       self._manifest.manifestFile)
       return CommitIdsResult(None, True)
 
     # Parse lines like the following to select lines starting with '160000' and
@@ -281,15 +304,17 @@ class Superproject(object):
       manifest_path: Path name of the file into which manifest is written instead of None.
     """
     if not os.path.exists(self._superproject_path):
-      self._LogWarning(f'missing superproject directory: {self._superproject_path}')
+      self._LogWarning('missing superproject directory: {}', self._superproject_path)
       return None
-    manifest_str = self._manifest.ToXml(groups=self._manifest.GetGroupsStr()).toxml()
+    manifest_str = self._manifest.ToXml(groups=self._manifest.GetGroupsStr(),
+                                        omit_local=True).toxml()
     manifest_path = self._manifest_path
     try:
       with open(manifest_path, 'w', encoding='utf-8') as fp:
         fp.write(manifest_str)
     except IOError as e:
-      self._LogError(f'cannot write manifest to : {manifest_path} {e}')
+      self._LogError('cannot write manifest to : {} {}',
+                     manifest_path, e)
       return None
     return manifest_path
 
@@ -311,17 +336,19 @@ class Superproject(object):
     if project.revisionId:
       return True
     # Skip the project if it comes from the local manifest.
-    return any(s.startswith(LOCAL_MANIFEST_GROUP_PREFIX) for s in project.groups)
+    return project.manifest.IsFromLocalManifest(project)
 
-  def UpdateProjectsRevisionId(self, projects):
+  def UpdateProjectsRevisionId(self, projects, git_event_log):
     """Update revisionId of every project in projects with the commit id.
 
     Args:
-      projects: List of projects whose revisionId needs to be updated.
+      projects: a list of projects whose revisionId needs to be updated.
+      git_event_log: an EventLog, for git tracing.
 
     Returns:
       UpdateProjectsResult
     """
+    self._git_event_log = git_event_log
     commit_ids_result = self._GetAllProjectsCommitIds()
     commit_ids = commit_ids_result.commit_ids
     if not commit_ids:
@@ -339,8 +366,9 @@ class Superproject(object):
     # If superproject doesn't have a commit id for a project, then report an
     # error event and continue as if do not use superproject is specified.
     if projects_missing_commit_ids:
-      self._LogWarning(f'please file a bug using {self._manifest.contactinfo.bugurl} '
-                       f'to report missing commit_ids for: {projects_missing_commit_ids}')
+      self._LogWarning('please file a bug using {} to report missing '
+                       'commit_ids for: {}', self._manifest.contactinfo.bugurl,
+                       projects_missing_commit_ids)
       return UpdateProjectsResult(None, False)
 
     for project in projects:
@@ -351,6 +379,13 @@ class Superproject(object):
     return UpdateProjectsResult(manifest_path, False)
 
 
+@functools.lru_cache(maxsize=10)
+def _PrintBetaNotice():
+  """Print the notice of beta status."""
+  print('NOTICE: --use-superproject is in beta; report any issues to the '
+        'address described in `repo version`', file=sys.stderr)
+
+
 @functools.lru_cache(maxsize=None)
 def _UseSuperprojectFromConfiguration():
   """Returns the user choice of whether to use superproject."""
@@ -395,21 +430,37 @@ def _UseSuperprojectFromConfiguration():
   return False
 
 
-def PrintMessages(opt, manifest):
-  """Returns a boolean if error/warning messages are to be printed."""
-  return opt.use_superproject is not None or manifest.superproject
+def PrintMessages(use_superproject, manifest):
+  """Returns a boolean if error/warning messages are to be printed.
+
+  Args:
+    use_superproject: option value from optparse.
+    manifest: manifest to use.
+  """
+  return use_superproject is not None or bool(manifest.superproject)
 
 
-def UseSuperproject(opt, manifest):
-  """Returns a boolean if use-superproject option is enabled."""
+def UseSuperproject(use_superproject, manifest):
+  """Returns a boolean if use-superproject option is enabled.
 
-  if opt.use_superproject is not None:
-    return opt.use_superproject
+  Args:
+    use_superproject: option value from optparse.
+    manifest: manifest to use.
+
+  Returns:
+    Whether the superproject should be used.
+  """
+
+  if not manifest.superproject:
+    # This (sub) manifest does not have a superproject definition.
+    return False
+  elif use_superproject is not None:
+    return use_superproject
   else:
-    client_value = manifest.manifestProject.config.GetBoolean('repo.superproject')
+    client_value = manifest.manifestProject.use_superproject
     if client_value is not None:
       return client_value
-    else:
-      if not manifest.superproject:
-        return False
+    elif manifest.superproject:
       return _UseSuperprojectFromConfiguration()
+    else:
+      return False

git_trace2_event_log.py

@@ -29,8 +29,10 @@ https://git-scm.com/docs/api-trace2#_the_event_format_target
 
 
 import datetime
+import errno
 import json
 import os
+import socket
 import sys
 import tempfile
 import threading
@@ -108,7 +110,7 @@ class EventLog(object):
     return {
         'event': event_name,
         'sid': self._full_sid,
-        'thread': threading.currentThread().getName(),
+        'thread': threading.current_thread().name,
         'time': datetime.datetime.utcnow().isoformat() + 'Z',
     }
 
@@ -218,20 +220,39 @@ class EventLog(object):
           retval, p.stderr), file=sys.stderr)
     return path
 
+  def _WriteLog(self, write_fn):
+    """Writes the log out using a provided writer function.
+
+    Generate compact JSON output for each item in the log, and write it using
+    write_fn.
+
+    Args:
+      write_fn: A function that accepts byts and writes them to a destination.
+    """
+
+    for e in self._log:
+      # Dump in compact encoding mode.
+      # See 'Compact encoding' in Python docs:
+      # https://docs.python.org/3/library/json.html#module-json
+      write_fn(json.dumps(e, indent=None, separators=(',', ':')).encode('utf-8') + b'\n')
+
   def Write(self, path=None):
-    """Writes the log out to a file.
+    """Writes the log out to a file or socket.
 
     Log is only written if 'path' or 'git config --get trace2.eventtarget'
-    provide a valid path to write logs to.
+    provide a valid path (or socket) to write logs to.
 
     Logging filename format follows the git trace2 style of being a unique
     (exclusive writable) file.
 
     Args:
-      path: Path to where logs should be written.
+      path: Path to where logs should be written. The path may have a prefix of
+          the form "af_unix:[{stream|dgram}:]", in which case the path is
+          treated as a Unix domain socket. See
+          https://git-scm.com/docs/api-trace2#_enabling_a_target for details.
 
     Returns:
-      log_path: Path to the log file if log is written, otherwise None
+      log_path: Path to the log file or socket if log is written, otherwise None
     """
     log_path = None
     # If no logging path is specified, get the path from 'trace2.eventtarget'.
@@ -242,29 +263,66 @@ class EventLog(object):
     if path is None:
       return None
 
+    path_is_socket = False
+    socket_type = None
     if isinstance(path, str):
-      # Get absolute path.
-      path = os.path.abspath(os.path.expanduser(path))
+      parts = path.split(':', 1)
+      if parts[0] == 'af_unix' and len(parts) == 2:
+        path_is_socket = True
+        path = parts[1]
+        parts = path.split(':', 1)
+        if parts[0] == 'stream' and len(parts) == 2:
+          socket_type = socket.SOCK_STREAM
+          path = parts[1]
+        elif parts[0] == 'dgram' and len(parts) == 2:
+          socket_type = socket.SOCK_DGRAM
+          path = parts[1]
+      else:
+        # Get absolute path.
+        path = os.path.abspath(os.path.expanduser(path))
     else:
       raise TypeError('path: str required but got %s.' % type(path))
 
     # Git trace2 requires a directory to write log to.
 
     # TODO(https://crbug.com/gerrit/13706): Support file (append) mode also.
-    if not os.path.isdir(path):
+    if not (path_is_socket or os.path.isdir(path)):
       return None
+
+    if path_is_socket:
+      if socket_type == socket.SOCK_STREAM or socket_type is None:
+        try:
+          with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
+            sock.connect(path)
+            self._WriteLog(sock.sendall)
+          return f'af_unix:stream:{path}'
+        except OSError as err:
+          # If we tried to connect to a DGRAM socket using STREAM, ignore the
+          # attempt and continue to DGRAM below. Otherwise, issue a warning.
+          if err.errno != errno.EPROTOTYPE:
+            print(f'repo: warning: git trace2 logging failed: {err}', file=sys.stderr)
+            return None
+      if socket_type == socket.SOCK_DGRAM or socket_type is None:
+        try:
+          with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
+            self._WriteLog(lambda bs: sock.sendto(bs, path))
+            return f'af_unix:dgram:{path}'
+        except OSError as err:
+          print(f'repo: warning: git trace2 logging failed: {err}', file=sys.stderr)
+          return None
+      # Tried to open a socket but couldn't connect (SOCK_STREAM) or write
+      # (SOCK_DGRAM).
+      print('repo: warning: git trace2 logging failed: could not write to socket', file=sys.stderr)
+      return None
+
+    # Path is an absolute path
     # Use NamedTemporaryFile to generate a unique filename as required by git trace2.
     try:
-      with tempfile.NamedTemporaryFile(mode='x', prefix=self._sid, dir=path,
+      with tempfile.NamedTemporaryFile(mode='xb', prefix=self._sid, dir=path,
                                        delete=False) as f:
         # TODO(https://crbug.com/gerrit/13706): Support writing events as they
         # occur.
-        for e in self._log:
-          # Dump in compact encoding mode.
-          # See 'Compact encoding' in Python docs:
-          # https://docs.python.org/3/library/json.html#module-json
-          json.dump(e, f, indent=None, separators=(',', ':'))
-          f.write('\n')
+        self._WriteLog(f.write)
         log_path = f.name
     except FileExistsError as err:
       print('repo: warning: git trace2 logging failed: %r' % err,

gitc_utils.py

@@ -14,7 +14,6 @@
 
 import os
 import multiprocessing
-import platform
 import re
 import sys
 import time

hooks/commit-msg

@@ -1,5 +1,5 @@
 #!/bin/sh
-# From Gerrit Code Review 3.1.3
+# From Gerrit Code Review 3.6.1 c67916dbdc07555c44e32a68f92ffc484b9b34f0
 #
 # Part of Gerrit Code Review (https://www.gerritcodereview.com/)
 #
@@ -17,6 +17,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -u
+
 # avoid [[ which is not POSIX sh.
 if test "$#" != 1 ; then
   echo "$0 requires an argument."
@@ -29,15 +31,25 @@ if test ! -f "$1" ; then
 fi
 
 # Do not create a change id if requested
-if test "false" = "`git config --bool --get gerrit.createChangeId`" ; then
+if test "false" = "$(git config --bool --get gerrit.createChangeId)" ; then
   exit 0
 fi
 
-# $RANDOM will be undefined if not using bash, so don't use set -u
-random=$( (whoami ; hostname ; date; cat $1 ; echo $RANDOM) | git hash-object --stdin)
+# Do not create a change id for squash commits.
+if head -n1 "$1" | grep -q '^squash! '; then
+  exit 0
+fi
+
+if git rev-parse --verify HEAD >/dev/null 2>&1; then
+  refhash="$(git rev-parse HEAD)"
+else
+  refhash="$(git hash-object -t tree /dev/null)"
+fi
+
+random=$({ git var GIT_COMMITTER_IDENT ; echo "$refhash" ; cat "$1"; } | git hash-object --stdin)
 dest="$1.tmp.${random}"
 
-trap 'rm -f "${dest}"' EXIT
+trap 'rm -f "$dest" "$dest-2"' EXIT
 
 if ! git stripspace --strip-comments < "$1" > "${dest}" ; then
    echo "cannot strip comments from $1"
@@ -49,11 +61,40 @@ if test ! -s "${dest}" ; then
   exit 1
 fi
 
+reviewurl="$(git config --get gerrit.reviewUrl)"
+if test -n "${reviewurl}" ; then
+  token="Link"
+  value="${reviewurl%/}/id/I$random"
+  pattern=".*/id/I[0-9a-f]\{40\}$"
+else
+  token="Change-Id"
+  value="I$random"
+  pattern=".*"
+fi
+
+if git interpret-trailers --parse < "$1" | grep -q "^$token: $pattern$" ; then
+  exit 0
+fi
+
+# There must be a Signed-off-by trailer for the code below to work. Insert a
+# sentinel at the end to make sure there is one.
 # Avoid the --in-place option which only appeared in Git 2.8
-# Avoid the --if-exists option which only appeared in Git 2.15
-if ! git -c trailer.ifexists=doNothing interpret-trailers \
-      --trailer "Change-Id: I${random}" < "$1" > "${dest}" ; then
-  echo "cannot insert change-id line in $1"
+if ! git interpret-trailers \
+         --trailer "Signed-off-by: SENTINEL" < "$1" > "$dest-2" ; then
+  echo "cannot insert Signed-off-by sentinel line in $1"
+  exit 1
+fi
+
+# Make sure the trailer appears before any Signed-off-by trailers by inserting
+# it as if it was a Signed-off-by trailer and then use sed to remove the
+# Signed-off-by prefix and the Signed-off-by sentinel line.
+# Avoid the --in-place option which only appeared in Git 2.8
+# Avoid the --where option which only appeared in Git 2.15
+if ! git -c trailer.where=before interpret-trailers \
+         --trailer "Signed-off-by: $token: $value" < "$dest-2" |
+     sed -re "s/^Signed-off-by: ($token: )/\1/" \
+         -e "/^Signed-off-by: SENTINEL/d" > "$dest" ; then
+  echo "cannot insert $token line in $1"
   exit 1
 fi
 

main.py

@@ -37,7 +37,7 @@ except ImportError:
 
 from color import SetDefaultColoring
 import event_log
-from repo_trace import SetTrace
+from repo_trace import SetTrace, Trace, SetTraceToStderr
 from git_command import user_agent
 from git_config import RepoConfig
 from git_trace2_event_log import EventLog
@@ -109,6 +109,9 @@ global_options.add_option('--color',
 global_options.add_option('--trace',
                           dest='trace', action='store_true',
                           help='trace git command execution (REPO_TRACE=1)')
+global_options.add_option('--trace-to-stderr',
+                          dest='trace_to_stderr', action='store_true',
+                          help='trace outputs go to stderr in addition to .repo/TRACE_FILE')
 global_options.add_option('--trace-python',
                           dest='trace_python', action='store_true',
                           help='trace python command execution')
@@ -127,6 +130,8 @@ global_options.add_option('--event-log',
                           help='filename of event log to append timeline to')
 global_options.add_option('--git-trace2-event-log', action='store',
                           help='directory to write git trace2 event log to')
+global_options.add_option('--submanifest-path', action='store',
+                          metavar='REL_PATH', help='submanifest path')
 
 
 class _Repo(object):
@@ -196,9 +201,6 @@ class _Repo(object):
     """Execute the requested subcommand."""
     result = 0
 
-    if gopts.trace:
-      SetTrace()
-
     # Handle options that terminate quickly first.
     if gopts.help or gopts.help_all:
       self._PrintHelp(short=False, all_commands=gopts.help_all)
@@ -214,10 +216,30 @@ class _Repo(object):
       self._PrintHelp(short=True)
       return 1
 
+    run = lambda: self._RunLong(name, gopts, argv) or 0
+    with Trace('starting new command: %s', ', '.join([name] + argv),
+               first_trace=True):
+      if gopts.trace_python:
+        import trace
+        tracer = trace.Trace(count=False, trace=True, timing=True,
+                             ignoredirs=set(sys.path[1:]))
+        result = tracer.runfunc(run)
+      else:
+        result = run()
+    return result
+
+  def _RunLong(self, name, gopts, argv):
+    """Execute the (longer running) requested subcommand."""
+    result = 0
     SetDefaultColoring(gopts.color)
 
     git_trace2_event_log = EventLog()
-    repo_client = RepoClient(self.repodir)
+    outer_client = RepoClient(self.repodir)
+    repo_client = outer_client
+    if gopts.submanifest_path:
+      repo_client = RepoClient(self.repodir,
+                               submanifest_path=gopts.submanifest_path,
+                               outer_client=outer_client)
     gitc_manifest = None
     gitc_client_name = gitc_utils.parse_clientdir(os.getcwd())
     if gitc_client_name:
@@ -229,6 +251,8 @@ class _Repo(object):
           repodir=self.repodir,
           client=repo_client,
           manifest=repo_client.manifest,
+          outer_client=outer_client,
+          outer_manifest=outer_client.manifest,
           gitc_manifest=gitc_manifest,
           git_event_log=git_trace2_event_log)
     except KeyError:
@@ -283,7 +307,36 @@ class _Repo(object):
     try:
       cmd.CommonValidateOptions(copts, cargs)
       cmd.ValidateOptions(copts, cargs)
-      result = cmd.Execute(copts, cargs)
+
+      this_manifest_only = copts.this_manifest_only
+      outer_manifest = copts.outer_manifest
+      if cmd.MULTI_MANIFEST_SUPPORT or this_manifest_only:
+        result = cmd.Execute(copts, cargs)
+      elif outer_manifest and repo_client.manifest.is_submanifest:
+        # The command does not support multi-manifest, we are using a
+        # submanifest, and the command line is for the outermost manifest.
+        # Re-run using the outermost manifest, which will recurse through the
+        # submanifests.
+        gopts.submanifest_path = ''
+        result = self._Run(name, gopts, argv)
+      else:
+        # No multi-manifest support. Run the command in the current
+        # (sub)manifest, and then any child submanifests.
+        result = cmd.Execute(copts, cargs)
+        for submanifest in repo_client.manifest.submanifests.values():
+          spec = submanifest.ToSubmanifestSpec()
+          gopts.submanifest_path = submanifest.repo_client.path_prefix
+          child_argv = argv[:]
+          child_argv.append('--no-outer-manifest')
+          # Not all subcommands support the 3 manifest options, so only add them
+          # if the original command includes them.
+          if hasattr(copts, 'manifest_url'):
+            child_argv.extend(['--manifest-url', spec.manifestUrl])
+          if hasattr(copts, 'manifest_name'):
+            child_argv.extend(['--manifest-name', spec.manifestName])
+          if hasattr(copts, 'manifest_branch'):
+            child_argv.extend(['--manifest-branch', spec.revision])
+          result = self._Run(name, gopts, child_argv) or result
     except (DownloadError, ManifestInvalidRevisionError,
             NoManifestException) as e:
       print('error: in `%s`: %s' % (' '.join([name] + argv), str(e)),
@@ -614,17 +667,18 @@ def _Main(argv):
   Version.wrapper_path = opt.wrapper_path
 
   repo = _Repo(opt.repodir)
+
   try:
     init_http()
     name, gopts, argv = repo._ParseArgs(argv)
-    run = lambda: repo._Run(name, gopts, argv) or 0
-    if gopts.trace_python:
-      import trace
-      tracer = trace.Trace(count=False, trace=True, timing=True,
-                           ignoredirs=set(sys.path[1:]))
-      result = tracer.runfunc(run)
-    else:
-      result = run()
+
+    if gopts.trace:
+      SetTrace()
+
+    if gopts.trace_to_stderr:
+      SetTraceToStderr()
+
+    result = repo._Run(name, gopts, argv) or 0
   except KeyboardInterrupt:
     print('aborted by user', file=sys.stderr)
     result = 1

man/repo-abandon.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo abandon" "Repo Manual"
+.TH REPO "1" "July 2022" "repo abandon" "Repo Manual"
 .SH NAME
 repo \- repo abandon - manual page for repo abandon
 .SH SYNOPSIS
@@ -32,5 +32,18 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help abandon` to view the detailed manual.

man/repo-branches.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo branches" "Repo Manual"
+.TH REPO "1" "July 2022" "repo branches" "Repo Manual"
 .SH NAME
 repo \- repo branches - manual page for repo branches
 .SH SYNOPSIS
@@ -55,5 +55,18 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help branches` to view the detailed manual.

man/repo-checkout.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo checkout" "Repo Manual"
+.TH REPO "1" "July 2022" "repo checkout" "Repo Manual"
 .SH NAME
 repo \- repo checkout - manual page for repo checkout
 .SH SYNOPSIS
@@ -24,6 +24,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help checkout` to view the detailed manual.
 .SH DETAILS

man/repo-cherry-pick.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo cherry-pick" "Repo Manual"
+.TH REPO "1" "July 2022" "repo cherry-pick" "Repo Manual"
 .SH NAME
 repo \- repo cherry-pick - manual page for repo cherry-pick
 .SH SYNOPSIS
@@ -20,6 +20,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help cherry\-pick` to view the detailed manual.
 .SH DETAILS

man/repo-diff.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo diff" "Repo Manual"
+.TH REPO "1" "July 2022" "repo diff" "Repo Manual"
 .SH NAME
 repo \- repo diff - manual page for repo diff
 .SH SYNOPSIS
@@ -31,5 +31,18 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help diff` to view the detailed manual.

man/repo-diffmanifests.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo diffmanifests" "Repo Manual"
+.TH REPO "1" "July 2022" "repo diffmanifests" "Repo Manual"
 .SH NAME
 repo \- repo diffmanifests - manual page for repo diffmanifests
 .SH SYNOPSIS
@@ -29,6 +29,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help diffmanifests` to view the detailed manual.
 .SH DETAILS

man/repo-download.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo download" "Repo Manual"
+.TH REPO "1" "July 2022" "repo download" "Repo Manual"
 .SH NAME
 repo \- repo download - manual page for repo download
 .SH SYNOPSIS
@@ -35,6 +35,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help download` to view the detailed manual.
 .SH DETAILS

man/repo-forall.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo forall" "Repo Manual"
+.TH REPO "1" "July 2022" "repo forall" "Repo Manual"
 .SH NAME
 repo \- repo forall - manual page for repo forall
 .SH SYNOPSIS
@@ -54,6 +54,19 @@ only show errors
 .TP
 \fB\-p\fR
 show project headers before output
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help forall` to view the detailed manual.
 .SH DETAILS
@@ -93,6 +106,11 @@ REPO_PROJECT is set to the unique name of the project.
 .PP
 REPO_PATH is the path relative the the root of the client.
 .PP
+REPO_OUTERPATH is the path of the sub manifest's root relative to the root of
+the client.
+.PP
+REPO_INNERPATH is the path relative to the root of the sub manifest.
+.PP
 REPO_REMOTE is the name of the remote system from the manifest.
 .PP
 REPO_LREV is the name of the revision from the manifest, translated to a local

man/repo-gitc-delete.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo gitc-delete" "Repo Manual"
+.TH REPO "1" "July 2022" "repo gitc-delete" "Repo Manual"
 .SH NAME
 repo \- repo gitc-delete - manual page for repo gitc-delete
 .SH SYNOPSIS
@@ -23,6 +23,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help gitc\-delete` to view the detailed manual.
 .SH DETAILS

man/repo-gitc-init.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "September 2021" "repo gitc-init" "Repo Manual"
+.TH REPO "1" "October 2022" "repo gitc-init" "Repo Manual"
 .SH NAME
 repo \- repo gitc-init - manual page for repo gitc-init
 .SH SYNOPSIS
@@ -31,10 +31,6 @@ manifest branch or revision (use HEAD for default)
 \fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
 initial manifest file
 .TP
-\fB\-\-standalone\-manifest\fR
-download the manifest as a static file rather then
-create a git checkout of the manifest repo
-.TP
 \fB\-g\fR GROUP, \fB\-\-groups\fR=\fI\,GROUP\/\fR
 restrict manifest projects to ones with specified
 group(s) [default|all|G1,G2,G3|G4,\-G5,\-G6]
@@ -45,10 +41,19 @@ platform group [auto|all|none|linux|darwin|...]
 .TP
 \fB\-\-submodules\fR
 sync any submodules associated with the manifest repo
+.TP
+\fB\-\-standalone\-manifest\fR
+download the manifest as a static file rather then
+create a git checkout of the manifest repo
+.TP
+\fB\-\-manifest\-depth\fR=\fI\,DEPTH\/\fR
+create a shallow clone of the manifest repo with given
+depth (0 for full clone); see git clone (default: 0)
 .SS Manifest (only) checkout options:
 .TP
 \fB\-\-current\-branch\fR
 fetch only current manifest branch from server
+(default)
 .TP
 \fB\-\-no\-current\-branch\fR
 fetch all manifest branches from server
@@ -96,7 +101,8 @@ filter for use with \fB\-\-partial\-clone\fR [default:
 blob:none]
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects
@@ -108,6 +114,12 @@ not \fB\-\-partial\-clone\fR)
 \fB\-\-no\-clone\-bundle\fR
 disable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS (default if
 \fB\-\-partial\-clone\fR)
+.TP
+\fB\-\-git\-lfs\fR
+enable Git LFS support
+.TP
+\fB\-\-no\-git\-lfs\fR
+disable Git LFS support
 .SS repo Version options:
 .TP
 \fB\-\-repo\-url\fR=\fI\,URL\/\fR
@@ -129,6 +141,19 @@ Optional manifest file to use for this GITC client.
 .TP
 \fB\-c\fR GITC_CLIENT, \fB\-\-gitc\-client\fR=\fI\,GITC_CLIENT\/\fR
 Name of the gitc_client instance to create or modify.
+.SS Multi\-manifest:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help gitc\-init` to view the detailed manual.
 .SH DETAILS

man/repo-grep.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo grep" "Repo Manual"
+.TH REPO "1" "July 2022" "repo grep" "Repo Manual"
 .SH NAME
 repo \- repo grep - manual page for repo grep
 .SH SYNOPSIS
@@ -24,6 +24,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .SS Sources:
 .TP
 \fB\-\-cached\fR

man/repo-help.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo help" "Repo Manual"
+.TH REPO "1" "July 2022" "repo help" "Repo Manual"
 .SH NAME
 repo \- repo help - manual page for repo help
 .SH SYNOPSIS
@@ -26,6 +26,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help help` to view the detailed manual.
 .SH DETAILS

man/repo-info.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo info" "Repo Manual"
+.TH REPO "1" "July 2022" "repo info" "Repo Manual"
 .SH NAME
 repo \- repo info - manual page for repo info
 .SH SYNOPSIS
@@ -36,5 +36,18 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help info` to view the detailed manual.

man/repo-init.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "September 2021" "repo init" "Repo Manual"
+.TH REPO "1" "October 2022" "repo init" "Repo Manual"
 .SH NAME
 repo \- repo init - manual page for repo init
 .SH SYNOPSIS
@@ -31,10 +31,6 @@ manifest branch or revision (use HEAD for default)
 \fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
 initial manifest file
 .TP
-\fB\-\-standalone\-manifest\fR
-download the manifest as a static file rather then
-create a git checkout of the manifest repo
-.TP
 \fB\-g\fR GROUP, \fB\-\-groups\fR=\fI\,GROUP\/\fR
 restrict manifest projects to ones with specified
 group(s) [default|all|G1,G2,G3|G4,\-G5,\-G6]
@@ -45,10 +41,19 @@ platform group [auto|all|none|linux|darwin|...]
 .TP
 \fB\-\-submodules\fR
 sync any submodules associated with the manifest repo
+.TP
+\fB\-\-standalone\-manifest\fR
+download the manifest as a static file rather then
+create a git checkout of the manifest repo
+.TP
+\fB\-\-manifest\-depth\fR=\fI\,DEPTH\/\fR
+create a shallow clone of the manifest repo with given
+depth (0 for full clone); see git clone (default: 0)
 .SS Manifest (only) checkout options:
 .TP
 \fB\-c\fR, \fB\-\-current\-branch\fR
 fetch only current manifest branch from server
+(default)
 .TP
 \fB\-\-no\-current\-branch\fR
 fetch all manifest branches from server
@@ -96,7 +101,8 @@ filter for use with \fB\-\-partial\-clone\fR [default:
 blob:none]
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects
@@ -108,6 +114,12 @@ not \fB\-\-partial\-clone\fR)
 \fB\-\-no\-clone\-bundle\fR
 disable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS (default if
 \fB\-\-partial\-clone\fR)
+.TP
+\fB\-\-git\-lfs\fR
+enable Git LFS support
+.TP
+\fB\-\-no\-git\-lfs\fR
+disable Git LFS support
 .SS repo Version options:
 .TP
 \fB\-\-repo\-url\fR=\fI\,URL\/\fR
@@ -122,6 +134,19 @@ do not verify repo source code
 .TP
 \fB\-\-config\-name\fR
 Always prompt for name/e\-mail
+.SS Multi\-manifest:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help init` to view the detailed manual.
 .SH DETAILS

man/repo-list.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo list" "Repo Manual"
+.TH REPO "1" "July 2022" "repo list" "Repo Manual"
 .SH NAME
 repo \- repo list - manual page for repo list
 .SH SYNOPSIS
@@ -47,6 +47,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help list` to view the detailed manual.
 .SH DETAILS

man/repo-manifest.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo manifest" "Repo Manual"
+.TH REPO "1" "October 2022" "repo manifest" "Repo Manual"
 .SH NAME
 repo \- repo manifest - manual page for repo manifest
 .SH SYNOPSIS
@@ -40,7 +40,8 @@ format output for humans to read
 ignore local manifests
 .TP
 \fB\-o\fR \-|NAME.xml, \fB\-\-output\-file\fR=\fI\,\-\/\fR|NAME.xml
-file to save the manifest to
+file to save the manifest to. (Filename prefix for
+multi\-tree.)
 .SS Logging options:
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR
@@ -48,6 +49,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help manifest` to view the detailed manual.
 .SH DETAILS
@@ -88,6 +102,7 @@ A manifest XML file (e.g. `default.xml`) roughly conforms to the following DTD:
 remote*,
 default?,
 manifest\-server?,
+submanifest*?,
 remove\-project*,
 project*,
 extend\-project*,
@@ -118,6 +133,16 @@ include*)>
 .IP
 <!ELEMENT manifest\-server EMPTY>
 <!ATTLIST manifest\-server url CDATA #REQUIRED>
+.IP
+<!ELEMENT submanifest EMPTY>
+<!ATTLIST submanifest name           ID #REQUIRED>
+<!ATTLIST submanifest remote         IDREF #IMPLIED>
+<!ATTLIST submanifest project        CDATA #IMPLIED>
+<!ATTLIST submanifest manifest\-name  CDATA #IMPLIED>
+<!ATTLIST submanifest revision       CDATA #IMPLIED>
+<!ATTLIST submanifest path           CDATA #IMPLIED>
+<!ATTLIST submanifest groups         CDATA #IMPLIED>
+<!ATTLIST submanifest default\-groups CDATA #IMPLIED>
 .TP
 <!ELEMENT project (annotation*,
 project*,
@@ -161,9 +186,12 @@ CDATA #IMPLIED>
 <!ELEMENT extend\-project EMPTY>
 <!ATTLIST extend\-project name CDATA #REQUIRED>
 <!ATTLIST extend\-project path CDATA #IMPLIED>
+<!ATTLIST extend\-project dest\-path CDATA #IMPLIED>
 <!ATTLIST extend\-project groups CDATA #IMPLIED>
 <!ATTLIST extend\-project revision CDATA #IMPLIED>
 <!ATTLIST extend\-project remote CDATA #IMPLIED>
+<!ATTLIST extend\-project dest\-branch CDATA #IMPLIED>
+<!ATTLIST extend\-project upstream CDATA #IMPLIED>
 .IP
 <!ELEMENT remove\-project EMPTY>
 <!ATTLIST remove\-project name  CDATA #REQUIRED>
@@ -174,8 +202,9 @@ CDATA #IMPLIED>
 <!ATTLIST repo\-hooks enabled\-list CDATA #REQUIRED>
 .IP
 <!ELEMENT superproject EMPTY>
-<!ATTLIST superproject name    CDATA #REQUIRED>
-<!ATTLIST superproject remote  IDREF #IMPLIED>
+<!ATTLIST superproject name     CDATA #REQUIRED>
+<!ATTLIST superproject remote   IDREF #IMPLIED>
+<!ATTLIST superproject revision CDATA #IMPLIED>
 .IP
 <!ELEMENT contactinfo EMPTY>
 <!ATTLIST contactinfo bugurl  CDATA #REQUIRED>
@@ -293,6 +322,65 @@ GetManifest(tag)
 Return a manifest in which each project is pegged to the revision at the
 specified tag. This is used by repo sync when the \fB\-\-smart\-tag\fR option is given.
 .PP
+Element submanifest
+.PP
+One or more submanifest elements may be specified. Each element describes a
+single manifest to be checked out as a child.
+.PP
+Attribute `name`: A unique name (within the current (sub)manifest) for this
+submanifest. It acts as a default for `revision` below. The same name can be
+used for submanifests with different parent (sub)manifests.
+.PP
+Attribute `remote`: Name of a previously defined remote element. If not supplied
+the remote given by the default element is used.
+.PP
+Attribute `project`: The manifest project name. The project's name is appended
+onto its remote's fetch URL to generate the actual URL to configure the Git
+remote with. The URL gets formed as:
+.IP
+${remote_fetch}/${project_name}.git
+.PP
+where ${remote_fetch} is the remote's fetch attribute and ${project_name} is the
+project's name attribute. The suffix ".git" is always appended as repo assumes
+the upstream is a forest of bare Git repositories. If the project has a parent
+element, its name will be prefixed by the parent's.
+.PP
+The project name must match the name Gerrit knows, if Gerrit is being used for
+code reviews.
+.PP
+`project` must not be empty, and may not be an absolute path or use "." or ".."
+path components. It is always interpreted relative to the remote's fetch
+settings, so if a different base path is needed, declare a different remote with
+the new settings needed.
+.PP
+If not supplied the remote and project for this manifest will be used: `remote`
+cannot be supplied.
+.PP
+Projects from a submanifest and its submanifests are added to the
+submanifest::path:<path_prefix> group.
+.PP
+Attribute `manifest\-name`: The manifest filename in the manifest project. If not
+supplied, `default.xml` is used.
+.PP
+Attribute `revision`: Name of a Git branch (e.g. "main" or "refs/heads/main"),
+tag (e.g. "refs/tags/stable"), or a commit hash. If not supplied, `name` is
+used.
+.PP
+Attribute `path`: An optional path relative to the top directory of the repo
+client where the submanifest repo client top directory should be placed. If not
+supplied, `revision` is used.
+.PP
+`path` may not be an absolute path or use "." or ".." path components.
+.PP
+Attribute `groups`: List of additional groups to which all projects in the
+included submanifest belong. This appends and recurses, meaning all projects in
+submanifests carry all parent submanifest groups. Same syntax as the
+corresponding element of `project`.
+.PP
+Attribute `default\-groups`: The list of manifest groups to sync if no
+`\-\-groups=` parameter was specified at init. When that list is empty, use this
+list instead of "default" as the list of groups to sync.
+.PP
 Element project
 .PP
 One or more project elements may be specified. Each element describes a single
@@ -385,6 +473,11 @@ original manifest.
 Attribute `path`: If specified, limit the change to projects checked out at the
 specified path, rather than all projects with the given name.
 .PP
+Attribute `dest\-path`: If specified, a path relative to the top directory of the
+repo client where the Git working directory for this project should be placed.
+This is used to move a project in the checkout by overriding the existing `path`
+setting.
+.PP
 Attribute `groups`: List of additional groups to which this project belongs.
 Same syntax as the corresponding element of `project`.
 .PP
@@ -394,6 +487,12 @@ project. Same syntax as the corresponding element of `project`.
 Attribute `remote`: If specified, overrides the remote of the original project.
 Same syntax as the corresponding element of `project`.
 .PP
+Attribute `dest\-branch`: If specified, overrides the dest\-branch of the original
+project. Same syntax as the corresponding element of `project`.
+.PP
+Attribute `upstream`: If specified, overrides the upstream of the original
+project. Same syntax as the corresponding element of `project`.
+.PP
 Element annotation
 .PP
 Zero or more annotation elements may be specified as children of a project or
@@ -477,6 +576,10 @@ project](#element\-project) for more information.
 Attribute `remote`: Name of a previously defined remote element. If not supplied
 the remote given by the default element is used.
 .PP
+Attribute `revision`: Name of the Git branch the manifest wants to track for
+this superproject. If not supplied the revision given by the remote element is
+used if applicable, else the default element is used.
+.PP
 Element contactinfo
 .PP
 *** *Note*: This is currently a WIP. ***
@@ -502,10 +605,10 @@ restrictions are not enforced for [Local Manifests].
 .PP
 Attribute `groups`: List of additional groups to which all projects in the
 included manifest belong. This appends and recurses, meaning all projects in
-sub\-manifests carry all parent include groups. Same syntax as the corresponding
-element of `project`.
+included manifests carry all parent include groups. Same syntax as the
+corresponding element of `project`.
 .PP
-Local Manifests
+Local Manifests 
 .PP
 Additional remotes and projects may be added through local manifest files stored
 in `$TOP_DIR/.repo/local_manifests/*.xml`.

man/repo-overview.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo overview" "Repo Manual"
+.TH REPO "1" "July 2022" "repo overview" "Repo Manual"
 .SH NAME
 repo \- repo overview - manual page for repo overview
 .SH SYNOPSIS
@@ -26,6 +26,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help overview` to view the detailed manual.
 .SH DETAILS

man/repo-prune.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo prune" "Repo Manual"
+.TH REPO "1" "July 2022" "repo prune" "Repo Manual"
 .SH NAME
 repo \- repo prune - manual page for repo prune
 .SH SYNOPSIS
@@ -24,5 +24,18 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help prune` to view the detailed manual.

man/repo-rebase.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo rebase" "Repo Manual"
+.TH REPO "1" "July 2022" "repo rebase" "Repo Manual"
 .SH NAME
 repo \- repo rebase - manual page for repo rebase
 .SH SYNOPSIS
@@ -46,6 +46,19 @@ only show errors
 .TP
 \fB\-i\fR, \fB\-\-interactive\fR
 interactive rebase (single project only)
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help rebase` to view the detailed manual.
 .SH DETAILS

man/repo-selfupdate.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo selfupdate" "Repo Manual"
+.TH REPO "1" "July 2022" "repo selfupdate" "Repo Manual"
 .SH NAME
 repo \- repo selfupdate - manual page for repo selfupdate
 .SH SYNOPSIS
@@ -20,6 +20,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .SS repo Version options:
 .TP
 \fB\-\-no\-repo\-verify\fR

man/repo-smartsync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo smartsync" "Repo Manual"
+.TH REPO "1" "November 2022" "repo smartsync" "Repo Manual"
 .SH NAME
 repo \- repo smartsync - manual page for repo smartsync
 .SH SYNOPSIS
@@ -20,11 +20,11 @@ number of CPU cores)
 .TP
 \fB\-\-jobs\-network\fR=\fI\,JOBS\/\fR
 number of network jobs to run in parallel (defaults to
-\fB\-\-jobs\fR)
+\fB\-\-jobs\fR or 1)
 .TP
 \fB\-\-jobs\-checkout\fR=\fI\,JOBS\/\fR
 number of local checkout jobs to run in parallel
-(defaults to \fB\-\-jobs\fR)
+(defaults to \fB\-\-jobs\fR or 8)
 .TP
 \fB\-f\fR, \fB\-\-force\-broken\fR
 obsolete option (to be deleted in the future)
@@ -80,7 +80,8 @@ password to authenticate with the manifest server
 fetch submodules from server
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects
@@ -89,7 +90,7 @@ disable use of manifest superprojects
 fetch tags
 .TP
 \fB\-\-no\-tags\fR
-don't fetch tags
+don't fetch tags (default)
 .TP
 \fB\-\-optimized\-fetch\fR
 only fetch projects fixed to sha1 if revision does not
@@ -100,6 +101,17 @@ number of times to retry fetches on transient errors
 .TP
 \fB\-\-prune\fR
 delete refs that no longer exist on the remote
+(default)
+.TP
+\fB\-\-no\-prune\fR
+do not delete refs that no longer exist on the remote
+.TP
+\fB\-\-auto\-gc\fR
+run garbage collection on all synced projects
+.TP
+\fB\-\-no\-auto\-gc\fR
+do not run garbage collection on any projects
+(default)
 .SS Logging options:
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR
@@ -107,6 +119,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .SS repo Version options:
 .TP
 \fB\-\-no\-repo\-verify\fR

man/repo-stage.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo stage" "Repo Manual"
+.TH REPO "1" "July 2022" "repo stage" "Repo Manual"
 .SH NAME
 repo \- repo stage - manual page for repo stage
 .SH SYNOPSIS
@@ -23,6 +23,19 @@ only show errors
 .TP
 \fB\-i\fR, \fB\-\-interactive\fR
 use interactive staging
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help stage` to view the detailed manual.
 .SH DETAILS

man/repo-start.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo start" "Repo Manual"
+.TH REPO "1" "July 2022" "repo start" "Repo Manual"
 .SH NAME
 repo \- repo start - manual page for repo start
 .SH SYNOPSIS
@@ -33,6 +33,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help start` to view the detailed manual.
 .SH DETAILS

man/repo-status.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo status" "Repo Manual"
+.TH REPO "1" "July 2022" "repo status" "Repo Manual"
 .SH NAME
 repo \- repo status - manual page for repo status
 .SH SYNOPSIS
@@ -28,6 +28,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help status` to view the detailed manual.
 .SH DETAILS

man/repo-sync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo sync" "Repo Manual"
+.TH REPO "1" "November 2022" "repo sync" "Repo Manual"
 .SH NAME
 repo \- repo sync - manual page for repo sync
 .SH SYNOPSIS
@@ -20,11 +20,11 @@ number of CPU cores)
 .TP
 \fB\-\-jobs\-network\fR=\fI\,JOBS\/\fR
 number of network jobs to run in parallel (defaults to
-\fB\-\-jobs\fR)
+\fB\-\-jobs\fR or 1)
 .TP
 \fB\-\-jobs\-checkout\fR=\fI\,JOBS\/\fR
 number of local checkout jobs to run in parallel
-(defaults to \fB\-\-jobs\fR)
+(defaults to \fB\-\-jobs\fR or 8)
 .TP
 \fB\-f\fR, \fB\-\-force\-broken\fR
 obsolete option (to be deleted in the future)
@@ -80,7 +80,8 @@ password to authenticate with the manifest server
 fetch submodules from server
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects
@@ -89,7 +90,7 @@ disable use of manifest superprojects
 fetch tags
 .TP
 \fB\-\-no\-tags\fR
-don't fetch tags
+don't fetch tags (default)
 .TP
 \fB\-\-optimized\-fetch\fR
 only fetch projects fixed to sha1 if revision does not
@@ -100,6 +101,17 @@ number of times to retry fetches on transient errors
 .TP
 \fB\-\-prune\fR
 delete refs that no longer exist on the remote
+(default)
+.TP
+\fB\-\-no\-prune\fR
+do not delete refs that no longer exist on the remote
+.TP
+\fB\-\-auto\-gc\fR
+run garbage collection on all synced projects
+.TP
+\fB\-\-no\-auto\-gc\fR
+do not run garbage collection on any projects
+(default)
 .TP
 \fB\-s\fR, \fB\-\-smart\-sync\fR
 smart sync using manifest from the latest known good
@@ -114,6 +126,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .SS repo Version options:
 .TP
 \fB\-\-no\-repo\-verify\fR
@@ -182,6 +207,9 @@ to a sha1 revision if the sha1 revision does not already exist locally.
 The \fB\-\-prune\fR option can be used to remove any refs that no longer exist on the
 remote.
 .PP
+The \fB\-\-auto\-gc\fR option can be used to trigger garbage collection on all projects.
+By default, repo does not run garbage collection.
+.PP
 SSH Connections
 .PP
 If at least one project remote URL uses an SSH connection (ssh://, git+ssh://,

man/repo-upload.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo upload" "Repo Manual"
+.TH REPO "1" "August 2022" "repo upload" "Repo Manual"
 .SH NAME
 repo \- repo upload - manual page for repo upload
 .SH SYNOPSIS
@@ -54,6 +54,9 @@ upload as a private change (deprecated; use \fB\-\-wip\fR)
 \fB\-w\fR, \fB\-\-wip\fR
 upload as a work\-in\-progress change
 .TP
+\fB\-r\fR, \fB\-\-ready\fR
+mark change as ready (clears work\-in\-progress setting)
+.TP
 \fB\-o\fR PUSH_OPTIONS, \fB\-\-push\-option\fR=\fI\,PUSH_OPTIONS\/\fR
 additional push options to transmit
 .TP
@@ -66,6 +69,12 @@ do everything except actually upload the CL
 \fB\-y\fR, \fB\-\-yes\fR
 answer yes to all safe prompts
 .TP
+\fB\-\-ignore\-untracked\-files\fR
+ignore untracked files in the working copy
+.TP
+\fB\-\-no\-ignore\-untracked\-files\fR
+always ask about untracked files in the working copy
+.TP
 \fB\-\-no\-cert\-checks\fR
 disable verifying ssl certs (unsafe)
 .SS Logging options:
@@ -75,6 +84,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .SS pre\-upload hooks:
 .TP
 \fB\-\-no\-verify\fR
@@ -105,6 +127,12 @@ respective list of users, and emails are sent to any new users. Users passed as
 \fB\-\-reviewers\fR must already be registered with the code review system, or the
 upload will fail.
 .PP
+While most normal Gerrit options have dedicated command line options, direct
+access to the Gerit options is available via \fB\-\-push\-options\fR. This is useful when
+Gerrit has newer functionality that repo upload doesn't yet support, or doesn't
+have plans to support. See the Push Options documentation for more details:
+https://gerrit\-review.googlesource.com/Documentation/user\-upload.html#push_options
+.PP
 Configuration
 .PP
 review.URL.autoupload:

man/repo-version.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo version" "Repo Manual"
+.TH REPO "1" "July 2022" "repo version" "Repo Manual"
 .SH NAME
 repo \- repo version - manual page for repo version
 .SH SYNOPSIS
@@ -20,5 +20,18 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help version` to view the detailed manual.

man/repo.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo" "Repo Manual"
+.TH REPO "1" "November 2022" "repo" "Repo Manual"
 .SH NAME
 repo \- repository management tool built on top of git
 .SH SYNOPSIS
@@ -25,6 +25,10 @@ control color usage: auto, always, never
 \fB\-\-trace\fR
 trace git command execution (REPO_TRACE=1)
 .TP
+\fB\-\-trace\-to\-stderr\fR
+trace outputs go to stderr in addition to
+\&.repo/TRACE_FILE
+.TP
 \fB\-\-trace\-python\fR
 trace python command execution
 .TP
@@ -43,7 +47,10 @@ filename of event log to append timeline to
 .TP
 \fB\-\-git\-trace2\-event\-log\fR=\fI\,GIT_TRACE2_EVENT_LOG\/\fR
 directory to write git trace2 event log to
-.SS "The complete list of recognized repo commands are:"
+.TP
+\fB\-\-submanifest\-path\fR=\fI\,REL_PATH\/\fR
+submanifest path
+.SS "The complete list of recognized repo commands is:"
 .TP
 abandon
 Permanently abandon a development branch

pager.py

@@ -56,8 +56,11 @@ def _PipePager(pager):
   global pager_process, old_stdout, old_stderr
   assert pager_process is None, "Only one active pager process at a time"
   # Create pager process, piping stdout/err into its stdin
-  pager_process = subprocess.Popen([pager], stdin=subprocess.PIPE, stdout=sys.stdout,
-                                   stderr=sys.stderr)
+  try:
+    pager_process = subprocess.Popen([pager], stdin=subprocess.PIPE, stdout=sys.stdout,
+                                     stderr=sys.stderr)
+  except FileNotFoundError:
+    sys.exit(f'fatal: cannot start pager "{pager}"')
   old_stdout = sys.stdout
   old_stderr = sys.stderr
   sys.stdout = pager_process.stdin

progress.py

@@ -15,7 +15,7 @@
 import os
 import sys
 from time import time
-from repo_trace import IsTrace
+from repo_trace import IsTraceToStderr
 
 _NOT_TTY = not os.isatty(2)
 
@@ -24,6 +24,11 @@ _NOT_TTY = not os.isatty(2)
 # column 0.
 CSI_ERASE_LINE = '\x1b[2K'
 
+# This will erase all content in the current line after the cursor.  This is
+# useful for partial updates & progress messages as the terminal can display
+# it better.
+CSI_ERASE_LINE_AFTER = '\x1b[K'
+
 
 def duration_str(total):
   """A less noisy timedelta.__str__.
@@ -75,7 +80,7 @@ class Progress(object):
   def update(self, inc=1, msg=''):
     self._done += inc
 
-    if _NOT_TTY or IsTrace():
+    if _NOT_TTY or IsTraceToStderr():
       return
 
     if not self._show:
@@ -85,10 +90,10 @@ class Progress(object):
         return
 
     if self._total <= 0:
-      sys.stderr.write('%s\r%s: %d,' % (
-          CSI_ERASE_LINE,
+      sys.stderr.write('\r%s: %d,%s' % (
           self._title,
-          self._done))
+          self._done,
+          CSI_ERASE_LINE_AFTER))
       sys.stderr.flush()
     else:
       p = (100 * self._done) / self._total
@@ -96,36 +101,36 @@ class Progress(object):
         jobs = '[%d job%s] ' % (self._active, 's' if self._active > 1 else '')
       else:
         jobs = ''
-      sys.stderr.write('%s\r%s: %2d%% %s(%d%s/%d%s)%s%s%s' % (
-          CSI_ERASE_LINE,
+      sys.stderr.write('\r%s: %2d%% %s(%d%s/%d%s)%s%s%s%s' % (
           self._title,
           p,
           jobs,
           self._done, self._units,
           self._total, self._units,
           ' ' if msg else '', msg,
+          CSI_ERASE_LINE_AFTER,
           '\n' if self._print_newline else ''))
       sys.stderr.flush()
 
   def end(self):
-    if _NOT_TTY or IsTrace() or not self._show:
+    if _NOT_TTY or IsTraceToStderr() or not self._show:
       return
 
     duration = duration_str(time() - self._start)
     if self._total <= 0:
-      sys.stderr.write('%s\r%s: %d, done in %s\n' % (
-          CSI_ERASE_LINE,
+      sys.stderr.write('\r%s: %d, done in %s%s\n' % (
           self._title,
           self._done,
-          duration))
+          duration,
+          CSI_ERASE_LINE_AFTER))
       sys.stderr.flush()
     else:
       p = (100 * self._done) / self._total
-      sys.stderr.write('%s\r%s: %3d%% (%d%s/%d%s), done in %s\n' % (
-          CSI_ERASE_LINE,
+      sys.stderr.write('\r%s: %3d%% (%d%s/%d%s), done in %s%s\n' % (
           self._title,
           p,
           self._done, self._units,
           self._total, self._units,
-          duration))
+          duration,
+          CSI_ERASE_LINE_AFTER))
       sys.stderr.flush()

release/update-manpages

@@ -18,85 +18,8 @@
 This is intended to be run before every official Repo release.
 """
 
-from pathlib import Path
-from functools import partial
-import argparse
-import multiprocessing
-import os
-import re
-import shutil
-import subprocess
 import sys
-import tempfile
 
-TOPDIR = Path(__file__).resolve().parent.parent
-MANDIR = TOPDIR.joinpath('man')
+import update_manpages
 
-# Load repo local modules.
-sys.path.insert(0, str(TOPDIR))
-from git_command import RepoSourceVersion
-import subcmds
-
-def worker(cmd, **kwargs):
-  subprocess.run(cmd, **kwargs)
-
-def main(argv):
-  parser = argparse.ArgumentParser(description=__doc__)
-  opts = parser.parse_args(argv)
-
-  if not shutil.which('help2man'):
-    sys.exit('Please install help2man to continue.')
-
-  # Let repo know we're generating man pages so it can avoid some dynamic
-  # behavior (like probing active number of CPUs).  We use a weird name &
-  # value to make it less likely for users to set this var themselves.
-  os.environ['_REPO_GENERATE_MANPAGES_'] = ' indeed! '
-
-  # "repo branch" is an alias for "repo branches".
-  del subcmds.all_commands['branch']
-  (MANDIR / 'repo-branch.1').write_text('.so man1/repo-branches.1')
-
-  version = RepoSourceVersion()
-  cmdlist = [['help2man', '-N', '-n', f'repo {cmd} - manual page for repo {cmd}',
-    '-S', f'repo {cmd}', '-m', 'Repo Manual', f'--version-string={version}',
-    '-o', MANDIR.joinpath(f'repo-{cmd}.1.tmp'), TOPDIR.joinpath('repo'),
-    '-h', f'help {cmd}'] for cmd in subcmds.all_commands]
-  cmdlist.append(['help2man', '-N', '-n', 'repository management tool built on top of git',
-    '-S', 'repo', '-m', 'Repo Manual', f'--version-string={version}',
-    '-o', MANDIR.joinpath('repo.1.tmp'), TOPDIR.joinpath('repo'),
-    '-h', '--help-all'])
-
-  with tempfile.TemporaryDirectory() as tempdir:
-    repo_dir = Path(tempdir) / '.repo'
-    repo_dir.mkdir()
-    (repo_dir / 'repo').symlink_to(TOPDIR)
-
-    # Run all cmd in parallel, and wait for them to finish.
-    with multiprocessing.Pool() as pool:
-      pool.map(partial(worker, cwd=tempdir, check=True), cmdlist)
-
-  regex = (
-      (r'(It was generated by help2man) [0-9.]+', '\g<1>.'),
-      (r'^\.IP\n(.*:)\n', '.SS \g<1>\n'),
-      (r'^\.PP\nDescription', '.SH DETAILS'),
-  )
-  for tmp_path in MANDIR.glob('*.1.tmp'):
-    path = tmp_path.parent / tmp_path.stem
-    old_data = path.read_text() if path.exists() else ''
-
-    data = tmp_path.read_text()
-    tmp_path.unlink()
-
-    for pattern, replacement in regex:
-      data = re.sub(pattern, replacement, data, flags=re.M)
-
-    # If the only thing that changed was the date, don't refresh.  This avoids
-    # a lot of noise when only one file actually updates.
-    old_data = re.sub(r'^(\.TH REPO "1" ")([^"]+)', r'\1', old_data, flags=re.M)
-    new_data = re.sub(r'^(\.TH REPO "1" ")([^"]+)', r'\1', data, flags=re.M)
-    if old_data != new_data:
-      path.write_text(data)
-
-
-if __name__ == '__main__':
-  sys.exit(main(sys.argv[1:]))
+sys.exit(update_manpages.main(sys.argv[1:]))

release/update_manpages.py

@@ -0,0 +1,119 @@
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helper tool for generating manual page for all repo commands.
+
+Most code lives in this module so it can be unittested.
+"""
+
+from pathlib import Path
+from functools import partial
+import argparse
+import multiprocessing
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+
+TOPDIR = Path(__file__).resolve().parent.parent
+MANDIR = TOPDIR.joinpath('man')
+
+# Load repo local modules.
+sys.path.insert(0, str(TOPDIR))
+from git_command import RepoSourceVersion
+import subcmds
+
+def worker(cmd, **kwargs):
+  subprocess.run(cmd, **kwargs)
+
+def main(argv):
+  parser = argparse.ArgumentParser(description=__doc__)
+  opts = parser.parse_args(argv)
+
+  if not shutil.which('help2man'):
+    sys.exit('Please install help2man to continue.')
+
+  # Let repo know we're generating man pages so it can avoid some dynamic
+  # behavior (like probing active number of CPUs).  We use a weird name &
+  # value to make it less likely for users to set this var themselves.
+  os.environ['_REPO_GENERATE_MANPAGES_'] = ' indeed! '
+
+  # "repo branch" is an alias for "repo branches".
+  del subcmds.all_commands['branch']
+  (MANDIR / 'repo-branch.1').write_text('.so man1/repo-branches.1')
+
+  version = RepoSourceVersion()
+  cmdlist = [['help2man', '-N', '-n', f'repo {cmd} - manual page for repo {cmd}',
+    '-S', f'repo {cmd}', '-m', 'Repo Manual', f'--version-string={version}',
+    '-o', MANDIR.joinpath(f'repo-{cmd}.1.tmp'), './repo',
+    '-h', f'help {cmd}'] for cmd in subcmds.all_commands]
+  cmdlist.append(['help2man', '-N', '-n', 'repository management tool built on top of git',
+    '-S', 'repo', '-m', 'Repo Manual', f'--version-string={version}',
+    '-o', MANDIR.joinpath('repo.1.tmp'), './repo',
+    '-h', '--help-all'])
+
+  with tempfile.TemporaryDirectory() as tempdir:
+    tempdir = Path(tempdir)
+    repo_dir = tempdir / '.repo'
+    repo_dir.mkdir()
+    (repo_dir / 'repo').symlink_to(TOPDIR)
+
+    # Create a repo wrapper using the active Python executable.  We can't pass
+    # this directly to help2man as it's too simple, so insert it via shebang.
+    data = (TOPDIR / 'repo').read_text(encoding='utf-8')
+    tempbin = tempdir / 'repo'
+    tempbin.write_text(f'#!{sys.executable}\n' + data, encoding='utf-8')
+    tempbin.chmod(0o755)
+
+    # Run all cmd in parallel, and wait for them to finish.
+    with multiprocessing.Pool() as pool:
+      pool.map(partial(worker, cwd=tempdir, check=True), cmdlist)
+
+  for tmp_path in MANDIR.glob('*.1.tmp'):
+    path = tmp_path.parent / tmp_path.stem
+    old_data = path.read_text() if path.exists() else ''
+
+    data = tmp_path.read_text()
+    tmp_path.unlink()
+
+    data = replace_regex(data)
+
+    # If the only thing that changed was the date, don't refresh.  This avoids
+    # a lot of noise when only one file actually updates.
+    old_data = re.sub(r'^(\.TH REPO "1" ")([^"]+)', r'\1', old_data, flags=re.M)
+    new_data = re.sub(r'^(\.TH REPO "1" ")([^"]+)', r'\1', data, flags=re.M)
+    if old_data != new_data:
+      path.write_text(data)
+
+
+def replace_regex(data):
+  """Replace semantically null regexes in the data.
+
+  Args:
+    data: manpage text.
+
+  Returns:
+    Updated manpage text.
+  """
+  regex = (
+      (r'(It was generated by help2man) [0-9.]+', r'\g<1>.'),
+      (r'^\033\[[0-9;]*m([^\033]*)\033\[m', r'\g<1>'),
+      (r'^\.IP\n(.*:)\n', r'.SS \g<1>\n'),
+      (r'^\.PP\nDescription', r'.SH DETAILS'),
+  )
+  for pattern, replacement in regex:
+    data = re.sub(pattern, replacement, data, flags=re.M)
+  return data

repo

@@ -149,7 +149,7 @@ if not REPO_REV:
 BUG_URL = 'https://bugs.chromium.org/p/gerrit/issues/entry?template=Repo+tool+issue'
 
 # increment this whenever we make important changes to this script
-VERSION = (2, 17)
+VERSION = (2, 32)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (2, 3)
@@ -265,7 +265,8 @@ else:
   urllib.error = urllib2
 
 
-home_dot_repo = os.path.expanduser('~/.repoconfig')
+repo_config_dir = os.getenv('REPO_CONFIG_DIR', os.path.expanduser('~'))
+home_dot_repo = os.path.join(repo_config_dir, '.repoconfig')
 gpg_dir = os.path.join(home_dot_repo, 'gnupg')
 
 
@@ -316,6 +317,10 @@ def InitParser(parser, gitc_init=False):
                    help='download the manifest as a static file '
                         'rather then create a git checkout of '
                         'the manifest repo')
+  group.add_option('--manifest-depth', type='int', default=0, metavar='DEPTH',
+                   help='create a shallow clone of the manifest repo with '
+                        'given depth (0 for full clone); see git clone '
+                        '(default: %default)')
 
   # Options that only affect manifest project, and not any of the projects
   # specified in the manifest itself.
@@ -325,9 +330,9 @@ def InitParser(parser, gitc_init=False):
   # want -c, so try to satisfy both as best we can.
   if not gitc_init:
     cbr_opts += ['-c']
-  group.add_option(*cbr_opts,
+  group.add_option(*cbr_opts, default=True,
                    dest='current_branch_only', action='store_true',
-                   help='fetch only current manifest branch from server')
+                   help='fetch only current manifest branch from server (default)')
   group.add_option('--no-current-branch',
                    dest='current_branch_only', action='store_false',
                    help='fetch all manifest branches from server')
@@ -372,7 +377,7 @@ def InitParser(parser, gitc_init=False):
                    help='filter for use with --partial-clone '
                         '[default: %default]')
   group.add_option('--use-superproject', action='store_true', default=None,
-                   help='use the manifest superproject to sync projects')
+                   help='use the manifest superproject to sync projects; implies -c')
   group.add_option('--no-use-superproject', action='store_false',
                    dest='use_superproject',
                    help='disable use of manifest superprojects')
@@ -382,6 +387,11 @@ def InitParser(parser, gitc_init=False):
   group.add_option('--no-clone-bundle',
                    dest='clone_bundle', action='store_false',
                    help='disable use of /clone.bundle on HTTP/HTTPS (default if --partial-clone)')
+  group.add_option('--git-lfs', action='store_true',
+                   help='enable Git LFS support')
+  group.add_option('--no-git-lfs',
+                   dest='git_lfs', action='store_false',
+                   help='disable Git LFS support')
 
   # Tool.
   group = parser.add_option_group('repo Version options')
@@ -438,8 +448,7 @@ def run_command(cmd, **kwargs):
     except UnicodeError:
       print('repo: warning: Invalid UTF-8 output:\ncmd: %r\n%r' % (cmd, output),
             file=sys.stderr)
-      # TODO(vapier): Once we require Python 3, use 'backslashreplace'.
-      return output.decode('utf-8', 'replace')
+      return output.decode('utf-8', 'backslashreplace')
 
   # Run & package the results.
   proc = subprocess.Popen(cmd, **kwargs)
@@ -607,17 +616,23 @@ def _Init(args, gitc_init=False):
   try:
     if not opt.quiet:
       print('Downloading Repo source from', url)
-    dst = os.path.abspath(os.path.join(repodir, S_repo))
+    dst_final = os.path.abspath(os.path.join(repodir, S_repo))
+    dst = dst_final + '.tmp'
+    shutil.rmtree(dst, ignore_errors=True)
     _Clone(url, dst, opt.clone_bundle, opt.quiet, opt.verbose)
 
     remote_ref, rev = check_repo_rev(dst, rev, opt.repo_verify, quiet=opt.quiet)
     _Checkout(dst, remote_ref, rev, opt.quiet)
 
     if not os.path.isfile(os.path.join(dst, 'repo')):
-      print("warning: '%s' does not look like a git-repo repository, is "
-            "REPO_URL set correctly?" % url, file=sys.stderr)
+      print("fatal: '%s' does not look like a git-repo repository, is "
+            "--repo-url set correctly?" % url, file=sys.stderr)
+      raise CloneFailure()
+
+    os.rename(dst, dst_final)
 
   except CloneFailure:
+    print('fatal: double check your --repo-rev setting.', file=sys.stderr)
     if opt.quiet:
       print('fatal: repo init failed; run without --quiet to see why',
             file=sys.stderr)
@@ -1311,6 +1326,7 @@ def main(orig_args):
         print("fatal: cloning the git-repo repository failed, will remove "
               "'%s' " % path, file=sys.stderr)
         shutil.rmtree(path, ignore_errors=True)
+        shutil.rmtree(path + '.tmp', ignore_errors=True)
         sys.exit(1)
       repo_main, rel_repo_dir = _FindRepo()
     else:

repo_trace.py

@@ -15,26 +15,152 @@
 """Logic for tracing repo interactions.
 
 Activated via `repo --trace ...` or `REPO_TRACE=1 repo ...`.
+
+Temporary: Tracing is always on. Set `REPO_TRACE=0` to turn off.
+To also include trace outputs in stderr do `repo --trace_to_stderr ...`
 """
 
 import sys
 import os
+import time
+import tempfile
+from contextlib import ContextDecorator
+
+import platform_utils
 
 # Env var to implicitly turn on tracing.
 REPO_TRACE = 'REPO_TRACE'
 
-_TRACE = os.environ.get(REPO_TRACE) == '1'
+# Temporarily set tracing to always on unless user expicitly sets to 0.
+_TRACE = os.environ.get(REPO_TRACE) != '0'
+_TRACE_TO_STDERR = False
+_TRACE_FILE = None
+_TRACE_FILE_NAME = 'TRACE_FILE'
+_MAX_SIZE = 70  # in MiB
+_NEW_COMMAND_SEP = '+++++++++++++++NEW COMMAND+++++++++++++++++++'
+
+
+def IsTraceToStderr():
+  """Whether traces are written to stderr."""
+  return _TRACE_TO_STDERR
 
 
 def IsTrace():
+  """Whether tracing is enabled."""
   return _TRACE
 
 
+def SetTraceToStderr():
+  """Enables tracing logging to stderr."""
+  global _TRACE_TO_STDERR
+  _TRACE_TO_STDERR = True
+
+
 def SetTrace():
+  """Enables tracing."""
   global _TRACE
   _TRACE = True
 
 
-def Trace(fmt, *args):
-  if IsTrace():
-    print(fmt % args, file=sys.stderr)
+def _SetTraceFile(quiet):
+  """Sets the trace file location."""
+  global _TRACE_FILE
+  _TRACE_FILE = _GetTraceFile(quiet)
+
+
+class Trace(ContextDecorator):
+  """Used to capture and save git traces."""
+
+  def _time(self):
+    """Generate nanoseconds of time in a py3.6 safe way"""
+    return int(time.time() * 1e+9)
+
+  def __init__(self, fmt, *args, first_trace=False, quiet=True):
+    """Initialize the object.
+
+    Args:
+      fmt: The format string for the trace.
+      *args: Arguments to pass to formatting.
+      first_trace: Whether this is the first trace of a `repo` invocation.
+      quiet: Whether to suppress notification of trace file location.
+    """
+    if not IsTrace():
+      return
+    self._trace_msg = fmt % args
+
+    if not _TRACE_FILE:
+      _SetTraceFile(quiet)
+
+    if first_trace:
+      _ClearOldTraces()
+      self._trace_msg = f'{_NEW_COMMAND_SEP} {self._trace_msg}'
+
+  def __enter__(self):
+    if not IsTrace():
+      return self
+
+    print_msg = f'PID: {os.getpid()} START: {self._time()} :{self._trace_msg}\n'
+
+    with open(_TRACE_FILE, 'a') as f:
+      print(print_msg, file=f)
+
+    if _TRACE_TO_STDERR:
+      print(print_msg, file=sys.stderr)
+
+    return self
+
+  def __exit__(self, *exc):
+    if not IsTrace():
+      return False
+
+    print_msg = f'PID: {os.getpid()} END: {self._time()} :{self._trace_msg}\n'
+
+    with open(_TRACE_FILE, 'a') as f:
+      print(print_msg, file=f)
+
+    if _TRACE_TO_STDERR:
+      print(print_msg, file=sys.stderr)
+
+    return False
+
+
+def _GetTraceFile(quiet):
+  """Get the trace file or create one."""
+  # TODO: refactor to pass repodir to Trace.
+  repo_dir = os.path.dirname(os.path.dirname(__file__))
+  trace_file = os.path.join(repo_dir, _TRACE_FILE_NAME)
+  if not quiet:
+    print(f'Trace outputs in {trace_file}', file=sys.stderr)
+  return trace_file
+
+
+def _ClearOldTraces():
+  """Clear the oldest commands if trace file is too big."""
+  try:
+    with open(_TRACE_FILE, 'r', errors='ignore') as f:
+      if os.path.getsize(f.name) / (1024 * 1024) <= _MAX_SIZE:
+        return
+      trace_lines = f.readlines()
+  except FileNotFoundError:
+    return
+
+  while sum(len(x) for x in trace_lines) / (1024 * 1024) > _MAX_SIZE:
+    for i, line in enumerate(trace_lines):
+      if 'END:' in line and _NEW_COMMAND_SEP in line:
+        trace_lines = trace_lines[i + 1:]
+        break
+    else:
+      # The last chunk is bigger than _MAX_SIZE, so just throw everything away.
+      trace_lines = []
+
+  while trace_lines and trace_lines[-1] == '\n':
+    trace_lines = trace_lines[:-1]
+  # Write to a temporary file with a unique name in the same filesystem
+  # before replacing the original trace file.
+  temp_dir, temp_prefix = os.path.split(_TRACE_FILE)
+  with tempfile.NamedTemporaryFile('w',
+                                   dir=temp_dir,
+                                   prefix=temp_prefix,
+                                   delete=False) as f:
+    f.writelines(trace_lines)
+  platform_utils.rename(f.name, _TRACE_FILE)

run_tests

@@ -15,47 +15,8 @@
 
 """Wrapper to run pytest with the right settings."""
 
-import errno
-import os
-import shutil
-import subprocess
 import sys
-
-
-def find_pytest():
-  """Try to locate a good version of pytest."""
-  # If we're in a virtualenv, assume that it's provided the right pytest.
-  if 'VIRTUAL_ENV' in os.environ:
-    return 'pytest'
-
-  # Use the Python 3 version if available.
-  ret = shutil.which('pytest-3')
-  if ret:
-    return ret
-
-  # Hopefully this is a Python 3 version.
-  ret = shutil.which('pytest')
-  if ret:
-    return ret
-
-  print('%s: unable to find pytest.' % (__file__,), file=sys.stderr)
-  print('%s: Try installing: sudo apt-get install python-pytest' % (__file__,),
-        file=sys.stderr)
-
-
-def main(argv):
-  """The main entry."""
-  # Add the repo tree to PYTHONPATH as the tests expect to be able to import
-  # modules directly.
-  pythonpath = os.path.dirname(os.path.realpath(__file__))
-  oldpythonpath = os.environ.get('PYTHONPATH', None)
-  if oldpythonpath is not None:
-    pythonpath += os.pathsep + oldpythonpath
-  os.environ['PYTHONPATH'] = pythonpath
-
-  pytest = find_pytest()
-  return subprocess.run([pytest] + argv, check=False).returncode
-
+import pytest
 
 if __name__ == '__main__':
-  sys.exit(main(sys.argv[1:]))
+  sys.exit(pytest.main(sys.argv[1:]))

run_tests.vpython3

@@ -0,0 +1,61 @@
+# This is a vpython "spec" file.
+#
+# Read more about `vpython` and how to modify this file here:
+#   https://chromium.googlesource.com/infra/infra/+/main/doc/users/vpython.md
+# List of available wheels:
+#   https://chromium.googlesource.com/infra/infra/+/main/infra/tools/dockerbuild/wheels.md
+
+python_version: "3.8"
+
+wheel: <
+  name: "infra/python/wheels/pytest-py3"
+  version: "version:6.2.2"
+>
+
+# Required by pytest==6.2.2
+wheel: <
+  name: "infra/python/wheels/py-py2_py3"
+  version: "version:1.10.0"
+>
+
+# Required by pytest==6.2.2
+wheel: <
+  name: "infra/python/wheels/iniconfig-py3"
+  version: "version:1.1.1"
+>
+
+# Required by pytest==6.2.2
+wheel: <
+  name: "infra/python/wheels/packaging-py2_py3"
+  version: "version:16.8"
+>
+
+# Required by pytest==6.2.2
+wheel: <
+  name: "infra/python/wheels/pluggy-py3"
+  version: "version:0.13.1"
+>
+
+# Required by pytest==6.2.2
+wheel: <
+  name: "infra/python/wheels/toml-py3"
+  version: "version:0.10.1"
+>
+
+# Required by pytest==6.2.2
+wheel: <
+  name: "infra/python/wheels/pyparsing-py3"
+  version: "version:3.0.7"
+>
+
+# Required by pytest==6.2.2
+wheel: <
+  name: "infra/python/wheels/attrs-py2_py3"
+  version: "version:21.4.0"
+>
+
+# Required by packaging==16.8
+wheel: <
+  name: "infra/python/wheels/six-py2_py3"
+  version: "version:1.16.0"
+>

setup.py

@@ -40,7 +40,7 @@ setuptools.setup(
     long_description_content_type='text/plain',
     url='https://gerrit.googlesource.com/git-repo/',
     project_urls={
-        'Bug Tracker': 'https://bugs.chromium.org/p/gerrit/issues/list?q=component:repo',
+        'Bug Tracker': 'https://bugs.chromium.org/p/gerrit/issues/list?q=component:Applications%3Erepo',
     },
     # https://pypi.org/classifiers/
     classifiers=[

ssh.py

@@ -182,28 +182,29 @@ class ProxyManager:
     # be important because we can't tell that that 'git@myhost.com' is the same
     # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file.
     check_command = command_base + ['-O', 'check']
-    try:
-      Trace(': %s', ' '.join(check_command))
-      check_process = subprocess.Popen(check_command,
-                                       stdout=subprocess.PIPE,
-                                       stderr=subprocess.PIPE)
-      check_process.communicate()  # read output, but ignore it...
-      isnt_running = check_process.wait()
+    with Trace('Call to ssh (check call): %s', ' '.join(check_command)):
+      try:
+        check_process = subprocess.Popen(check_command,
+                                        stdout=subprocess.PIPE,
+                                        stderr=subprocess.PIPE)
+        check_process.communicate()  # read output, but ignore it...
+        isnt_running = check_process.wait()
 
-      if not isnt_running:
-        # Our double-check found that the master _was_ infact running.  Add to
-        # the list of keys.
-        self._master_keys[key] = True
-        return True
-    except Exception:
-      # Ignore excpetions.  We we will fall back to the normal command and print
-      # to the log there.
-      pass
+        if not isnt_running:
+          # Our double-check found that the master _was_ infact running.  Add to
+          # the list of keys.
+          self._master_keys[key] = True
+          return True
+      except Exception:
+        # Ignore excpetions.  We we will fall back to the normal command and
+        # print to the log there.
+        pass
 
     command = command_base[:1] + ['-M', '-N'] + command_base[1:]
+    p = None
     try:
-      Trace(': %s', ' '.join(command))
-      p = subprocess.Popen(command)
+      with Trace('Call to ssh: %s', ' '.join(command)):
+        p = subprocess.Popen(command)
     except Exception as e:
       self._master_broken.value = True
       print('\nwarn: cannot enable ssh control master for %s:%s\n%s'

subcmds/abandon.py

@@ -69,7 +69,8 @@ It is equivalent to "git branch -D <branchname>".
     nb = args[0]
     err = defaultdict(list)
     success = defaultdict(list)
-    all_projects = self.GetProjects(args[1:])
+    all_projects = self.GetProjects(args[1:], all_manifests=not opt.this_manifest_only)
+    _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
 
     def _ProcessResults(_pool, pm, states):
       for (results, project) in states:
@@ -94,7 +95,7 @@ It is equivalent to "git branch -D <branchname>".
         err_msg = "error: cannot abandon %s" % br
         print(err_msg, file=sys.stderr)
         for proj in err[br]:
-          print(' ' * len(err_msg) + " | %s" % proj.relpath, file=sys.stderr)
+          print(' ' * len(err_msg) + " | %s" % _RelPath(proj), file=sys.stderr)
       sys.exit(1)
     elif not success:
       print('error: no project has local branch(es) : %s' % nb,
@@ -110,5 +111,5 @@ It is equivalent to "git branch -D <branchname>".
           result = "all project"
         else:
           result = "%s" % (
-              ('\n' + ' ' * width + '| ').join(p.relpath for p in success[br]))
+              ('\n' + ' ' * width + '| ').join(_RelPath(p) for p in success[br]))
         print("%s%s| %s\n" % (br, ' ' * (width - len(br)), result))

subcmds/branches.py

@@ -98,7 +98,7 @@ is shown, then the branch appears in all projects.
   PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
   def Execute(self, opt, args):
-    projects = self.GetProjects(args)
+    projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
     out = BranchColoring(self.manifest.manifestProject.config)
     all_branches = {}
     project_cnt = len(projects)
@@ -147,26 +147,28 @@ is shown, then the branch appears in all projects.
       hdr('%c%c %-*s' % (current, published, width, name))
       out.write(' |')
 
+      _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
       if in_cnt < project_cnt:
         fmt = out.write
         paths = []
         non_cur_paths = []
-        if i.IsSplitCurrent or (in_cnt < project_cnt - in_cnt):
+        if i.IsSplitCurrent or (in_cnt <= project_cnt - in_cnt):
           in_type = 'in'
           for b in i.projects:
+            relpath = _RelPath(b.project)
             if not i.IsSplitCurrent or b.current:
-              paths.append(b.project.relpath)
+              paths.append(relpath)
             else:
-              non_cur_paths.append(b.project.relpath)
+              non_cur_paths.append(relpath)
         else:
           fmt = out.notinproject
           in_type = 'not in'
           have = set()
           for b in i.projects:
-            have.add(b.project)
+            have.add(_RelPath(b.project))
           for p in projects:
-            if p not in have:
-              paths.append(p.relpath)
+            if _RelPath(p) not in have:
+              paths.append(_RelPath(p))
 
         s = ' %s %s' % (in_type, ', '.join(paths))
         if not i.IsSplitCurrent and (width + 7 + len(s) < 80):

subcmds/checkout.py

@@ -47,7 +47,7 @@ The command is equivalent to:
     nb = args[0]
     err = []
     success = []
-    all_projects = self.GetProjects(args[1:])
+    all_projects = self.GetProjects(args[1:], all_manifests=not opt.this_manifest_only)
 
     def _ProcessResults(_pool, pm, results):
       for status, project in results:

subcmds/cherry_pick.py

@@ -60,8 +60,10 @@ change id will be added.
                    capture_stderr=True)
     status = p.Wait()
 
-    print(p.stdout, file=sys.stdout)
-    print(p.stderr, file=sys.stderr)
+    if p.stdout:
+      print(p.stdout.strip(), file=sys.stdout)
+    if p.stderr:
+      print(p.stderr.strip(), file=sys.stderr)
 
     if status == 0:
       # The cherry-pick was applied correctly. We just need to edit the

subcmds/diff.py

@@ -35,22 +35,25 @@ to the Unix 'patch' command.
                  dest='absolute', action='store_true',
                  help='paths are relative to the repository root')
 
-  def _ExecuteOne(self, absolute, project):
+  def _ExecuteOne(self, absolute, local, project):
     """Obtains the diff for a specific project.
 
     Args:
       absolute: Paths are relative to the root.
+      local: a boolean, if True, the path is relative to the local
+             (sub)manifest.  If false, the path is relative to the
+             outermost manifest.
       project: Project to get status of.
 
     Returns:
       The status of the project.
     """
     buf = io.StringIO()
-    ret = project.PrintWorkTreeDiff(absolute, output_redir=buf)
+    ret = project.PrintWorkTreeDiff(absolute, output_redir=buf, local=local)
     return (ret, buf.getvalue())
 
   def Execute(self, opt, args):
-    all_projects = self.GetProjects(args)
+    all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     def _ProcessResults(_pool, _output, results):
       ret = 0
@@ -63,7 +66,7 @@ to the Unix 'patch' command.
 
     return self.ExecuteInParallel(
         opt.jobs,
-        functools.partial(self._ExecuteOne, opt.absolute),
+        functools.partial(self._ExecuteOne, opt.absolute, opt.this_manifest_only),
         all_projects,
         callback=_ProcessResults,
         ordered=True)

subcmds/diffmanifests.py

@@ -77,33 +77,35 @@ synced and their revisions won't be found.
                  metavar='<FORMAT>',
                  help='print the log using a custom git pretty format string')
 
-  def _printRawDiff(self, diff, pretty_format=None):
+  def _printRawDiff(self, diff, pretty_format=None, local=False):
+    _RelPath = lambda p: p.RelPath(local=local)
     for project in diff['added']:
-      self.printText("A %s %s" % (project.relpath, project.revisionExpr))
+      self.printText("A %s %s" % (_RelPath(project), project.revisionExpr))
       self.out.nl()
 
     for project in diff['removed']:
-      self.printText("R %s %s" % (project.relpath, project.revisionExpr))
+      self.printText("R %s %s" % (_RelPath(project), project.revisionExpr))
       self.out.nl()
 
     for project, otherProject in diff['changed']:
-      self.printText("C %s %s %s" % (project.relpath, project.revisionExpr,
+      self.printText("C %s %s %s" % (_RelPath(project), project.revisionExpr,
                                      otherProject.revisionExpr))
       self.out.nl()
       self._printLogs(project, otherProject, raw=True, color=False, pretty_format=pretty_format)
 
     for project, otherProject in diff['unreachable']:
-      self.printText("U %s %s %s" % (project.relpath, project.revisionExpr,
+      self.printText("U %s %s %s" % (_RelPath(project), project.revisionExpr,
                                      otherProject.revisionExpr))
       self.out.nl()
 
-  def _printDiff(self, diff, color=True, pretty_format=None):
+  def _printDiff(self, diff, color=True, pretty_format=None, local=False):
+    _RelPath = lambda p: p.RelPath(local=local)
     if diff['added']:
       self.out.nl()
       self.printText('added projects : \n')
       self.out.nl()
       for project in diff['added']:
-        self.printProject('\t%s' % (project.relpath))
+        self.printProject('\t%s' % (_RelPath(project)))
         self.printText(' at revision ')
         self.printRevision(project.revisionExpr)
         self.out.nl()
@@ -113,7 +115,17 @@ synced and their revisions won't be found.
       self.printText('removed projects : \n')
       self.out.nl()
       for project in diff['removed']:
-        self.printProject('\t%s' % (project.relpath))
+        self.printProject('\t%s' % (_RelPath(project)))
+        self.printText(' at revision ')
+        self.printRevision(project.revisionExpr)
+        self.out.nl()
+
+    if diff['missing']:
+      self.out.nl()
+      self.printText('missing projects : \n')
+      self.out.nl()
+      for project in diff['missing']:
+        self.printProject('\t%s' % (_RelPath(project)))
         self.printText(' at revision ')
         self.printRevision(project.revisionExpr)
         self.out.nl()
@@ -123,7 +135,7 @@ synced and their revisions won't be found.
       self.printText('changed projects : \n')
       self.out.nl()
       for project, otherProject in diff['changed']:
-        self.printProject('\t%s' % (project.relpath))
+        self.printProject('\t%s' % (_RelPath(project)))
         self.printText(' changed from ')
         self.printRevision(project.revisionExpr)
         self.printText(' to ')
@@ -138,7 +150,7 @@ synced and their revisions won't be found.
       self.printText('projects with unreachable revisions : \n')
       self.out.nl()
       for project, otherProject in diff['unreachable']:
-        self.printProject('\t%s ' % (project.relpath))
+        self.printProject('\t%s ' % (_RelPath(project)))
         self.printRevision(project.revisionExpr)
         self.printText(' or ')
         self.printRevision(otherProject.revisionExpr)
@@ -179,6 +191,9 @@ synced and their revisions won't be found.
   def ValidateOptions(self, opt, args):
     if not args or len(args) > 2:
       self.OptionParser.error('missing manifests to diff')
+    if opt.this_manifest_only is False:
+      raise self.OptionParser.error(
+          '`diffmanifest` only supports the current tree')
 
   def Execute(self, opt, args):
     self.out = _Coloring(self.client.globalConfig)
@@ -201,6 +216,8 @@ synced and their revisions won't be found.
 
     diff = manifest1.projectsDiff(manifest2)
     if opt.raw:
-      self._printRawDiff(diff, pretty_format=opt.pretty_format)
+      self._printRawDiff(diff, pretty_format=opt.pretty_format,
+                         local=opt.this_manifest_only)
     else:
-      self._printDiff(diff, color=opt.color, pretty_format=opt.pretty_format)
+      self._printDiff(diff, color=opt.color, pretty_format=opt.pretty_format,
+                      local=opt.this_manifest_only)

subcmds/download.py

@@ -48,7 +48,7 @@ If no project is specified try to use current directory as a project.
                  dest='ffonly', action='store_true',
                  help="force fast-forward merge")
 
-  def _ParseChangeIds(self, args):
+  def _ParseChangeIds(self, opt, args):
     if not args:
       self.Usage()
 
@@ -77,7 +77,7 @@ If no project is specified try to use current directory as a project.
                 ps_id = max(int(match.group(1)), ps_id)
         to_get.append((project, chg_id, ps_id))
       else:
-        projects = self.GetProjects([a])
+        projects = self.GetProjects([a], all_manifests=not opt.this_manifest_only)
         if len(projects) > 1:
           # If the cwd is one of the projects, assume they want that.
           try:
@@ -88,8 +88,8 @@ If no project is specified try to use current directory as a project.
             print('error: %s matches too many projects; please re-run inside '
                   'the project checkout.' % (a,), file=sys.stderr)
             for project in projects:
-              print('  %s/ @ %s' % (project.relpath, project.revisionExpr),
-                    file=sys.stderr)
+              print('  %s/ @ %s' % (project.RelPath(local=opt.this_manifest_only),
+                                    project.revisionExpr), file=sys.stderr)
             sys.exit(1)
         else:
           project = projects[0]
@@ -105,7 +105,7 @@ If no project is specified try to use current directory as a project.
         self.OptionParser.error('-x and --ff are mutually exclusive options')
 
   def Execute(self, opt, args):
-    for project, change_id, ps_id in self._ParseChangeIds(args):
+    for project, change_id, ps_id in self._ParseChangeIds(opt, args):
       dl = project.DownloadPatchSet(change_id, ps_id)
       if not dl:
         print('[%s] change %d/%d not found'

subcmds/forall.py

@@ -84,6 +84,11 @@ REPO_PROJECT is set to the unique name of the project.
 
 REPO_PATH is the path relative the the root of the client.
 
+REPO_OUTERPATH is the path of the sub manifest's root relative to the root of
+the client.
+
+REPO_INNERPATH is the path relative to the root of the sub manifest.
+
 REPO_REMOTE is the name of the remote system from the manifest.
 
 REPO_LREV is the name of the revision from the manifest, translated
@@ -168,6 +173,7 @@ without iterating through the remaining projects.
 
   def Execute(self, opt, args):
     cmd = [opt.command[0]]
+    all_trees = not opt.this_manifest_only
 
     shell = True
     if re.compile(r'^[a-z0-9A-Z_/\.-]+$').match(cmd[0]):
@@ -213,11 +219,11 @@ without iterating through the remaining projects.
       self.manifest.Override(smart_sync_manifest_path)
 
     if opt.regex:
-      projects = self.FindProjects(args)
+      projects = self.FindProjects(args, all_manifests=all_trees)
     elif opt.inverse_regex:
-      projects = self.FindProjects(args, inverse=True)
+      projects = self.FindProjects(args, inverse=True, all_manifests=all_trees)
     else:
-      projects = self.GetProjects(args, groups=opt.groups)
+      projects = self.GetProjects(args, groups=opt.groups, all_manifests=all_trees)
 
     os.environ['REPO_COUNT'] = str(len(projects))
 
@@ -289,7 +295,9 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
     env[name] = val
 
   setenv('REPO_PROJECT', project.name)
-  setenv('REPO_PATH', project.relpath)
+  setenv('REPO_OUTERPATH', project.manifest.path_prefix)
+  setenv('REPO_INNERPATH', project.relpath)
+  setenv('REPO_PATH', project.RelPath(local=opt.this_manifest_only))
   setenv('REPO_REMOTE', project.remote.name)
   try:
     # If we aren't in a fully synced state and we don't have the ref the manifest
@@ -320,7 +328,7 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
     output = ''
     if ((opt.project_header and opt.verbose)
             or not opt.project_header):
-      output = 'skipping %s/' % project.relpath
+      output = 'skipping %s/' % project.RelPath(local=opt.this_manifest_only)
     return (1, output)
 
   if opt.verbose:
@@ -344,7 +352,7 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
       if mirror:
         project_header_path = project.name
       else:
-        project_header_path = project.relpath
+        project_header_path = project.RelPath(local=opt.this_manifest_only)
       out.project('project %s/' % project_header_path)
       out.nl()
       buf.write(output)

subcmds/gitc_init.py

@@ -24,6 +24,7 @@ import wrapper
 
 class GitcInit(init.Init, GitcAvailableCommand):
   COMMON = True
+  MULTI_MANIFEST_SUPPORT = False
   helpSummary = "Initialize a GITC Client."
   helpUsage = """
 %prog [options] [client name]
@@ -67,7 +68,8 @@ use for this GITC client.
         sys.exit(1)
       manifest_file = opt.manifest_file
 
-    manifest = GitcManifest(self.repodir, gitc_client)
+    manifest = GitcManifest(self.repodir, os.path.join(self.client_dir,
+                                                       '.manifest'))
     manifest.Override(manifest_file)
     gitc_utils.generate_gitc_manifest(None, manifest)
     print('Please run `cd %s` to view your GITC client.' %

subcmds/grep.py

@@ -172,15 +172,16 @@ contain a line that matches both expressions:
     return (project, p.Wait(), p.stdout, p.stderr)
 
   @staticmethod
-  def _ProcessResults(full_name, have_rev, _pool, out, results):
+  def _ProcessResults(full_name, have_rev, opt, _pool, out, results):
     git_failed = False
     bad_rev = False
     have_match = False
+    _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
 
     for project, rc, stdout, stderr in results:
       if rc < 0:
         git_failed = True
-        out.project('--- project %s ---' % project.relpath)
+        out.project('--- project %s ---' % _RelPath(project))
         out.nl()
         out.fail('%s', stderr)
         out.nl()
@@ -192,7 +193,7 @@ contain a line that matches both expressions:
           if have_rev and 'fatal: ambiguous argument' in stderr:
             bad_rev = True
           else:
-            out.project('--- project %s ---' % project.relpath)
+            out.project('--- project %s ---' % _RelPath(project))
             out.nl()
             out.fail('%s', stderr.strip())
             out.nl()
@@ -208,13 +209,13 @@ contain a line that matches both expressions:
           rev, line = line.split(':', 1)
           out.write("%s", rev)
           out.write(':')
-          out.project(project.relpath)
+          out.project(_RelPath(project))
           out.write('/')
           out.write("%s", line)
           out.nl()
       elif full_name:
         for line in r:
-          out.project(project.relpath)
+          out.project(_RelPath(project))
           out.write('/')
           out.write("%s", line)
           out.nl()
@@ -239,7 +240,7 @@ contain a line that matches both expressions:
       cmd_argv.append(args[0])
       args = args[1:]
 
-    projects = self.GetProjects(args)
+    projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     full_name = False
     if len(projects) > 1:
@@ -259,7 +260,7 @@ contain a line that matches both expressions:
         opt.jobs,
         functools.partial(self._ExecuteOne, cmd_argv),
         projects,
-        callback=functools.partial(self._ProcessResults, full_name, have_rev),
+        callback=functools.partial(self._ProcessResults, full_name, have_rev, opt),
         output=out,
         ordered=True)
 

subcmds/help.py

@@ -53,7 +53,7 @@ Displays detailed usage information about a command.
     self.PrintAllCommandsBody()
 
   def PrintAllCommandsBody(self):
-    print('The complete list of recognized repo commands are:')
+    print('The complete list of recognized repo commands is:')
     commandNames = list(sorted(all_commands))
     self._PrintCommands(commandNames)
     print("See 'repo help <command>' for more information on a "

subcmds/info.py

@@ -61,10 +61,11 @@ class Info(PagedCommand):
 
     self.opt = opt
 
+    if not opt.this_manifest_only:
+      self.manifest = self.manifest.outer_client
     manifestConfig = self.manifest.manifestProject.config
     mergeBranch = manifestConfig.GetBranch("default").merge
-    manifestGroups = (manifestConfig.GetString('manifest.groups')
-                      or 'all,-notdefault')
+    manifestGroups = self.manifest.GetGroupsStr()
 
     self.heading("Manifest branch: ")
     if self.manifest.default.revisionExpr:
@@ -80,17 +81,17 @@ class Info(PagedCommand):
     self.printSeparator()
 
     if not opt.overview:
-      self.printDiffInfo(args)
+      self._printDiffInfo(opt, args)
     else:
-      self.printCommitOverview(args)
+      self._printCommitOverview(opt, args)
 
   def printSeparator(self):
     self.text("----------------------------")
     self.out.nl()
 
-  def printDiffInfo(self, args):
+  def _printDiffInfo(self, opt, args):
     # We let exceptions bubble up to main as they'll be well structured.
-    projs = self.GetProjects(args)
+    projs = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     for p in projs:
       self.heading("Project: ")
@@ -179,9 +180,9 @@ class Info(PagedCommand):
       self.text(" ".join(split[1:]))
       self.out.nl()
 
-  def printCommitOverview(self, args):
+  def _printCommitOverview(self, opt, args):
     all_branches = []
-    for project in self.GetProjects(args):
+    for project in self.GetProjects(args, all_manifests=not opt.this_manifest_only):
       br = [project.GetUploadableBranch(x)
             for x in project.GetBranches()]
       br = [x for x in br if x]
@@ -200,7 +201,7 @@ class Info(PagedCommand):
       if project != branch.project:
         project = branch.project
         self.out.nl()
-        self.headtext(project.relpath)
+        self.headtext(project.RelPath(local=opt.this_manifest_only))
         self.out.nl()
 
       commits = branch.commits

subcmds/init.py

@@ -13,26 +13,17 @@
 # limitations under the License.
 
 import os
-import platform
-import re
-import subprocess
 import sys
-import urllib.parse
 
 from color import Coloring
 from command import InteractiveCommand, MirrorSafeCommand
-from error import ManifestParseError
-from project import SyncBuffer
-from git_config import GitConfig
 from git_command import git_require, MIN_GIT_VERSION_SOFT, MIN_GIT_VERSION_HARD
-import fetch
-import git_superproject
-import platform_utils
 from wrapper import Wrapper
 
 
 class Init(InteractiveCommand, MirrorSafeCommand):
   COMMON = True
+  MULTI_MANIFEST_SUPPORT = True
   helpSummary = "Initialize a repo client checkout in the current directory"
   helpUsage = """
 %prog [options] [manifest url]
@@ -91,269 +82,65 @@ to update the working directory files.
 
   def _Options(self, p, gitc_init=False):
     Wrapper().InitParser(p, gitc_init=gitc_init)
+    m = p.add_option_group('Multi-manifest')
+    m.add_option('--outer-manifest', action='store_true', default=True,
+                 help='operate starting at the outermost manifest')
+    m.add_option('--no-outer-manifest', dest='outer_manifest',
+                 action='store_false', help='do not operate on outer manifests')
+    m.add_option('--this-manifest-only', action='store_true', default=None,
+                 help='only operate on this (sub)manifest')
+    m.add_option('--no-this-manifest-only', '--all-manifests',
+                 dest='this_manifest_only', action='store_false',
+                 help='operate on this manifest and its submanifests')
 
   def _RegisteredEnvironmentOptions(self):
     return {'REPO_MANIFEST_URL': 'manifest_url',
             'REPO_MIRROR_LOCATION': 'reference'}
 
-  def _CloneSuperproject(self, opt):
-    """Clone the superproject based on the superproject's url and branch.
+  def _SyncManifest(self, opt):
+    """Call manifestProject.Sync with arguments from opt.
 
     Args:
-      opt: Program options returned from optparse.  See _Options().
+      opt: options from optparse.
     """
-    superproject = git_superproject.Superproject(self.manifest,
-                                                 self.repodir,
-                                                 self.git_event_log,
-                                                 quiet=opt.quiet)
-    sync_result = superproject.Sync()
-    if not sync_result.success:
-      print('warning: git update of superproject failed, repo sync will not '
-            'use superproject to fetch source; while this error is not fatal, '
-            'and you can continue to run repo sync, please run repo init with '
-            'the --no-use-superproject option to stop seeing this warning',
-            file=sys.stderr)
-      if sync_result.fatal and opt.use_superproject is not None:
-        sys.exit(1)
-
-  def _SyncManifest(self, opt):
-    m = self.manifest.manifestProject
-    is_new = not m.Exists
-
-    # If repo has already been initialized, we take -u with the absence of
-    # --standalone-manifest to mean "transition to a standard repo set up",
-    # which necessitates starting fresh.
-    # If --standalone-manifest is set, we always tear everything down and start
-    # anew.
-    if not is_new:
-      was_standalone_manifest = m.config.GetString('manifest.standalone')
-      if was_standalone_manifest and not opt.manifest_url:
-        print('fatal: repo was initialized with a standlone manifest, '
-              'cannot be re-initialized without --manifest-url/-u')
-        sys.exit(1)
-
-      if opt.standalone_manifest or (
-          was_standalone_manifest and opt.manifest_url):
-        m.config.ClearCache()
-        if m.gitdir and os.path.exists(m.gitdir):
-          platform_utils.rmtree(m.gitdir)
-        if m.worktree and os.path.exists(m.worktree):
-          platform_utils.rmtree(m.worktree)
-
-    is_new = not m.Exists
-    if is_new:
-      if not opt.manifest_url:
-        print('fatal: manifest url is required.', file=sys.stderr)
-        sys.exit(1)
-
-      if not opt.quiet:
-        print('Downloading manifest from %s' %
-              (GitConfig.ForUser().UrlInsteadOf(opt.manifest_url),),
-              file=sys.stderr)
-
-      # The manifest project object doesn't keep track of the path on the
-      # server where this git is located, so let's save that here.
-      mirrored_manifest_git = None
-      if opt.reference:
-        manifest_git_path = urllib.parse.urlparse(opt.manifest_url).path[1:]
-        mirrored_manifest_git = os.path.join(opt.reference, manifest_git_path)
-        if not mirrored_manifest_git.endswith(".git"):
-          mirrored_manifest_git += ".git"
-        if not os.path.exists(mirrored_manifest_git):
-          mirrored_manifest_git = os.path.join(opt.reference,
-                                               '.repo/manifests.git')
-
-      m._InitGitDir(mirror_git=mirrored_manifest_git)
-
-    # If standalone_manifest is set, mark the project as "standalone" -- we'll
-    # still do much of the manifests.git set up, but will avoid actual syncs to
-    # a remote.
-    standalone_manifest = False
-    if opt.standalone_manifest:
-      standalone_manifest = True
-      m.config.SetString('manifest.standalone', opt.manifest_url)
-    elif not opt.manifest_url and not opt.manifest_branch:
-      # If -u is set and --standalone-manifest is not, then we're not in
-      # standalone mode. Otherwise, use config to infer what we were in the last
-      # init.
-      standalone_manifest = bool(m.config.GetString('manifest.standalone'))
-    if not standalone_manifest:
-      m.config.SetString('manifest.standalone', None)
-
-    self._ConfigureDepth(opt)
-
-    # Set the remote URL before the remote branch as we might need it below.
-    if opt.manifest_url:
-      r = m.GetRemote(m.remote.name)
-      r.url = opt.manifest_url
-      r.ResetFetch()
-      r.Save()
-
-    if not standalone_manifest:
-      if opt.manifest_branch:
-        if opt.manifest_branch == 'HEAD':
-          opt.manifest_branch = m.ResolveRemoteHead()
-          if opt.manifest_branch is None:
-            print('fatal: unable to resolve HEAD', file=sys.stderr)
-            sys.exit(1)
-        m.revisionExpr = opt.manifest_branch
-      else:
-        if is_new:
-          default_branch = m.ResolveRemoteHead()
-          if default_branch is None:
-            # If the remote doesn't have HEAD configured, default to master.
-            default_branch = 'refs/heads/master'
-          m.revisionExpr = default_branch
-        else:
-          m.PreSync()
-
-    groups = re.split(r'[,\s]+', opt.groups)
-    all_platforms = ['linux', 'darwin', 'windows']
-    platformize = lambda x: 'platform-' + x
-    if opt.platform == 'auto':
-      if (not opt.mirror and
-              not m.config.GetString('repo.mirror') == 'true'):
-        groups.append(platformize(platform.system().lower()))
-    elif opt.platform == 'all':
-      groups.extend(map(platformize, all_platforms))
-    elif opt.platform in all_platforms:
-      groups.append(platformize(opt.platform))
-    elif opt.platform != 'none':
-      print('fatal: invalid platform flag', file=sys.stderr)
-      sys.exit(1)
-
-    groups = [x for x in groups if x]
-    groupstr = ','.join(groups)
-    if opt.platform == 'auto' and groupstr == self.manifest.GetDefaultGroupsStr():
-      groupstr = None
-    m.config.SetString('manifest.groups', groupstr)
-
-    if opt.reference:
-      m.config.SetString('repo.reference', opt.reference)
-
-    if opt.dissociate:
-      m.config.SetBoolean('repo.dissociate', opt.dissociate)
-
-    if opt.worktree:
-      if opt.mirror:
-        print('fatal: --mirror and --worktree are incompatible',
-              file=sys.stderr)
-        sys.exit(1)
-      if opt.submodules:
-        print('fatal: --submodules and --worktree are incompatible',
-              file=sys.stderr)
-        sys.exit(1)
-      m.config.SetBoolean('repo.worktree', opt.worktree)
-      if is_new:
-        m.use_git_worktrees = True
-      print('warning: --worktree is experimental!', file=sys.stderr)
-
-    if opt.archive:
-      if is_new:
-        m.config.SetBoolean('repo.archive', opt.archive)
-      else:
-        print('fatal: --archive is only supported when initializing a new '
-              'workspace.', file=sys.stderr)
-        print('Either delete the .repo folder in this workspace, or initialize '
-              'in another location.', file=sys.stderr)
-        sys.exit(1)
-
-    if opt.mirror:
-      if is_new:
-        m.config.SetBoolean('repo.mirror', opt.mirror)
-      else:
-        print('fatal: --mirror is only supported when initializing a new '
-              'workspace.', file=sys.stderr)
-        print('Either delete the .repo folder in this workspace, or initialize '
-              'in another location.', file=sys.stderr)
-        sys.exit(1)
-
-    if opt.partial_clone is not None:
-      if opt.mirror:
-        print('fatal: --mirror and --partial-clone are mutually exclusive',
-              file=sys.stderr)
-        sys.exit(1)
-      m.config.SetBoolean('repo.partialclone', opt.partial_clone)
-      if opt.clone_filter:
-        m.config.SetString('repo.clonefilter', opt.clone_filter)
-    elif m.config.GetBoolean('repo.partialclone'):
-      opt.clone_filter = m.config.GetString('repo.clonefilter')
-    else:
-      opt.clone_filter = None
-
-    if opt.partial_clone_exclude is not None:
-      m.config.SetString('repo.partialcloneexclude', opt.partial_clone_exclude)
-
-    if opt.clone_bundle is None:
-      opt.clone_bundle = False if opt.partial_clone else True
-    else:
-      m.config.SetBoolean('repo.clonebundle', opt.clone_bundle)
-
-    if opt.submodules:
-      m.config.SetBoolean('repo.submodules', opt.submodules)
-
-    if opt.use_superproject is not None:
-      m.config.SetBoolean('repo.superproject', opt.use_superproject)
-
-    if standalone_manifest:
-      if is_new:
-        manifest_name = 'default.xml'
-        manifest_data = fetch.fetch_file(opt.manifest_url, verbose=opt.verbose)
-        dest = os.path.join(m.worktree, manifest_name)
-        os.makedirs(os.path.dirname(dest), exist_ok=True)
-        with open(dest, 'wb') as f:
-          f.write(manifest_data)
-      return
-
-    if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet, verbose=opt.verbose,
-                              clone_bundle=opt.clone_bundle,
-                              current_branch_only=opt.current_branch_only,
-                              tags=opt.tags, submodules=opt.submodules,
-                              clone_filter=opt.clone_filter,
-                              partial_clone_exclude=self.manifest.PartialCloneExclude):
-      r = m.GetRemote(m.remote.name)
-      print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
-
-      # Better delete the manifest git dir if we created it; otherwise next
-      # time (when user fixes problems) we won't go through the "is_new" logic.
-      if is_new:
-        platform_utils.rmtree(m.gitdir)
-      sys.exit(1)
-
-    if opt.manifest_branch:
-      m.MetaBranchSwitch(submodules=opt.submodules)
-
-    syncbuf = SyncBuffer(m.config)
-    m.Sync_LocalHalf(syncbuf, submodules=opt.submodules)
-    syncbuf.Finish()
-
-    if is_new or m.CurrentBranch is None:
-      if not m.StartBranch('default'):
-        print('fatal: cannot create default in manifest', file=sys.stderr)
-        sys.exit(1)
-
-  def _LinkManifest(self, name):
-    if not name:
-      print('fatal: manifest name (-m) is required.', file=sys.stderr)
-      sys.exit(1)
-
-    try:
-      self.manifest.Link(name)
-    except ManifestParseError as e:
-      print("fatal: manifest '%s' not available" % name, file=sys.stderr)
-      print('fatal: %s' % str(e), file=sys.stderr)
+    # Normally this value is set when instantiating the project, but the
+    # manifest project is special and is created when instantiating the
+    # manifest which happens before we parse options.
+    self.manifest.manifestProject.clone_depth = opt.manifest_depth
+    if not self.manifest.manifestProject.Sync(
+        manifest_url=opt.manifest_url,
+        manifest_branch=opt.manifest_branch,
+        standalone_manifest=opt.standalone_manifest,
+        groups=opt.groups,
+        platform=opt.platform,
+        mirror=opt.mirror,
+        dissociate=opt.dissociate,
+        reference=opt.reference,
+        worktree=opt.worktree,
+        submodules=opt.submodules,
+        archive=opt.archive,
+        partial_clone=opt.partial_clone,
+        clone_filter=opt.clone_filter,
+        partial_clone_exclude=opt.partial_clone_exclude,
+        clone_bundle=opt.clone_bundle,
+        git_lfs=opt.git_lfs,
+        use_superproject=opt.use_superproject,
+        verbose=opt.verbose,
+        current_branch_only=opt.current_branch_only,
+        tags=opt.tags,
+        depth=opt.depth,
+        git_event_log=self.git_event_log,
+        manifest_name=opt.manifest_name):
       sys.exit(1)
 
   def _Prompt(self, prompt, value):
-    print('%-10s [%s]: ' % (prompt, value), end='')
-    # TODO: When we require Python 3, use flush=True w/print above.
-    sys.stdout.flush()
+    print('%-10s [%s]: ' % (prompt, value), end='', flush=True)
     a = sys.stdin.readline().strip()
     if a == '':
       return value
     return a
 
-  def _ShouldConfigureUser(self, opt):
+  def _ShouldConfigureUser(self, opt, existing_checkout):
     gc = self.client.globalConfig
     mp = self.manifest.manifestProject
 
@@ -365,7 +152,7 @@ to update the working directory files.
       mp.config.SetString('user.name', gc.GetString('user.name'))
       mp.config.SetString('user.email', gc.GetString('user.email'))
 
-    if not opt.quiet:
+    if not opt.quiet and not existing_checkout or opt.verbose:
       print()
       print('Your identity is: %s <%s>' % (mp.config.GetString('user.name'),
                                            mp.config.GetString('user.email')))
@@ -384,9 +171,7 @@ to update the working directory files.
       if not opt.quiet:
         print()
       print('Your identity is: %s <%s>' % (name, email))
-      print('is this correct [y/N]? ', end='')
-      # TODO: When we require Python 3, use flush=True w/print above.
-      sys.stdout.flush()
+      print('is this correct [y/N]? ', end='', flush=True)
       a = sys.stdin.readline().strip().lower()
       if a in ('yes', 'y', 't', 'true'):
         break
@@ -428,48 +213,25 @@ to update the working directory files.
       out.printer(fg='black', attr=c)(' %-6s ', c)
     out.nl()
 
-    print('Enable color display in this user account (y/N)? ', end='')
-    # TODO: When we require Python 3, use flush=True w/print above.
-    sys.stdout.flush()
+    print('Enable color display in this user account (y/N)? ', end='', flush=True)
     a = sys.stdin.readline().strip().lower()
     if a in ('y', 'yes', 't', 'true', 'on'):
       gc.SetString('color.ui', 'auto')
 
-  def _ConfigureDepth(self, opt):
-    """Configure the depth we'll sync down.
-
-    Args:
-      opt: Options from optparse.  We care about opt.depth.
-    """
-    # Opt.depth will be non-None if user actually passed --depth to repo init.
-    if opt.depth is not None:
-      if opt.depth > 0:
-        # Positive values will set the depth.
-        depth = str(opt.depth)
-      else:
-        # Negative numbers will clear the depth; passing None to SetString
-        # will do that.
-        depth = None
-
-      # We store the depth in the main manifest project.
-      self.manifest.manifestProject.config.SetString('repo.depth', depth)
-
-  def _DisplayResult(self, opt):
+  def _DisplayResult(self):
     if self.manifest.IsMirror:
       init_type = 'mirror '
     else:
       init_type = ''
 
-    if not opt.quiet:
-      print()
-      print('repo %shas been initialized in %s' %
-            (init_type, self.manifest.topdir))
+    print()
+    print('repo %shas been initialized in %s' % (init_type, self.manifest.topdir))
 
     current_dir = os.getcwd()
     if current_dir != self.manifest.topdir:
       print('If this is not the directory in which you want to initialize '
             'repo, please run:')
-      print('   rm -r %s/.repo' % self.manifest.topdir)
+      print('   rm -r %s' % os.path.join(self.manifest.topdir, '.repo'))
       print('and try again.')
 
   def ValidateOptions(self, opt, args):
@@ -478,11 +240,19 @@ to update the working directory files.
 
     # Check this here, else manifest will be tagged "not new" and init won't be
     # possible anymore without removing the .repo/manifests directory.
-    if opt.archive and opt.mirror:
-      self.OptionParser.error('--mirror and --archive cannot be used together.')
+    if opt.mirror:
+      if opt.archive:
+        self.OptionParser.error('--mirror and --archive cannot be used '
+                                'together.')
+      if opt.use_superproject is not None:
+        self.OptionParser.error('--mirror and --use-superproject cannot be '
+                                'used together.')
+    if opt.archive and opt.use_superproject is not None:
+      self.OptionParser.error('--archive and --use-superproject cannot be used '
+                              'together.')
 
-    if opt.standalone_manifest and (
-        opt.manifest_branch or opt.manifest_name != 'default.xml'):
+    if opt.standalone_manifest and (opt.manifest_branch or
+                                    opt.manifest_name != 'default.xml'):
       self.OptionParser.error('--manifest-branch and --manifest-name cannot'
                               ' be used with --standalone-manifest.')
 
@@ -516,8 +286,12 @@ to update the working directory files.
     # Handle new --repo-rev requests.
     if opt.repo_rev:
       wrapper = Wrapper()
-      remote_ref, rev = wrapper.check_repo_rev(
-          rp.gitdir, opt.repo_rev, repo_verify=opt.repo_verify, quiet=opt.quiet)
+      try:
+        remote_ref, rev = wrapper.check_repo_rev(
+            rp.gitdir, opt.repo_rev, repo_verify=opt.repo_verify, quiet=opt.quiet)
+      except wrapper.CloneFailure:
+        print('fatal: double check your --repo-rev setting.', file=sys.stderr)
+        sys.exit(1)
       branch = rp.GetBranch('default')
       branch.merge = remote_ref
       rp.work_git.reset('--hard', rev)
@@ -527,15 +301,19 @@ to update the working directory files.
       # Older versions of git supported worktree, but had dangerous gc bugs.
       git_require((2, 15, 0), fail=True, msg='git gc worktree corruption')
 
-    self._SyncManifest(opt)
-    self._LinkManifest(opt.manifest_name)
+    # Provide a short notice that we're reinitializing an existing checkout.
+    # Sometimes developers might not realize that they're in one, or that
+    # repo doesn't do nested checkouts.
+    existing_checkout = self.manifest.manifestProject.Exists
+    if not opt.quiet and existing_checkout:
+      print('repo: reusing existing repo client checkout in', self.manifest.topdir)
 
-    if self.manifest.manifestProject.config.GetBoolean('repo.superproject'):
-      self._CloneSuperproject(opt)
+    self._SyncManifest(opt)
 
     if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
-      if opt.config_name or self._ShouldConfigureUser(opt):
+      if opt.config_name or self._ShouldConfigureUser(opt, existing_checkout):
         self._ConfigureUser(opt)
       self._ConfigureColor()
 
-    self._DisplayResult(opt)
+    if not opt.quiet:
+      self._DisplayResult()

subcmds/list.py

@@ -77,16 +77,17 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
       args: Positional args.  Can be a list of projects to list, or empty.
     """
     if not opt.regex:
-      projects = self.GetProjects(args, groups=opt.groups, missing_ok=opt.all)
+      projects = self.GetProjects(args, groups=opt.groups, missing_ok=opt.all,
+                                  all_manifests=not opt.this_manifest_only)
     else:
-      projects = self.FindProjects(args)
+      projects = self.FindProjects(args, all_manifests=not opt.this_manifest_only)
 
     def _getpath(x):
       if opt.fullpath:
         return x.worktree
       if opt.relative_to:
         return os.path.relpath(x.worktree, opt.relative_to)
-      return x.relpath
+      return x.RelPath(local=opt.this_manifest_only)
 
     lines = []
     for project in projects:

subcmds/manifest.py

@@ -75,7 +75,7 @@ to indicate the remote ref to push changes to via 'repo upload'.
     p.add_option('-o', '--output-file',
                  dest='output_file',
                  default='-',
-                 help='file to save the manifest to',
+                 help='file to save the manifest to. (Filename prefix for multi-tree.)',
                  metavar='-|NAME.xml')
 
   def _Output(self, opt):
@@ -83,36 +83,45 @@ to indicate the remote ref to push changes to via 'repo upload'.
     if opt.manifest_name:
       self.manifest.Override(opt.manifest_name, False)
 
-    if opt.output_file == '-':
-      fd = sys.stdout
-    else:
-      fd = open(opt.output_file, 'w')
+    for manifest in self.ManifestList(opt):
+      output_file = opt.output_file
+      if output_file == '-':
+        fd = sys.stdout
+      else:
+        if manifest.path_prefix:
+          output_file = f'{opt.output_file}:{manifest.path_prefix.replace("/", "%2f")}'
+        fd = open(output_file, 'w')
 
-    self.manifest.SetUseLocalManifests(not opt.ignore_local_manifests)
+      manifest.SetUseLocalManifests(not opt.ignore_local_manifests)
 
-    if opt.json:
-      print('warning: --json is experimental!', file=sys.stderr)
-      doc = self.manifest.ToDict(peg_rev=opt.peg_rev,
-                                 peg_rev_upstream=opt.peg_rev_upstream,
-                                 peg_rev_dest_branch=opt.peg_rev_dest_branch)
+      if opt.json:
+        print('warning: --json is experimental!', file=sys.stderr)
+        doc = manifest.ToDict(peg_rev=opt.peg_rev,
+                                   peg_rev_upstream=opt.peg_rev_upstream,
+                                   peg_rev_dest_branch=opt.peg_rev_dest_branch)
+
+        json_settings = {
+            # JSON style guide says Uunicode characters are fully allowed.
+            'ensure_ascii': False,
+            # We use 2 space indent to match JSON style guide.
+            'indent': 2 if opt.pretty else None,
+            'separators': (',', ': ') if opt.pretty else (',', ':'),
+            'sort_keys': True,
+        }
+        fd.write(json.dumps(doc, **json_settings))
+      else:
+        manifest.Save(fd,
+                      peg_rev=opt.peg_rev,
+                      peg_rev_upstream=opt.peg_rev_upstream,
+                      peg_rev_dest_branch=opt.peg_rev_dest_branch)
+      if output_file != '-':
+        fd.close()
+        if manifest.path_prefix:
+          print(f'Saved {manifest.path_prefix} submanifest to {output_file}',
+                file=sys.stderr)
+        else:
+          print(f'Saved manifest to {output_file}', file=sys.stderr)
 
-      json_settings = {
-          # JSON style guide says Uunicode characters are fully allowed.
-          'ensure_ascii': False,
-          # We use 2 space indent to match JSON style guide.
-          'indent': 2 if opt.pretty else None,
-          'separators': (',', ': ') if opt.pretty else (',', ':'),
-          'sort_keys': True,
-      }
-      fd.write(json.dumps(doc, **json_settings))
-    else:
-      self.manifest.Save(fd,
-                         peg_rev=opt.peg_rev,
-                         peg_rev_upstream=opt.peg_rev_upstream,
-                         peg_rev_dest_branch=opt.peg_rev_dest_branch)
-    fd.close()
-    if opt.output_file != '-':
-      print('Saved manifest to %s' % opt.output_file, file=sys.stderr)
 
   def ValidateOptions(self, opt, args):
     if args:

subcmds/overview.py

@@ -47,7 +47,7 @@ are displayed.
 
   def Execute(self, opt, args):
     all_branches = []
-    for project in self.GetProjects(args):
+    for project in self.GetProjects(args, all_manifests=not opt.this_manifest_only):
       br = [project.GetUploadableBranch(x)
             for x in project.GetBranches()]
       br = [x for x in br if x]
@@ -76,7 +76,7 @@ are displayed.
       if project != branch.project:
         project = branch.project
         out.nl()
-        out.project('project %s/' % project.relpath)
+        out.project('project %s/' % project.RelPath(local=opt.this_manifest_only))
         out.nl()
 
       commits = branch.commits

subcmds/prune.py

@@ -31,7 +31,7 @@ class Prune(PagedCommand):
     return project.PruneHeads()
 
   def Execute(self, opt, args):
-    projects = self.GetProjects(args)
+    projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     # NB: Should be able to refactor this module to display summary as results
     # come back from children.
@@ -63,7 +63,7 @@ class Prune(PagedCommand):
       if project != branch.project:
         project = branch.project
         out.nl()
-        out.project('project %s/' % project.relpath)
+        out.project('project %s/' % project.RelPath(local=opt.this_manifest_only))
         out.nl()
 
       print('%s %-33s ' % (

subcmds/rebase.py

@@ -69,7 +69,7 @@ branch but need to incorporate new upstream changes "underneath" them.
                       'consistent if you previously synced to a manifest)')
 
   def Execute(self, opt, args):
-    all_projects = self.GetProjects(args)
+    all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
     one_project = len(all_projects) == 1
 
     if opt.interactive and not one_project:
@@ -98,6 +98,7 @@ branch but need to incorporate new upstream changes "underneath" them.
     config = self.manifest.manifestProject.config
     out = RebaseColoring(config)
     out.redirect(sys.stdout)
+    _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
 
     ret = 0
     for project in all_projects:
@@ -107,7 +108,7 @@ branch but need to incorporate new upstream changes "underneath" them.
       cb = project.CurrentBranch
       if not cb:
         if one_project:
-          print("error: project %s has a detached HEAD" % project.relpath,
+          print("error: project %s has a detached HEAD" % _RelPath(project),
                 file=sys.stderr)
           return 1
         # ignore branches with detatched HEADs
@@ -117,7 +118,7 @@ branch but need to incorporate new upstream changes "underneath" them.
       if not upbranch.LocalMerge:
         if one_project:
           print("error: project %s does not track any remote branches"
-                % project.relpath, file=sys.stderr)
+                % _RelPath(project), file=sys.stderr)
           return 1
         # ignore branches without remotes
         continue
@@ -130,7 +131,7 @@ branch but need to incorporate new upstream changes "underneath" them.
       args.append(upbranch.LocalMerge)
 
       out.project('project %s: rebasing %s -> %s',
-                  project.relpath, cb, upbranch.LocalMerge)
+                  _RelPath(project), cb, upbranch.LocalMerge)
       out.nl()
       out.flush()
 

subcmds/selfupdate.py

@@ -51,7 +51,7 @@ need to be performed by an end-user.
       _PostRepoUpgrade(self.manifest)
 
     else:
-      if not rp.Sync_NetworkHalf():
+      if not rp.Sync_NetworkHalf().success:
         print("error: can't update repo", file=sys.stderr)
         sys.exit(1)
 

subcmds/stage.py

@@ -50,7 +50,9 @@ The '%prog' command stages files to prepare the next commit.
       self.Usage()
 
   def _Interactive(self, opt, args):
-    all_projects = [p for p in self.GetProjects(args) if p.IsDirty()]
+    all_projects = [
+        p for p in self.GetProjects(args, all_manifests=not opt.this_manifest_only)
+        if p.IsDirty()]
     if not all_projects:
       print('no projects have uncommitted modifications', file=sys.stderr)
       return
@@ -62,7 +64,8 @@ The '%prog' command stages files to prepare the next commit.
 
       for i in range(len(all_projects)):
         project = all_projects[i]
-        out.write('%3d:    %s', i + 1, project.relpath + '/')
+        out.write('%3d:    %s', i + 1,
+                  project.RelPath(local=opt.this_manifest_only) + '/')
         out.nl()
       out.nl()
 
@@ -72,6 +75,7 @@ The '%prog' command stages files to prepare the next commit.
       out.nl()
 
       out.prompt('project> ')
+      out.flush()
       try:
         a = sys.stdin.readline()
       except KeyboardInterrupt:
@@ -99,7 +103,9 @@ The '%prog' command stages files to prepare the next commit.
           _AddI(all_projects[a_index - 1])
           continue
 
-      projects = [p for p in all_projects if a in [p.name, p.relpath]]
+      projects = [
+          p for p in all_projects
+          if a in [p.name, p.RelPath(local=opt.this_manifest_only)]]
       if len(projects) == 1:
         _AddI(projects[0])
         continue

subcmds/start.py

@@ -84,7 +84,8 @@ revision specified in the manifest.
         projects = ['.']  # start it in the local project by default
 
     all_projects = self.GetProjects(projects,
-                                    missing_ok=bool(self.gitc_manifest))
+                                    missing_ok=bool(self.gitc_manifest),
+                                    all_manifests=not opt.this_manifest_only)
 
     # This must happen after we find all_projects, since GetProjects may need
     # the local directory, which will disappear once we save the GITC manifest.
@@ -137,6 +138,6 @@ revision specified in the manifest.
 
     if err:
       for p in err:
-        print("error: %s/: cannot start %s" % (p.relpath, nb),
+        print("error: %s/: cannot start %s" % (p.RelPath(local=opt.this_manifest_only), nb),
               file=sys.stderr)
       sys.exit(1)

subcmds/status.py

@@ -83,7 +83,7 @@ the following meanings:
                  dest='orphans', action='store_true',
                  help="include objects in working directory outside of repo projects")
 
-  def _StatusHelper(self, quiet, project):
+  def _StatusHelper(self, quiet, local, project):
     """Obtains the status for a specific project.
 
     Obtains the status for a project, redirecting the output to
@@ -91,13 +91,17 @@ the following meanings:
 
     Args:
       quiet: Where to output the status.
+      local: a boolean, if True, the path is relative to the local
+             (sub)manifest.  If false, the path is relative to the
+             outermost manifest.
       project: Project to get status of.
 
     Returns:
       The status of the project.
     """
     buf = io.StringIO()
-    ret = project.PrintWorkTreeStatus(quiet=quiet, output_redir=buf)
+    ret = project.PrintWorkTreeStatus(quiet=quiet, output_redir=buf,
+                                      local=local)
     return (ret, buf.getvalue())
 
   def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring):
@@ -117,7 +121,7 @@ the following meanings:
       outstring.append(''.join([status_header, item, '/']))
 
   def Execute(self, opt, args):
-    all_projects = self.GetProjects(args)
+    all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     def _ProcessResults(_pool, _output, results):
       ret = 0
@@ -130,7 +134,7 @@ the following meanings:
 
     counter = self.ExecuteInParallel(
         opt.jobs,
-        functools.partial(self._StatusHelper, opt.quiet),
+        functools.partial(self._StatusHelper, opt.quiet, opt.this_manifest_only),
         all_projects,
         callback=_ProcessResults,
         ordered=True)
@@ -141,9 +145,10 @@ the following meanings:
     if opt.orphans:
       proj_dirs = set()
       proj_dirs_parents = set()
-      for project in self.GetProjects(None, missing_ok=True):
-        proj_dirs.add(project.relpath)
-        (head, _tail) = os.path.split(project.relpath)
+      for project in self.GetProjects(None, missing_ok=True, all_manifests=not opt.this_manifest_only):
+        relpath = project.RelPath(local=opt.this_manifest_only)
+        proj_dirs.add(relpath)
+        (head, _tail) = os.path.split(relpath)
         while head != "":
           proj_dirs_parents.add(head)
           (head, _tail) = os.path.split(head)

subcmds/upload.py

@@ -17,6 +17,7 @@ import functools
 import optparse
 import re
 import sys
+from typing import List
 
 from command import DEFAULT_LOCAL_JOBS, InteractiveCommand
 from editor import Editor
@@ -24,21 +25,54 @@ from error import UploadError
 from git_command import GitCommand
 from git_refs import R_HEADS
 from hooks import RepoHook
+from project import ReviewableBranch
 
 
-UNUSUAL_COMMIT_THRESHOLD = 5
+_DEFAULT_UNUSUAL_COMMIT_THRESHOLD = 5
 
 
-def _ConfirmManyUploads(multiple_branches=False):
-  if multiple_branches:
-    print('ATTENTION: One or more branches has an unusually high number '
-          'of commits.')
-  else:
-    print('ATTENTION: You are uploading an unusually high number of commits.')
-  print('YOU PROBABLY DO NOT MEAN TO DO THIS. (Did you rebase across '
-        'branches?)')
-  answer = input("If you are sure you intend to do this, type 'yes': ").strip()
-  return answer == "yes"
+def _VerifyPendingCommits(branches: List[ReviewableBranch]) -> bool:
+  """Perform basic safety checks on the given set of branches.
+
+  Ensures that each branch does not have a "large" number of commits
+  and, if so, prompts the user to confirm they want to proceed with
+  the upload.
+
+  Returns true if all branches pass the safety check or the user
+  confirmed. Returns false if the upload should be aborted.
+  """
+
+  # Determine if any branch has a suspicious number of commits.
+  many_commits = False
+  for branch in branches:
+    # Get the user's unusual threshold for the branch.
+    #
+    # Each branch may be configured to have a different threshold.
+    remote = branch.project.GetBranch(branch.name).remote
+    key = f'review.{remote.review}.uploadwarningthreshold'
+    threshold = branch.project.config.GetInt(key)
+    if threshold is None:
+      threshold = _DEFAULT_UNUSUAL_COMMIT_THRESHOLD
+
+    # If the branch has more commits than the threshold, show a warning.
+    if len(branch.commits) > threshold:
+      many_commits = True
+      break
+
+  # If any branch has many commits, prompt the user.
+  if many_commits:
+    if len(branches) > 1:
+      print('ATTENTION: One or more branches has an unusually high number '
+            'of commits.')
+    else:
+      print('ATTENTION: You are uploading an unusually high number of commits.')
+    print('YOU PROBABLY DO NOT MEAN TO DO THIS. (Did you rebase across '
+          'branches?)')
+    answer = input(
+        "If you are sure you intend to do this, type 'yes': ").strip()
+    return answer == 'yes'
+
+  return True
 
 
 def _die(fmt, *args):
@@ -78,6 +112,13 @@ added to the respective list of users, and emails are sent to any
 new users.  Users passed as --reviewers must already be registered
 with the code review system, or the upload will fail.
 
+While most normal Gerrit options have dedicated command line options,
+direct access to the Gerit options is available via --push-options.
+This is useful when Gerrit has newer functionality that %prog doesn't
+yet support, or doesn't have plans to support.  See the Push Options
+documentation for more details:
+https://gerrit-review.googlesource.com/Documentation/user-upload.html#push_options
+
 # Configuration
 
 review.URL.autoupload:
@@ -142,6 +183,13 @@ review.URL.uploadnotify:
 Control e-mail notifications when uploading.
 https://gerrit-review.googlesource.com/Documentation/user-upload.html#notify
 
+review.URL.uploadwarningthreshold:
+
+Repo will warn you if you are attempting to upload a large number
+of commits in one or more branches. By default, the threshold
+is five commits. This option allows you to override the warning
+threshold to a different value.
+
 # References
 
 Gerrit Code Review:  https://www.gerritcodereview.com/
@@ -190,6 +238,9 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     p.add_option('-w', '--wip',
                  action='store_true', dest='wip', default=False,
                  help='upload as a work-in-progress change')
+    p.add_option('-r', '--ready',
+                 action='store_true', default=False,
+                 help='mark change as ready (clears work-in-progress setting)')
     p.add_option('-o', '--push-option',
                  type='string', action='append', dest='push_options',
                  default=[],
@@ -204,6 +255,12 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     p.add_option('-y', '--yes',
                  default=False, action='store_true',
                  help='answer yes to all safe prompts')
+    p.add_option('--ignore-untracked-files',
+                 action='store_true', default=False,
+                 help='ignore untracked files in the working copy')
+    p.add_option('--no-ignore-untracked-files',
+                 dest='ignore_untracked_files', action='store_false',
+                 help='always ask about untracked files in the working copy')
     p.add_option('--no-cert-checks',
                  dest='validate_certs', action='store_false', default=True,
                  help='disable verifying ssl certs (unsafe)')
@@ -226,7 +283,8 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
 
       destination = opt.dest_branch or project.dest_branch or project.revisionExpr
       print('Upload project %s/ to remote branch %s%s:' %
-            (project.relpath, destination, ' (private)' if opt.private else ''))
+            (project.RelPath(local=opt.this_manifest_only), destination,
+             ' (private)' if opt.private else ''))
       print('  branch %s (%2d commit%s, %s):' % (
           name,
           len(commit_list),
@@ -235,25 +293,22 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
       for commit in commit_list:
         print('         %s' % commit)
 
-      print('to %s (y/N)? ' % remote.review, end='')
-      # TODO: When we require Python 3, use flush=True w/print above.
-      sys.stdout.flush()
+      print('to %s (y/N)? ' % remote.review, end='', flush=True)
       if opt.yes:
         print('<--yes>')
         answer = True
       else:
         answer = sys.stdin.readline().strip().lower()
         answer = answer in ('y', 'yes', '1', 'true', 't')
+      if not answer:
+        _die("upload aborted by user")
 
-    if answer:
-      if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD:
-        answer = _ConfirmManyUploads()
-
-    if answer:
-      self._UploadAndReport(opt, [branch], people)
-    else:
+    # Perform some basic safety checks prior to uploading.
+    if not opt.yes and not _VerifyPendingCommits([branch]):
       _die("upload aborted by user")
 
+    self._UploadAndReport(opt, [branch], people)
+
   def _MultipleBranches(self, opt, pending, people):
     projects = {}
     branches = {}
@@ -261,8 +316,9 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     script = []
     script.append('# Uncomment the branches to upload:')
     for project, avail in pending:
+      project_path = project.RelPath(local=opt.this_manifest_only)
       script.append('#')
-      script.append('# project %s/:' % project.relpath)
+      script.append(f'# project {project_path}/:')
 
       b = {}
       for branch in avail:
@@ -285,8 +341,8 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
           script.append('#         %s' % commit)
         b[name] = branch
 
-      projects[project.relpath] = project
-      branches[project.name] = b
+      projects[project_path] = project
+      branches[project_path] = b
     script.append('')
 
     script = Editor.EditString("\n".join(script)).split("\n")
@@ -311,21 +367,17 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
         name = m.group(1)
         if not project:
           _die('project for branch %s not in script', name)
-        branch = branches[project.name].get(name)
+        project_path = project.RelPath(local=opt.this_manifest_only)
+        branch = branches[project_path].get(name)
         if not branch:
-          _die('branch %s not in %s', name, project.relpath)
+          _die('branch %s not in %s', name, project_path)
         todo.append(branch)
     if not todo:
       _die("nothing uncommented for upload")
 
-    many_commits = False
-    for branch in todo:
-      if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD:
-        many_commits = True
-        break
-    if many_commits:
-      if not _ConfirmManyUploads(multiple_branches=True):
-        _die("upload aborted by user")
+    # Perform some basic safety checks prior to uploading.
+    if not opt.yes and not _VerifyPendingCommits(todo):
+      _die("upload aborted by user")
 
     self._UploadAndReport(opt, todo, people)
 
@@ -369,6 +421,10 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
 
         # Check if there are local changes that may have been forgotten
         changes = branch.project.UncommitedFiles()
+        if opt.ignore_untracked_files:
+          untracked = set(branch.project.UntrackedFiles())
+          changes = [x for x in changes if x not in untracked]
+
         if changes:
           key = 'review.%s.autoupload' % branch.project.remote.review
           answer = branch.project.config.GetBoolean(key)
@@ -379,9 +435,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
             print('Uncommitted changes in %s (did you forget to amend?):'
                   % branch.project.name)
             print('\n'.join(changes))
-            print('Continue uploading? (y/N) ', end='')
-            # TODO: When we require Python 3, use flush=True w/print above.
-            sys.stdout.flush()
+            print('Continue uploading? (y/N) ', end='', flush=True)
             if opt.yes:
               print('<--yes>')
               a = 'yes'
@@ -420,12 +474,6 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
         labels = set(_ExpandCommaList(branch.project.config.GetString(key)))
         for label in opt.labels:
           labels.update(_ExpandCommaList(label))
-        # Basic sanity check on label syntax.
-        for label in labels:
-          if not re.match(r'^.+[+-][0-9]+$', label):
-            print('repo: error: invalid label syntax "%s": labels use forms '
-                  'like CodeReview+1 or Verified-1' % (label,), file=sys.stderr)
-            sys.exit(1)
 
         # Handle e-mail notifications.
         if opt.notify is False:
@@ -436,19 +484,24 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
 
         destination = opt.dest_branch or branch.project.dest_branch
 
-        # Make sure our local branch is not setup to track a different remote branch
-        merge_branch = self._GetMergeBranch(branch.project)
-        if destination:
+        if branch.project.dest_branch and not opt.dest_branch:
+
+          merge_branch = self._GetMergeBranch(
+            branch.project, local_branch=branch.name)
+
           full_dest = destination
           if not full_dest.startswith(R_HEADS):
             full_dest = R_HEADS + full_dest
 
-          if not opt.dest_branch and merge_branch and merge_branch != full_dest:
-            print('merge branch %s does not match destination branch %s'
-                  % (merge_branch, full_dest))
+          # If the merge branch of the local branch is different from the
+          # project's revision AND destination, this might not be intentional.
+          if (merge_branch and merge_branch != branch.project.revisionExpr
+              and merge_branch != full_dest):
+            print(f'For local branch {branch.name}: merge branch '
+                  f'{merge_branch} does not match destination branch '
+                  f'{destination}')
             print('skipping upload.')
-            print('Please use `--destination %s` if this is intentional'
-                  % destination)
+            print(f'Please use `--destination {destination}` if this is intentional')
             branch.uploaded = False
             continue
 
@@ -460,6 +513,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
                                private=opt.private,
                                notify=notify,
                                wip=opt.wip,
+                               ready=opt.ready,
                                dest_branch=destination,
                                validate_certs=opt.validate_certs,
                                push_options=opt.push_options)
@@ -481,7 +535,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
           else:
             fmt = '\n       (%s)'
           print(('[FAILED] %-15s %-15s' + fmt) % (
-              branch.project.relpath + '/',
+              branch.project.RelPath(local=opt.this_manifest_only) + '/',
               branch.name,
               str(branch.error)),
               file=sys.stderr)
@@ -490,20 +544,21 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     for branch in todo:
       if branch.uploaded:
         print('[OK    ] %-15s %s' % (
-            branch.project.relpath + '/',
+            branch.project.RelPath(local=opt.this_manifest_only) + '/',
             branch.name),
             file=sys.stderr)
 
     if have_errors:
       sys.exit(1)
 
-  def _GetMergeBranch(self, project):
-    p = GitCommand(project,
-                   ['rev-parse', '--abbrev-ref', 'HEAD'],
-                   capture_stdout=True,
-                   capture_stderr=True)
-    p.Wait()
-    local_branch = p.stdout.strip()
+  def _GetMergeBranch(self, project, local_branch=None):
+    if local_branch is None:
+      p = GitCommand(project,
+                     ['rev-parse', '--abbrev-ref', 'HEAD'],
+                     capture_stdout=True,
+                     capture_stderr=True)
+      p.Wait()
+      local_branch = p.stdout.strip()
     p = GitCommand(project,
                    ['config', '--get', 'branch.%s.merge' % local_branch],
                    capture_stdout=True,
@@ -524,7 +579,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     return (project, avail)
 
   def Execute(self, opt, args):
-    projects = self.GetProjects(args)
+    projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     def _ProcessResults(_pool, _out, results):
       pending = []
@@ -534,7 +589,8 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
           print('repo: error: %s: Unable to upload branch "%s". '
                 'You might be able to fix the branch by running:\n'
                 '  git branch --set-upstream-to m/%s' %
-                (project.relpath, project.CurrentBranch, self.manifest.branch),
+                (project.RelPath(local=opt.this_manifest_only), project.CurrentBranch,
+                 project.manifest.branch),
                 file=sys.stderr)
         elif avail:
           pending.append(result)
@@ -554,15 +610,22 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
               (opt.branch,), file=sys.stderr)
       return 1
 
-    pending_proj_names = [project.name for (project, available) in pending]
-    pending_worktrees = [project.worktree for (project, available) in pending]
-    hook = RepoHook.FromSubcmd(
-        hook_type='pre-upload', manifest=self.manifest,
-        opt=opt, abort_if_user_denies=True)
-    if not hook.Run(
-        project_list=pending_proj_names,
-        worktree_list=pending_worktrees):
-      return 1
+    manifests = {project.manifest.topdir: project.manifest
+                 for (project, available) in pending}
+    ret = 0
+    for manifest in manifests.values():
+      pending_proj_names = [project.name for (project, available) in pending
+                            if project.manifest.topdir == manifest.topdir]
+      pending_worktrees = [project.worktree for (project, available) in pending
+                           if project.manifest.topdir == manifest.topdir]
+      hook = RepoHook.FromSubcmd(
+          hook_type='pre-upload', manifest=manifest,
+          opt=opt, abort_if_user_denies=True)
+      if not hook.Run(project_list=pending_proj_names,
+                      worktree_list=pending_worktrees):
+        ret = 1
+    if ret:
+      return ret
 
     reviewers = _SplitEmails(opt.reviewers) if opt.reviewers else []
     cc = _SplitEmails(opt.cc) if opt.cc else []

subcmds/version.py

@@ -33,7 +33,7 @@ class Version(Command, MirrorSafeCommand):
 
   def Execute(self, opt, args):
     rp = self.manifest.repoProject
-    rem = rp.GetRemote(rp.remote.name)
+    rem = rp.GetRemote()
     branch = rp.GetBranch('default')
 
     # These might not be the same.  Report them both.

tests/conftest.py

@@ -0,0 +1,25 @@
+# Copyright 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Common fixtures for pytests."""
+
+import pytest
+
+import repo_trace
+
+
+@pytest.fixture(autouse=True)
+def disable_repo_trace(tmp_path):
+  """Set an environment marker to relax certain strict checks for test code."""
+  repo_trace._TRACE_FILE = str(tmp_path / 'TRACE_FILE_from_test')

tests/test_git_command.py

@@ -15,6 +15,7 @@
 """Unittests for the git_command.py module."""
 
 import re
+import os
 import unittest
 
 try:
@@ -26,6 +27,38 @@ import git_command
 import wrapper
 
 
+class GitCommandTest(unittest.TestCase):
+  """Tests the GitCommand class (via git_command.git)."""
+
+  def setUp(self):
+
+    def realpath_mock(val):
+      return val
+
+    mock.patch.object(os.path, 'realpath', side_effect=realpath_mock).start()
+
+  def tearDown(self):
+    mock.patch.stopall()
+
+  def test_alternative_setting_when_matching(self):
+    r = git_command._build_env(
+      objdir = os.path.join('zap', 'objects'),
+      gitdir = 'zap'
+    )
+
+    self.assertIsNone(r.get('GIT_ALTERNATE_OBJECT_DIRECTORIES'))
+    self.assertEqual(r.get('GIT_OBJECT_DIRECTORY'), os.path.join('zap', 'objects'))
+
+  def test_alternative_setting_when_different(self):
+    r = git_command._build_env(
+      objdir = os.path.join('wow', 'objects'),
+      gitdir = 'zap'
+    )
+
+    self.assertEqual(r.get('GIT_ALTERNATE_OBJECT_DIRECTORIES'), os.path.join('zap', 'objects'))
+    self.assertEqual(r.get('GIT_OBJECT_DIRECTORY'), os.path.join('wow', 'objects'))
+
+
 class GitCallUnitTest(unittest.TestCase):
   """Tests the _GitCall class (via git_command.git)."""
 
@@ -84,7 +117,8 @@ class GitRequireTests(unittest.TestCase):
   """Test the git_require helper."""
 
   def setUp(self):
-    ver = wrapper.GitVersion(1, 2, 3, 4)
+    self.wrapper = wrapper.Wrapper()
+    ver = self.wrapper.GitVersion(1, 2, 3, 4)
     mock.patch.object(git_command.git, 'version_tuple', return_value=ver).start()
 
   def tearDown(self):

tests/test_git_config.py

@@ -186,7 +186,3 @@ class GitConfigReadWriteTests(unittest.TestCase):
     for key, value in TESTS:
       self.assertEqual(sync_data[f'{git_config.SYNC_STATE_PREFIX}{key}'], value)
     self.assertTrue(sync_data[f'{git_config.SYNC_STATE_PREFIX}main.synctime'])
-
-
-if __name__ == '__main__':
-  unittest.main()

tests/test_git_superproject.py

@@ -24,7 +24,6 @@ from unittest import mock
 import git_superproject
 import git_trace2_event_log
 import manifest_xml
-import platform_utils
 from test_manifest_xml import sort_attributes
 
 
@@ -38,7 +37,8 @@ class SuperprojectTestCase(unittest.TestCase):
 
   def setUp(self):
     """Set up superproject every time."""
-    self.tempdir = tempfile.mkdtemp(prefix='repo_tests')
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    self.tempdir = self.tempdirobj.name
     self.repodir = os.path.join(self.tempdir, '.repo')
     self.manifest_file = os.path.join(
         self.repodir, manifest_xml.MANIFEST_FILE_NAME)
@@ -68,12 +68,14 @@ class SuperprojectTestCase(unittest.TestCase):
   <project path="art" name="platform/art" groups="notdefault,platform-""" + self.platform + """
   " /></manifest>
 """)
-    self._superproject = git_superproject.Superproject(manifest, self.repodir,
-                                                       self.git_event_log)
+    self._superproject = git_superproject.Superproject(
+        manifest, name='superproject',
+        remote=manifest.remotes.get('default-remote').ToRemoteSpec('superproject'),
+        revision='refs/heads/main')
 
   def tearDown(self):
     """Tear down superproject every time."""
-    platform_utils.rmtree(self.tempdir)
+    self.tempdirobj.cleanup()
 
   def getXmlManifest(self, data):
     """Helper to initialize a manifest for testing."""
@@ -125,12 +127,7 @@ class SuperprojectTestCase(unittest.TestCase):
 <manifest>
 </manifest>
 """)
-    superproject = git_superproject.Superproject(manifest, self.repodir, self.git_event_log)
-    # Test that exit condition is false when there is no superproject tag.
-    sync_result = superproject.Sync()
-    self.assertFalse(sync_result.success)
-    self.assertFalse(sync_result.fatal)
-    self.verifyErrorEvent()
+    self.assertIsNone(manifest.superproject)
 
   def test_superproject_get_superproject_invalid_url(self):
     """Test with an invalid url."""
@@ -141,8 +138,11 @@ class SuperprojectTestCase(unittest.TestCase):
   <superproject name="superproject"/>
 </manifest>
 """)
-    superproject = git_superproject.Superproject(manifest, self.repodir, self.git_event_log)
-    sync_result = superproject.Sync()
+    superproject = git_superproject.Superproject(
+        manifest, name='superproject',
+        remote=manifest.remotes.get('test-remote').ToRemoteSpec('superproject'),
+        revision='refs/heads/main')
+    sync_result = superproject.Sync(self.git_event_log)
     self.assertFalse(sync_result.success)
     self.assertTrue(sync_result.fatal)
 
@@ -155,17 +155,20 @@ class SuperprojectTestCase(unittest.TestCase):
   <superproject name="superproject"/>
 </manifest>
 """)
-    self._superproject = git_superproject.Superproject(manifest, self.repodir,
-                                                       self.git_event_log)
+    self._superproject = git_superproject.Superproject(
+        manifest, name='superproject',
+        remote=manifest.remotes.get('test-remote').ToRemoteSpec('superproject'),
+        revision='refs/heads/main')
     with mock.patch.object(self._superproject, '_branch', 'junk'):
-      sync_result = self._superproject.Sync()
+      sync_result = self._superproject.Sync(self.git_event_log)
       self.assertFalse(sync_result.success)
       self.assertTrue(sync_result.fatal)
+      self.verifyErrorEvent()
 
   def test_superproject_get_superproject_mock_init(self):
     """Test with _Init failing."""
     with mock.patch.object(self._superproject, '_Init', return_value=False):
-      sync_result = self._superproject.Sync()
+      sync_result = self._superproject.Sync(self.git_event_log)
       self.assertFalse(sync_result.success)
       self.assertTrue(sync_result.fatal)
 
@@ -174,7 +177,7 @@ class SuperprojectTestCase(unittest.TestCase):
     with mock.patch.object(self._superproject, '_Init', return_value=True):
       os.mkdir(self._superproject._superproject_path)
       with mock.patch.object(self._superproject, '_Fetch', return_value=False):
-        sync_result = self._superproject.Sync()
+        sync_result = self._superproject.Sync(self.git_event_log)
         self.assertFalse(sync_result.success)
         self.assertTrue(sync_result.fatal)
 
@@ -230,7 +233,7 @@ class SuperprojectTestCase(unittest.TestCase):
                                return_value=data):
           # Create temporary directory so that it can write the file.
           os.mkdir(self._superproject._superproject_path)
-          update_result = self._superproject.UpdateProjectsRevisionId(projects)
+          update_result = self._superproject.UpdateProjectsRevisionId(projects, self.git_event_log)
           self.assertIsNotNone(update_result.manifest_path)
           self.assertFalse(update_result.fatal)
           with open(update_result.manifest_path, 'r') as fp:
@@ -256,22 +259,13 @@ class SuperprojectTestCase(unittest.TestCase):
 </manifest>
 """)
     self.maxDiff = None
-    self._superproject = git_superproject.Superproject(manifest, self.repodir,
-                                                       self.git_event_log)
-    self.assertEqual(len(self._superproject._manifest.projects), 1)
-    projects = self._superproject._manifest.projects
-    project = projects[0]
-    project.SetRevisionId('ABCDEF')
-    update_result = self._superproject.UpdateProjectsRevisionId(projects)
-    self.assertIsNone(update_result.manifest_path)
-    self.assertFalse(update_result.fatal)
-    self.verifyErrorEvent()
+    self.assertIsNone(manifest.superproject)
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
         '<remote fetch="http://localhost" name="default-remote"/>'
         '<default remote="default-remote" revision="refs/heads/main"/>'
-        '<project name="test-name" revision="ABCDEF" upstream="refs/heads/main"/>'
+        '<project name="test-name"/>'
         '</manifest>')
 
   def test_superproject_update_project_revision_id_from_local_manifest_group(self):
@@ -290,8 +284,10 @@ class SuperprojectTestCase(unittest.TestCase):
   " /></manifest>
 """)
     self.maxDiff = None
-    self._superproject = git_superproject.Superproject(manifest, self.repodir,
-                                                       self.git_event_log)
+    self._superproject = git_superproject.Superproject(
+        manifest, name='superproject',
+        remote=manifest.remotes.get('default-remote').ToRemoteSpec('superproject'),
+        revision='refs/heads/main')
     self.assertEqual(len(self._superproject._manifest.projects), 2)
     projects = self._superproject._manifest.projects
     data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00')
@@ -302,7 +298,7 @@ class SuperprojectTestCase(unittest.TestCase):
                                return_value=data):
           # Create temporary directory so that it can write the file.
           os.mkdir(self._superproject._superproject_path)
-          update_result = self._superproject.UpdateProjectsRevisionId(projects)
+          update_result = self._superproject.UpdateProjectsRevisionId(projects, self.git_event_log)
           self.assertIsNotNone(update_result.manifest_path)
           self.assertFalse(update_result.fatal)
           with open(update_result.manifest_path, 'r') as fp:
@@ -317,9 +313,6 @@ class SuperprojectTestCase(unittest.TestCase):
               '<project groups="notdefault,platform-' + self.platform + '" '
               'name="platform/art" path="art" '
               'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea" upstream="refs/heads/main"/>'
-              '<project clone-depth="1" groups="' + local_group + '" '
-              'name="platform/vendor/x" path="vendor/x" remote="goog" '
-              'revision="master-with-vendor"/>'
               '<superproject name="superproject"/>'
               '</manifest>')
 
@@ -337,8 +330,10 @@ class SuperprojectTestCase(unittest.TestCase):
   " /></manifest>
 """)
     self.maxDiff = None
-    self._superproject = git_superproject.Superproject(manifest, self.repodir,
-                                                       self.git_event_log)
+    self._superproject = git_superproject.Superproject(
+        manifest, name='superproject',
+        remote=manifest.remotes.get('default-remote').ToRemoteSpec('superproject'),
+        revision='refs/heads/main')
     self.assertEqual(len(self._superproject._manifest.projects), 3)
     projects = self._superproject._manifest.projects
     data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00'
@@ -350,7 +345,7 @@ class SuperprojectTestCase(unittest.TestCase):
                                return_value=data):
           # Create temporary directory so that it can write the file.
           os.mkdir(self._superproject._superproject_path)
-          update_result = self._superproject.UpdateProjectsRevisionId(projects)
+          update_result = self._superproject.UpdateProjectsRevisionId(projects, self.git_event_log)
           self.assertIsNotNone(update_result.manifest_path)
           self.assertFalse(update_result.fatal)
           with open(update_result.manifest_path, 'r') as fp:
@@ -371,6 +366,40 @@ class SuperprojectTestCase(unittest.TestCase):
               '<superproject name="superproject"/>'
               '</manifest>')
 
+  def test_Fetch(self):
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <superproject name="superproject"/>
+  " /></manifest>
+""")
+    self.maxDiff = None
+    self._superproject = git_superproject.Superproject(
+        manifest, name='superproject',
+        remote=manifest.remotes.get('default-remote').ToRemoteSpec('superproject'),
+        revision='refs/heads/main')
+    os.mkdir(self._superproject._superproject_path)
+    os.mkdir(self._superproject._work_git)
+    with mock.patch.object(self._superproject, '_Init', return_value=True):
+      with mock.patch('git_superproject.GitCommand', autospec=True) as mock_git_command:
+        with mock.patch('git_superproject.GitRefs.get', autospec=True) as mock_git_refs:
+          instance = mock_git_command.return_value
+          instance.Wait.return_value = 0
+          mock_git_refs.side_effect = ['', '1234']
 
-if __name__ == '__main__':
-  unittest.main()
+          self.assertTrue(self._superproject._Fetch())
+          self.assertEqual(mock_git_command.call_args.args,(None, [
+              'fetch', 'http://localhost/superproject', '--depth', '1',
+              '--force', '--no-tags', '--filter', 'blob:none',
+              'refs/heads/main:refs/heads/main'
+          ]))
+
+          # If branch for revision exists, set as --negotiation-tip.
+          self.assertTrue(self._superproject._Fetch())
+          self.assertEqual(mock_git_command.call_args.args,(None, [
+              'fetch', 'http://localhost/superproject', '--depth', '1',
+              '--force', '--no-tags', '--filter', 'blob:none',
+              '--negotiation-tip', '1234',
+              'refs/heads/main:refs/heads/main'
+          ]))

tests/test_git_trace2_event_log.py

@@ -16,11 +16,42 @@
 
 import json
 import os
+import socket
 import tempfile
+import threading
 import unittest
 from unittest import mock
 
 import git_trace2_event_log
+import platform_utils
+
+
+def serverLoggingThread(socket_path, server_ready, received_traces):
+  """Helper function to receive logs over a Unix domain socket.
+
+  Appends received messages on the provided socket and appends to received_traces.
+
+  Args:
+    socket_path: path to a Unix domain socket on which to listen for traces
+    server_ready: a threading.Condition used to signal to the caller that this thread is ready to
+        accept connections
+    received_traces: a list to which received traces will be appended (after decoding to a utf-8
+        string).
+  """
+  platform_utils.remove(socket_path, missing_ok=True)
+  data = b''
+  with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
+    sock.bind(socket_path)
+    sock.listen(0)
+    with server_ready:
+      server_ready.notify()
+    with sock.accept()[0] as conn:
+      while True:
+        recved = conn.recv(4096)
+        if not recved:
+          break
+        data += recved
+  received_traces.extend(data.decode('utf-8').splitlines())
 
 
 class EventLogTestCase(unittest.TestCase):
@@ -324,6 +355,33 @@ class EventLogTestCase(unittest.TestCase):
     with self.assertRaises(TypeError):
       self._event_log_module.Write(path=1234)
 
+  def test_write_socket(self):
+    """Test Write() with Unix domain socket for |path| and validate received traces."""
+    received_traces = []
+    with tempfile.TemporaryDirectory(prefix='test_server_sockets') as tempdir:
+      socket_path = os.path.join(tempdir, "server.sock")
+      server_ready = threading.Condition()
+      # Start "server" listening on Unix domain socket at socket_path.
+      try:
+        server_thread = threading.Thread(
+            target=serverLoggingThread,
+            args=(socket_path, server_ready, received_traces))
+        server_thread.start()
 
-if __name__ == '__main__':
-  unittest.main()
+        with server_ready:
+          server_ready.wait(timeout=120)
+
+        self._event_log_module.StartEvent()
+        path = self._event_log_module.Write(path=f'af_unix:{socket_path}')
+      finally:
+        server_thread.join(timeout=5)
+
+    self.assertEqual(path, f'af_unix:stream:{socket_path}')
+    self.assertEqual(len(received_traces), 2)
+    version_event = json.loads(received_traces[0])
+    start_event = json.loads(received_traces[1])
+    self.verifyCommonKeys(version_event, expected_event_name='version')
+    self.verifyCommonKeys(start_event, expected_event_name='start')
+    # Check for 'start' event specific fields.
+    self.assertIn('argv', start_event)
+    self.assertIsInstance(start_event['argv'], list)

tests/test_manifest_xml.py

@@ -17,7 +17,6 @@
 import os
 import platform
 import re
-import shutil
 import tempfile
 import unittest
 import xml.dom.minidom
@@ -92,7 +91,8 @@ class ManifestParseTestCase(unittest.TestCase):
   """TestCase for parsing manifests."""
 
   def setUp(self):
-    self.tempdir = tempfile.mkdtemp(prefix='repo_tests')
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    self.tempdir = self.tempdirobj.name
     self.repodir = os.path.join(self.tempdir, '.repo')
     self.manifest_dir = os.path.join(self.repodir, 'manifests')
     self.manifest_file = os.path.join(
@@ -111,11 +111,11 @@ class ManifestParseTestCase(unittest.TestCase):
 """)
 
   def tearDown(self):
-    shutil.rmtree(self.tempdir, ignore_errors=True)
+    self.tempdirobj.cleanup()
 
   def getXmlManifest(self, data):
     """Helper to initialize a manifest for testing."""
-    with open(self.manifest_file, 'w') as fp:
+    with open(self.manifest_file, 'w', encoding="utf-8") as fp:
       fp.write(data)
     return manifest_xml.XmlManifest(self.repodir, self.manifest_file)
 
@@ -252,6 +252,37 @@ class XmlManifestTests(ManifestParseTestCase):
         '<manifest></manifest>')
     self.assertEqual(manifest.ToDict(), {})
 
+  def test_toxml_omit_local(self):
+    """Does not include local_manifests projects when omit_local=True."""
+    manifest = self.getXmlManifest(
+        '<?xml version="1.0" encoding="UTF-8"?><manifest>'
+        '<remote name="a" fetch=".."/><default remote="a" revision="r"/>'
+        '<project name="p" groups="local::me"/>'
+        '<project name="q"/>'
+        '<project name="r" groups="keep"/>'
+        '</manifest>')
+    self.assertEqual(
+        sort_attributes(manifest.ToXml(omit_local=True).toxml()),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch=".." name="a"/><default remote="a" revision="r"/>'
+        '<project name="q"/><project groups="keep" name="r"/></manifest>')
+
+  def test_toxml_with_local(self):
+    """Does include local_manifests projects when omit_local=False."""
+    manifest = self.getXmlManifest(
+        '<?xml version="1.0" encoding="UTF-8"?><manifest>'
+        '<remote name="a" fetch=".."/><default remote="a" revision="r"/>'
+        '<project name="p" groups="local::me"/>'
+        '<project name="q"/>'
+        '<project name="r" groups="keep"/>'
+        '</manifest>')
+    self.assertEqual(
+        sort_attributes(manifest.ToXml(omit_local=False).toxml()),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch=".." name="a"/><default remote="a" revision="r"/>'
+        '<project groups="local::me" name="p"/>'
+        '<project name="q"/><project groups="keep" name="r"/></manifest>')
+
   def test_repo_hooks(self):
     """Check repo-hooks settings."""
     manifest = self.getXmlManifest("""
@@ -289,8 +320,8 @@ class XmlManifestTests(ManifestParseTestCase):
   <x-custom-tag>X tags are always ignored</x-custom-tag>
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
+    self.assertEqual(manifest.superproject.name, 'superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'test-remote')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -395,7 +426,7 @@ class IncludeElementTests(ManifestParseTestCase):
     def parse(name):
       name = self.encodeXmlAttr(name)
       # Setup target of the include.
-      with open(os.path.join(self.manifest_dir, 'target.xml'), 'w') as fp:
+      with open(os.path.join(self.manifest_dir, 'target.xml'), 'w', encoding="utf-8") as fp:
         fp.write(f'<manifest><include name="{name}"/></manifest>')
 
       manifest = self.getXmlManifest("""
@@ -486,22 +517,22 @@ class ProjectElementTests(ManifestParseTestCase):
 """)
 
     manifest = parse('a/path/', 'foo')
-    self.assertEqual(manifest.projects[0].gitdir,
-                     os.path.join(self.tempdir, '.repo/projects/foo.git'))
-    self.assertEqual(manifest.projects[0].objdir,
-                     os.path.join(self.tempdir, '.repo/project-objects/a/path.git'))
+    self.assertEqual(os.path.normpath(manifest.projects[0].gitdir),
+                     os.path.join(self.tempdir, '.repo', 'projects', 'foo.git'))
+    self.assertEqual(os.path.normpath(manifest.projects[0].objdir),
+                     os.path.join(self.tempdir, '.repo', 'project-objects', 'a', 'path.git'))
 
     manifest = parse('a/path', 'foo/')
-    self.assertEqual(manifest.projects[0].gitdir,
-                     os.path.join(self.tempdir, '.repo/projects/foo.git'))
-    self.assertEqual(manifest.projects[0].objdir,
-                     os.path.join(self.tempdir, '.repo/project-objects/a/path.git'))
+    self.assertEqual(os.path.normpath(manifest.projects[0].gitdir),
+                     os.path.join(self.tempdir, '.repo', 'projects', 'foo.git'))
+    self.assertEqual(os.path.normpath(manifest.projects[0].objdir),
+                     os.path.join(self.tempdir, '.repo', 'project-objects', 'a', 'path.git'))
 
     manifest = parse('a/path', 'foo//////')
-    self.assertEqual(manifest.projects[0].gitdir,
-                     os.path.join(self.tempdir, '.repo/projects/foo.git'))
-    self.assertEqual(manifest.projects[0].objdir,
-                     os.path.join(self.tempdir, '.repo/project-objects/a/path.git'))
+    self.assertEqual(os.path.normpath(manifest.projects[0].gitdir),
+                     os.path.join(self.tempdir, '.repo', 'projects', 'foo.git'))
+    self.assertEqual(os.path.normpath(manifest.projects[0].objdir),
+                     os.path.join(self.tempdir, '.repo', 'project-objects', 'a', 'path.git'))
 
   def test_toplevel_path(self):
     """Check handling of path=. specially."""
@@ -518,8 +549,8 @@ class ProjectElementTests(ManifestParseTestCase):
 
     for path in ('.', './', './/', './//'):
       manifest = parse('server/path', path)
-      self.assertEqual(manifest.projects[0].gitdir,
-                       os.path.join(self.tempdir, '.repo/projects/..git'))
+      self.assertEqual(os.path.normpath(manifest.projects[0].gitdir),
+                       os.path.join(self.tempdir, '.repo', 'projects', '..git'))
 
   def test_bad_path_name_checks(self):
     """Check handling of bad path & name attributes."""
@@ -545,7 +576,7 @@ class ProjectElementTests(ManifestParseTestCase):
       parse('', 'ok')
 
     for path in INVALID_FS_PATHS:
-      if not path or path.endswith('/'):
+      if not path or path.endswith('/') or path.endswith(os.path.sep):
         continue
 
       with self.assertRaises(error.ManifestInvalidPathError):
@@ -569,10 +600,10 @@ class SuperProjectElementTests(ManifestParseTestCase):
   <superproject name="superproject"/>
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
-    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
-    self.assertEqual(manifest.superproject['revision'], 'refs/heads/main')
+    self.assertEqual(manifest.superproject.name, 'superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'test-remote')
+    self.assertEqual(manifest.superproject.remote.url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject.revision, 'refs/heads/main')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -591,10 +622,10 @@ class SuperProjectElementTests(ManifestParseTestCase):
   <superproject name="superproject" revision="refs/heads/stable" />
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
-    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
-    self.assertEqual(manifest.superproject['revision'], 'refs/heads/stable')
+    self.assertEqual(manifest.superproject.name, 'superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'test-remote')
+    self.assertEqual(manifest.superproject.remote.url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject.revision, 'refs/heads/stable')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -613,10 +644,10 @@ class SuperProjectElementTests(ManifestParseTestCase):
   <superproject name="superproject" revision="refs/heads/stable" />
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
-    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
-    self.assertEqual(manifest.superproject['revision'], 'refs/heads/stable')
+    self.assertEqual(manifest.superproject.name, 'superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'test-remote')
+    self.assertEqual(manifest.superproject.remote.url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject.revision, 'refs/heads/stable')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -635,10 +666,10 @@ class SuperProjectElementTests(ManifestParseTestCase):
   <superproject name="superproject" revision="refs/heads/stable" />
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
-    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
-    self.assertEqual(manifest.superproject['revision'], 'refs/heads/stable')
+    self.assertEqual(manifest.superproject.name, 'superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'test-remote')
+    self.assertEqual(manifest.superproject.remote.url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject.revision, 'refs/heads/stable')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -657,10 +688,10 @@ class SuperProjectElementTests(ManifestParseTestCase):
   <superproject name="platform/superproject" remote="superproject-remote"/>
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'platform/superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'superproject-remote')
-    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/platform/superproject')
-    self.assertEqual(manifest.superproject['revision'], 'refs/heads/main')
+    self.assertEqual(manifest.superproject.name, 'platform/superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'superproject-remote')
+    self.assertEqual(manifest.superproject.remote.url, 'http://localhost/platform/superproject')
+    self.assertEqual(manifest.superproject.revision, 'refs/heads/main')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -679,9 +710,9 @@ class SuperProjectElementTests(ManifestParseTestCase):
   <superproject name="superproject" remote="default-remote"/>
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'default-remote')
-    self.assertEqual(manifest.superproject['revision'], 'refs/heads/main')
+    self.assertEqual(manifest.superproject.name, 'superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'default-remote')
+    self.assertEqual(manifest.superproject.revision, 'refs/heads/main')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -843,3 +874,27 @@ class ExtendProjectElementTests(ManifestParseTestCase):
     else:
       self.assertEqual(manifest.projects[0].relpath, 'bar')
       self.assertEqual(manifest.projects[1].relpath, 'y')
+
+  def test_extend_project_dest_branch(self):
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" dest-branch="foo" />
+  <project name="myproject" />
+  <extend-project name="myproject" dest-branch="bar" />
+</manifest>
+""")
+    self.assertEqual(len(manifest.projects), 1)
+    self.assertEqual(manifest.projects[0].dest_branch, 'bar')
+
+  def test_extend_project_upstream(self):
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="myproject" />
+  <extend-project name="myproject" upstream="bar" />
+</manifest>
+""")
+    self.assertEqual(len(manifest.projects), 1)
+    self.assertEqual(manifest.projects[0].upstream, 'bar')

tests/test_project.py

@@ -16,12 +16,13 @@
 
 import contextlib
 import os
-import shutil
+from pathlib import Path
 import subprocess
 import tempfile
 import unittest
 
 import error
+import manifest_xml
 import git_command
 import git_config
 import platform_utils
@@ -31,11 +32,7 @@ import project
 @contextlib.contextmanager
 def TempGitTree():
   """Create a new empty git checkout for testing."""
-  # TODO(vapier): Convert this to tempfile.TemporaryDirectory once we drop
-  # Python 2 support entirely.
-  try:
-    tempdir = tempfile.mkdtemp(prefix='repo-tests')
-
+  with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
     # Tests need to assume, that main is default branch at init,
     # which is not supported in config until 2.28.
     cmd = ['git', 'init']
@@ -49,8 +46,6 @@ def TempGitTree():
       cmd += ['--template', templatedir]
     subprocess.check_call(cmd, cwd=tempdir)
     yield tempdir
-  finally:
-    platform_utils.rmtree(tempdir)
 
 
 class FakeProject(object):
@@ -111,7 +106,7 @@ class ReviewableBranchTests(unittest.TestCase):
 class CopyLinkTestCase(unittest.TestCase):
   """TestCase for stub repo client checkouts.
 
-  It'll have a layout like:
+  It'll have a layout like this:
     tempdir/          # self.tempdir
       checkout/       # self.topdir
         git-project/  # self.worktree
@@ -123,14 +118,15 @@ class CopyLinkTestCase(unittest.TestCase):
   """
 
   def setUp(self):
-    self.tempdir = tempfile.mkdtemp(prefix='repo_tests')
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    self.tempdir = self.tempdirobj.name
     self.topdir = os.path.join(self.tempdir, 'checkout')
     self.worktree = os.path.join(self.topdir, 'git-project')
     os.makedirs(self.topdir)
     os.makedirs(self.worktree)
 
   def tearDown(self):
-    shutil.rmtree(self.tempdir, ignore_errors=True)
+    self.tempdirobj.cleanup()
 
   @staticmethod
   def touch(path):
@@ -335,3 +331,152 @@ class LinkFile(CopyLinkTestCase):
     platform_utils.symlink(self.tempdir, dest)
     lf._Link()
     self.assertEqual(os.path.join('git-project', 'foo.txt'), os.readlink(dest))
+
+
+class MigrateWorkTreeTests(unittest.TestCase):
+  """Check _MigrateOldWorkTreeGitDir handling."""
+
+  _SYMLINKS = {
+      'config', 'description', 'hooks', 'info', 'logs', 'objects',
+      'packed-refs', 'refs', 'rr-cache', 'shallow', 'svn',
+  }
+  _FILES = {
+      'COMMIT_EDITMSG', 'FETCH_HEAD', 'HEAD', 'index', 'ORIG_HEAD',
+      'unknown-file-should-be-migrated',
+  }
+  _CLEAN_FILES = {
+      'a-vim-temp-file~', '#an-emacs-temp-file#',
+  }
+
+  @classmethod
+  @contextlib.contextmanager
+  def _simple_layout(cls):
+    """Create a simple repo client checkout to test against."""
+    with tempfile.TemporaryDirectory() as tempdir:
+      tempdir = Path(tempdir)
+
+      gitdir = tempdir / '.repo/projects/src/test.git'
+      gitdir.mkdir(parents=True)
+      cmd = ['git', 'init', '--bare', str(gitdir)]
+      subprocess.check_call(cmd)
+
+      dotgit = tempdir / 'src/test/.git'
+      dotgit.mkdir(parents=True)
+      for name in cls._SYMLINKS:
+        (dotgit / name).symlink_to(f'../../../.repo/projects/src/test.git/{name}')
+      for name in cls._FILES | cls._CLEAN_FILES:
+        (dotgit / name).write_text(name)
+
+      yield tempdir
+
+  def test_standard(self):
+    """Migrate a standard checkout that we expect."""
+    with self._simple_layout() as tempdir:
+      dotgit = tempdir / 'src/test/.git'
+      project.Project._MigrateOldWorkTreeGitDir(str(dotgit))
+
+      # Make sure the dir was transformed into a symlink.
+      self.assertTrue(dotgit.is_symlink())
+      self.assertEqual(os.readlink(dotgit), os.path.normpath('../../.repo/projects/src/test.git'))
+
+      # Make sure files were moved over.
+      gitdir = tempdir / '.repo/projects/src/test.git'
+      for name in self._FILES:
+        self.assertEqual(name, (gitdir / name).read_text())
+      # Make sure files were removed.
+      for name in self._CLEAN_FILES:
+        self.assertFalse((gitdir / name).exists())
+
+  def test_unknown(self):
+    """A checkout with unknown files should abort."""
+    with self._simple_layout() as tempdir:
+      dotgit = tempdir / 'src/test/.git'
+      (tempdir / '.repo/projects/src/test.git/random-file').write_text('one')
+      (dotgit / 'random-file').write_text('two')
+      with self.assertRaises(error.GitError):
+        project.Project._MigrateOldWorkTreeGitDir(str(dotgit))
+
+      # Make sure no content was actually changed.
+      self.assertTrue(dotgit.is_dir())
+      for name in self._FILES:
+        self.assertTrue((dotgit / name).is_file())
+      for name in self._CLEAN_FILES:
+        self.assertTrue((dotgit / name).is_file())
+      for name in self._SYMLINKS:
+        self.assertTrue((dotgit / name).is_symlink())
+
+
+class ManifestPropertiesFetchedCorrectly(unittest.TestCase):
+  """Ensure properties are fetched properly."""
+
+  def setUpManifest(self, tempdir):
+    repodir = os.path.join(tempdir, '.repo')
+    manifest_dir = os.path.join(repodir, 'manifests')
+    manifest_file = os.path.join(
+        repodir, manifest_xml.MANIFEST_FILE_NAME)
+    local_manifest_dir = os.path.join(
+        repodir, manifest_xml.LOCAL_MANIFESTS_DIR_NAME)
+    os.mkdir(repodir)
+    os.mkdir(manifest_dir)
+    manifest = manifest_xml.XmlManifest(repodir, manifest_file)
+
+    return project.ManifestProject(
+        manifest, 'test/manifest', os.path.join(tempdir, '.git'), tempdir)
+
+  def test_manifest_config_properties(self):
+    """Test we are fetching the manifest config properties correctly."""
+
+    with TempGitTree() as tempdir:
+      fakeproj = self.setUpManifest(tempdir)
+
+      # Set property using the expected Set method, then ensure
+      # the porperty functions are using the correct Get methods.
+      fakeproj.config.SetString(
+          'manifest.standalone', 'https://chicken/manifest.git')
+      self.assertEqual(
+          fakeproj.standalone_manifest_url, 'https://chicken/manifest.git')
+
+      fakeproj.config.SetString('manifest.groups', 'test-group, admin-group')
+      self.assertEqual(fakeproj.manifest_groups, 'test-group, admin-group')
+
+      fakeproj.config.SetString('repo.reference', 'mirror/ref')
+      self.assertEqual(fakeproj.reference, 'mirror/ref')
+
+      fakeproj.config.SetBoolean('repo.dissociate', False)
+      self.assertFalse(fakeproj.dissociate)
+
+      fakeproj.config.SetBoolean('repo.archive', False)
+      self.assertFalse(fakeproj.archive)
+
+      fakeproj.config.SetBoolean('repo.mirror', False)
+      self.assertFalse(fakeproj.mirror)
+
+      fakeproj.config.SetBoolean('repo.worktree', False)
+      self.assertFalse(fakeproj.use_worktree)
+
+      fakeproj.config.SetBoolean('repo.clonebundle', False)
+      self.assertFalse(fakeproj.clone_bundle)
+
+      fakeproj.config.SetBoolean('repo.submodules', False)
+      self.assertFalse(fakeproj.submodules)
+
+      fakeproj.config.SetBoolean('repo.git-lfs', False)
+      self.assertFalse(fakeproj.git_lfs)
+
+      fakeproj.config.SetBoolean('repo.superproject', False)
+      self.assertFalse(fakeproj.use_superproject)
+
+      fakeproj.config.SetBoolean('repo.partialclone', False)
+      self.assertFalse(fakeproj.partial_clone)
+
+      fakeproj.config.SetString('repo.depth', '48')
+      self.assertEqual(fakeproj.depth, '48')
+
+      fakeproj.config.SetString('repo.clonefilter', 'blob:limit=10M')
+      self.assertEqual(fakeproj.clone_filter, 'blob:limit=10M')
+
+      fakeproj.config.SetString('repo.partialcloneexclude', 'third_party/big_repo')
+      self.assertEqual(fakeproj.partial_clone_exclude, 'third_party/big_repo')
+
+      fakeproj.config.SetString('manifest.platform', 'auto')
+      self.assertEqual(fakeproj.manifest_platform, 'auto')

tests/test_repo_trace.py

@@ -0,0 +1,56 @@
+# Copyright 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for the repo_trace.py module."""
+
+import os
+import unittest
+from unittest import mock
+
+import repo_trace
+
+
+class TraceTests(unittest.TestCase):
+  """Check Trace behavior."""
+
+  def testTrace_MaxSizeEnforced(self):
+    content = 'git chicken'
+
+    with repo_trace.Trace(content, first_trace=True):
+      pass
+    first_trace_size = os.path.getsize(repo_trace._TRACE_FILE)
+
+    with repo_trace.Trace(content):
+      pass
+    self.assertGreater(
+        os.path.getsize(repo_trace._TRACE_FILE), first_trace_size)
+
+    # Check we clear everything is the last chunk is larger than _MAX_SIZE.
+    with mock.patch('repo_trace._MAX_SIZE', 0):
+      with repo_trace.Trace(content, first_trace=True):
+        pass
+      self.assertEqual(first_trace_size,
+                       os.path.getsize(repo_trace._TRACE_FILE))
+
+      # Check we only clear the chunks we need to.
+      repo_trace._MAX_SIZE = (first_trace_size + 1) / (1024 * 1024)
+      with repo_trace.Trace(content, first_trace=True):
+        pass
+      self.assertEqual(first_trace_size * 2,
+                       os.path.getsize(repo_trace._TRACE_FILE))
+
+      with repo_trace.Trace(content, first_trace=True):
+        pass
+      self.assertEqual(first_trace_size * 2,
+                       os.path.getsize(repo_trace._TRACE_FILE))

tests/test_subcmds_sync.py

@@ -0,0 +1,133 @@
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unittests for the subcmds/sync.py module."""
+
+import os
+import unittest
+from unittest import mock
+
+import pytest
+
+import command
+from subcmds import sync
+
+
+@pytest.mark.parametrize('use_superproject, cli_args, result', [
+    (True, ['--current-branch'], True),
+    (True, ['--no-current-branch'], True),
+    (True, [], True),
+    (False, ['--current-branch'], True),
+    (False, ['--no-current-branch'], False),
+    (False, [], None),
+])
+def test_get_current_branch_only(use_superproject, cli_args, result):
+  """Test Sync._GetCurrentBranchOnly logic.
+
+  Sync._GetCurrentBranchOnly should return True if a superproject is requested,
+  and otherwise the value of the current_branch_only option.
+  """
+  cmd = sync.Sync()
+  opts, _ = cmd.OptionParser.parse_args(cli_args)
+
+  with mock.patch('git_superproject.UseSuperproject',
+                  return_value=use_superproject):
+    assert cmd._GetCurrentBranchOnly(opts, cmd.manifest) == result
+
+
+# Used to patch os.cpu_count() for reliable results.
+OS_CPU_COUNT = 24
+
+@pytest.mark.parametrize('argv, jobs_manifest, jobs, jobs_net, jobs_check', [
+  # No user or manifest settings.
+  ([], None, OS_CPU_COUNT, 1, command.DEFAULT_LOCAL_JOBS),
+  # No user settings, so manifest settings control.
+  ([], 3, 3, 3, 3),
+  # User settings, but no manifest.
+  (['--jobs=4'], None, 4, 4, 4),
+  (['--jobs=4', '--jobs-network=5'], None, 4, 5, 4),
+  (['--jobs=4', '--jobs-checkout=6'], None, 4, 4, 6),
+  (['--jobs=4', '--jobs-network=5', '--jobs-checkout=6'], None, 4, 5, 6),
+  (['--jobs-network=5'], None, OS_CPU_COUNT, 5, command.DEFAULT_LOCAL_JOBS),
+  (['--jobs-checkout=6'], None, OS_CPU_COUNT, 1, 6),
+  (['--jobs-network=5', '--jobs-checkout=6'], None, OS_CPU_COUNT, 5, 6),
+  # User settings with manifest settings.
+  (['--jobs=4'], 3, 4, 4, 4),
+  (['--jobs=4', '--jobs-network=5'], 3, 4, 5, 4),
+  (['--jobs=4', '--jobs-checkout=6'], 3, 4, 4, 6),
+  (['--jobs=4', '--jobs-network=5', '--jobs-checkout=6'], 3, 4, 5, 6),
+  (['--jobs-network=5'], 3, 3, 5, 3),
+  (['--jobs-checkout=6'], 3, 3, 3, 6),
+  (['--jobs-network=5', '--jobs-checkout=6'], 3, 3, 5, 6),
+  # Settings that exceed rlimits get capped.
+  (['--jobs=1000000'], None, 83, 83, 83),
+  ([], 1000000, 83, 83, 83),
+])
+def test_cli_jobs(argv, jobs_manifest, jobs, jobs_net, jobs_check):
+  """Tests --jobs option behavior."""
+  mp = mock.MagicMock()
+  mp.manifest.default.sync_j = jobs_manifest
+
+  cmd = sync.Sync()
+  opts, args = cmd.OptionParser.parse_args(argv)
+  cmd.ValidateOptions(opts, args)
+
+  with mock.patch.object(sync, '_rlimit_nofile', return_value=(256, 256)):
+    with mock.patch.object(os, 'cpu_count', return_value=OS_CPU_COUNT):
+      cmd._ValidateOptionsWithManifest(opts, mp)
+      assert opts.jobs == jobs
+      assert opts.jobs_network == jobs_net
+      assert opts.jobs_checkout == jobs_check
+
+
+class GetPreciousObjectsState(unittest.TestCase):
+  """Tests for _GetPreciousObjectsState."""
+
+  def setUp(self):
+    """Common setup."""
+    self.cmd = sync.Sync()
+    self.project = p = mock.MagicMock(use_git_worktrees=False,
+                                      UseAlternates=False)
+    p.manifest.GetProjectsWithName.return_value = [p]
+
+    self.opt = mock.Mock(spec_set=['this_manifest_only'])
+    self.opt.this_manifest_only = False
+
+  def test_worktrees(self):
+    """False for worktrees."""
+    self.project.use_git_worktrees = True
+    self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt))
+
+  def test_not_shared(self):
+    """Singleton project."""
+    self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt))
+
+  def test_shared(self):
+    """Shared project."""
+    self.project.manifest.GetProjectsWithName.return_value = [
+        self.project, self.project
+    ]
+    self.assertTrue(self.cmd._GetPreciousObjectsState(self.project, self.opt))
+
+  def test_shared_with_alternates(self):
+    """Shared project, with alternates."""
+    self.project.manifest.GetProjectsWithName.return_value = [
+        self.project, self.project
+    ]
+    self.project.UseAlternates = True
+    self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt))
+
+  def test_not_found(self):
+    """Project not found in manifest."""
+    self.project.manifest.GetProjectsWithName.return_value = []
+    self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt))

tests/test_update_manpages.py

@@ -0,0 +1,28 @@
+# Copyright 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for the update_manpages module."""
+
+import unittest
+
+from release import update_manpages
+
+
+class UpdateManpagesTest(unittest.TestCase):
+  """Tests the update-manpages code."""
+
+  def test_replace_regex(self):
+    """Check that replace_regex works."""
+    data = '\n\033[1mSummary\033[m\n'
+    self.assertEqual(update_manpages.replace_regex(data),'\nSummary\n')

tests/test_wrapper.py

@@ -14,11 +14,9 @@
 
 """Unittests for the wrapper.py module."""
 
-import contextlib
 from io import StringIO
 import os
 import re
-import shutil
 import sys
 import tempfile
 import unittest
@@ -26,22 +24,9 @@ from unittest import mock
 
 import git_command
 import main
-import platform_utils
 import wrapper
 
 
-@contextlib.contextmanager
-def TemporaryDirectory():
-  """Create a new empty git checkout for testing."""
-  # TODO(vapier): Convert this to tempfile.TemporaryDirectory once we drop
-  # Python 2 support entirely.
-  try:
-    tempdir = tempfile.mkdtemp(prefix='repo-tests')
-    yield tempdir
-  finally:
-    platform_utils.rmtree(tempdir)
-
-
 def fixture(*paths):
   """Return a path relative to tests/fixtures.
   """
@@ -53,7 +38,7 @@ class RepoWrapperTestCase(unittest.TestCase):
 
   def setUp(self):
     """Load the wrapper module every time."""
-    wrapper._wrapper_module = None
+    wrapper.Wrapper.cache_clear()
     self.wrapper = wrapper.Wrapper()
 
 
@@ -74,12 +59,12 @@ class RepoWrapperUnitTest(RepoWrapperTestCase):
   def test_python_constraints(self):
     """The launcher should never require newer than main.py."""
     self.assertGreaterEqual(main.MIN_PYTHON_VERSION_HARD,
-                            wrapper.MIN_PYTHON_VERSION_HARD)
+                            self.wrapper.MIN_PYTHON_VERSION_HARD)
     self.assertGreaterEqual(main.MIN_PYTHON_VERSION_SOFT,
-                            wrapper.MIN_PYTHON_VERSION_SOFT)
+                            self.wrapper.MIN_PYTHON_VERSION_SOFT)
     # Make sure the versions are themselves in sync.
-    self.assertGreaterEqual(wrapper.MIN_PYTHON_VERSION_SOFT,
-                            wrapper.MIN_PYTHON_VERSION_HARD)
+    self.assertGreaterEqual(self.wrapper.MIN_PYTHON_VERSION_SOFT,
+                            self.wrapper.MIN_PYTHON_VERSION_HARD)
 
   def test_init_parser(self):
     """Make sure 'init' GetParser works."""
@@ -174,7 +159,9 @@ class RunCommand(RepoWrapperTestCase):
   def test_capture(self):
     """Check capture_output handling."""
     ret = self.wrapper.run_command(['echo', 'hi'], capture_output=True)
-    self.assertEqual(ret.stdout, 'hi\n')
+    # echo command appends OS specific linesep, but on Windows + Git Bash
+    # we get UNIX ending, so we allow both.
+    self.assertIn(ret.stdout, ['hi' + os.linesep, 'hi\n'])
 
   def test_check(self):
     """Check check handling."""
@@ -336,19 +323,19 @@ class NeedSetupGnuPG(RepoWrapperTestCase):
 
   def test_missing_dir(self):
     """The ~/.repoconfig tree doesn't exist yet."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = os.path.join(tempdir, 'foo')
       self.assertTrue(self.wrapper.NeedSetupGnuPG())
 
   def test_missing_keyring(self):
     """The keyring-version file doesn't exist yet."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       self.assertTrue(self.wrapper.NeedSetupGnuPG())
 
   def test_empty_keyring(self):
     """The keyring-version file exists, but is empty."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       with open(os.path.join(tempdir, 'keyring-version'), 'w'):
         pass
@@ -356,7 +343,7 @@ class NeedSetupGnuPG(RepoWrapperTestCase):
 
   def test_old_keyring(self):
     """The keyring-version file exists, but it's old."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       with open(os.path.join(tempdir, 'keyring-version'), 'w') as fp:
         fp.write('1.0\n')
@@ -364,7 +351,7 @@ class NeedSetupGnuPG(RepoWrapperTestCase):
 
   def test_new_keyring(self):
     """The keyring-version file exists, and is up-to-date."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       with open(os.path.join(tempdir, 'keyring-version'), 'w') as fp:
         fp.write('1000.0\n')
@@ -376,7 +363,7 @@ class SetupGnuPG(RepoWrapperTestCase):
 
   def test_full(self):
     """Make sure it works completely."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       self.wrapper.gpg_dir = os.path.join(self.wrapper.home_dot_repo, 'gnupg')
       self.assertTrue(self.wrapper.SetupGnuPG(True))
@@ -426,7 +413,8 @@ class GitCheckoutTestCase(RepoWrapperTestCase):
   @classmethod
   def setUpClass(cls):
     # Create a repo to operate on, but do it once per-class.
-    cls.GIT_DIR = tempfile.mkdtemp(prefix='repo-rev-tests')
+    cls.tempdirobj = tempfile.TemporaryDirectory(prefix='repo-rev-tests')
+    cls.GIT_DIR = cls.tempdirobj.name
     run_git = wrapper.Wrapper().run_git
 
     remote = os.path.join(cls.GIT_DIR, 'remote')
@@ -455,10 +443,10 @@ class GitCheckoutTestCase(RepoWrapperTestCase):
 
   @classmethod
   def tearDownClass(cls):
-    if not cls.GIT_DIR:
+    if not cls.tempdirobj:
       return
 
-    shutil.rmtree(cls.GIT_DIR)
+    cls.tempdirobj.cleanup()
 
 
 class ResolveRepoRev(GitCheckoutTestCase):
@@ -470,7 +458,7 @@ class ResolveRepoRev(GitCheckoutTestCase):
     self.assertEqual('refs/heads/stable', rrev)
     self.assertEqual(self.REV_LIST[1], lrev)
 
-    with self.assertRaises(wrapper.CloneFailure):
+    with self.assertRaises(self.wrapper.CloneFailure):
       self.wrapper.resolve_repo_rev(self.GIT_DIR, 'refs/heads/unknown')
 
   def test_explicit_tag(self):
@@ -479,7 +467,7 @@ class ResolveRepoRev(GitCheckoutTestCase):
     self.assertEqual('refs/tags/v1.0', rrev)
     self.assertEqual(self.REV_LIST[1], lrev)
 
-    with self.assertRaises(wrapper.CloneFailure):
+    with self.assertRaises(self.wrapper.CloneFailure):
       self.wrapper.resolve_repo_rev(self.GIT_DIR, 'refs/tags/unknown')
 
   def test_branch_name(self):
@@ -514,7 +502,7 @@ class ResolveRepoRev(GitCheckoutTestCase):
 
   def test_unknown(self):
     """Check unknown ref/commit argument."""
-    with self.assertRaises(wrapper.CloneFailure):
+    with self.assertRaises(self.wrapper.CloneFailure):
       self.wrapper.resolve_repo_rev(self.GIT_DIR, 'boooooooya')
 
 
@@ -565,7 +553,3 @@ class CheckRepoRev(GitCheckoutTestCase):
       rrev, lrev = self.wrapper.check_repo_rev(self.GIT_DIR, 'stable', repo_verify=False)
     self.assertEqual('refs/heads/stable', rrev)
     self.assertEqual(self.REV_LIST[1], lrev)
-
-
-if __name__ == '__main__':
-  unittest.main()

tox.ini

@@ -15,7 +15,7 @@
 # https://tox.readthedocs.io/
 
 [tox]
-envlist = py36, py37, py38, py39
+envlist = py36, py37, py38, py39, py310
 
 [gh-actions]
 python =
@@ -23,11 +23,17 @@ python =
     3.7: py37
     3.8: py38
     3.9: py39
+    3.10: py310
 
 [testenv]
-deps = pytest
-commands = {envpython} run_tests
+deps =
+    pytest
+    pytest-timeout
+commands = {envpython} run_tests {posargs}
 setenv =
     GIT_AUTHOR_NAME = Repo test author
     GIT_COMMITTER_NAME = Repo test committer
     EMAIL = repo@gerrit.nodomain
+
+[pytest]
+timeout = 300

wrapper.py

@@ -12,12 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-try:
-  from importlib.machinery import SourceFileLoader
-  _loader = lambda *args: SourceFileLoader(*args).load_module()
-except ImportError:
-  import imp
-  _loader = lambda *args: imp.load_source(*args)
+import functools
+import importlib.machinery
+import importlib.util
 import os
 
 
@@ -25,11 +22,11 @@ def WrapperPath():
   return os.path.join(os.path.dirname(__file__), 'repo')
 
 
-_wrapper_module = None
-
-
+@functools.lru_cache(maxsize=None)
 def Wrapper():
-  global _wrapper_module
-  if not _wrapper_module:
-    _wrapper_module = _loader('wrapper', WrapperPath())
-  return _wrapper_module
+  modname = 'wrapper'
+  loader = importlib.machinery.SourceFileLoader(modname, WrapperPath())
+  spec = importlib.util.spec_from_loader(modname, loader)
+  module = importlib.util.module_from_spec(spec)
+  loader.exec_module(module)
+  return module