trace: restore Progress indicator.

If we are not tracing to stderr, then we should still have progress
indication.

Change-Id: Ifc9678e1fccbd92251e972fcf25aad6369d60e15
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/351195
Reviewed-by: Sam Saccone <samccone@google.com>
Tested-by: LaMont Jones <lamontjones@google.com>
Reviewed-by: Xin Li <delphij@google.com>

command.py

@@ -61,13 +61,21 @@ class Command(object):
   # it is the number of parallel jobs to default to.
   PARALLEL_JOBS = None
 
+  # Whether this command supports Multi-manifest.  If False, then main.py will
+  # iterate over the manifests and invoke the command once per (sub)manifest.
+  # This is only checked after calling ValidateOptions, so that partially
+  # migrated subcommands can set it to False.
+  MULTI_MANIFEST_SUPPORT = True
+
   def __init__(self, repodir=None, client=None, manifest=None, gitc_manifest=None,
-               git_event_log=None):
+               git_event_log=None, outer_client=None, outer_manifest=None):
     self.repodir = repodir
     self.client = client
+    self.outer_client = outer_client or client
     self.manifest = manifest
     self.gitc_manifest = gitc_manifest
     self.git_event_log = git_event_log
+    self.outer_manifest = outer_manifest
 
     # Cache for the OptionParser property.
     self._optparse = None
@@ -135,6 +143,17 @@ class Command(object):
           type=int, default=self.PARALLEL_JOBS,
           help=f'number of jobs to run in parallel (default: {default})')
 
+    m = p.add_option_group('Multi-manifest options')
+    m.add_option('--outer-manifest', action='store_true', default=None,
+                 help='operate starting at the outermost manifest')
+    m.add_option('--no-outer-manifest', dest='outer_manifest',
+                 action='store_false', help='do not operate on outer manifests')
+    m.add_option('--this-manifest-only', action='store_true', default=None,
+                 help='only operate on this (sub)manifest')
+    m.add_option('--no-this-manifest-only', '--all-manifests',
+                 dest='this_manifest_only', action='store_false',
+                 help='operate on this manifest and its submanifests')
+
   def _Options(self, p):
     """Initialize the option parser with subcommand-specific options."""
 
@@ -166,6 +185,10 @@ class Command(object):
     """Validate common options."""
     opt.quiet = opt.output_mode is False
     opt.verbose = opt.output_mode is True
+    if opt.outer_manifest is None:
+      # By default, treat multi-manifest instances as a single manifest from
+      # the user's perspective.
+      opt.outer_manifest = True
 
   def ValidateOptions(self, opt, args):
     """Validate the user options & arguments before executing.
@@ -252,15 +275,30 @@ class Command(object):
     return project
 
   def GetProjects(self, args, manifest=None, groups='', missing_ok=False,
-                  submodules_ok=False):
+                  submodules_ok=False, all_manifests=False):
     """A list of projects that match the arguments.
-    """
-    if not manifest:
-      manifest = self.manifest
-    all_projects_list = manifest.projects
-    result = []
 
-    mp = manifest.manifestProject
+    Args:
+      args: a list of (case-insensitive) strings, projects to search for.
+      manifest: an XmlManifest, the manifest to use, or None for default.
+      groups: a string, the manifest groups in use.
+      missing_ok: a boolean, whether to allow missing projects.
+      submodules_ok: a boolean, whether to allow submodules.
+      all_manifests: a boolean, if True then all manifests and submanifests are
+                     used.  If False, then only the local (sub)manifest is used.
+
+    Returns:
+      A list of matching Project instances.
+    """
+    if all_manifests:
+      if not manifest:
+        manifest = self.manifest.outer_client
+      all_projects_list = manifest.all_projects
+    else:
+      if not manifest:
+        manifest = self.manifest
+      all_projects_list = manifest.projects
+    result = []
 
     if not groups:
       groups = manifest.GetGroupsStr()
@@ -282,12 +320,19 @@ class Command(object):
       for arg in args:
         # We have to filter by manifest groups in case the requested project is
         # checked out multiple times or differently based on them.
-        projects = [project for project in manifest.GetProjectsWithName(arg)
+        projects = [project for project in manifest.GetProjectsWithName(
+                        arg, all_manifests=all_manifests)
                     if project.MatchesGroups(groups)]
 
         if not projects:
           path = os.path.abspath(arg).replace('\\', '/')
-          project = self._GetProjectByPath(manifest, path)
+          tree = manifest
+          if all_manifests:
+            # Look for the deepest matching submanifest.
+            for tree in reversed(list(manifest.all_manifests)):
+              if path.startswith(tree.topdir):
+                break
+          project = self._GetProjectByPath(tree, path)
 
           # If it's not a derived project, update path->project mapping and
           # search again, as arg might actually point to a derived subproject.
@@ -308,7 +353,8 @@ class Command(object):
 
         for project in projects:
           if not missing_ok and not project.Exists:
-            raise NoSuchProjectError('%s (%s)' % (arg, project.relpath))
+            raise NoSuchProjectError('%s (%s)' % (
+                arg, project.RelPath(local=not all_manifests)))
           if not project.MatchesGroups(groups):
             raise InvalidProjectGroupsError(arg)
 
@@ -319,12 +365,22 @@ class Command(object):
     result.sort(key=_getpath)
     return result
 
-  def FindProjects(self, args, inverse=False):
+  def FindProjects(self, args, inverse=False, all_manifests=False):
+    """Find projects from command line arguments.
+
+    Args:
+      args: a list of (case-insensitive) strings, projects to search for.
+      inverse: a boolean, if True, then projects not matching any |args| are
+               returned.
+      all_manifests: a boolean, if True then all manifests and submanifests are
+                     used.  If False, then only the local (sub)manifest is used.
+    """
     result = []
     patterns = [re.compile(r'%s' % a, re.IGNORECASE) for a in args]
-    for project in self.GetProjects(''):
+    for project in self.GetProjects('', all_manifests=all_manifests):
+      paths = [project.name, project.RelPath(local=not all_manifests)]
       for pattern in patterns:
-        match = pattern.search(project.name) or pattern.search(project.relpath)
+        match = any(pattern.search(x) for x in paths)
         if not inverse and match:
           result.append(project)
           break
@@ -333,9 +389,24 @@ class Command(object):
       else:
         if inverse:
           result.append(project)
-    result.sort(key=lambda project: project.relpath)
+    result.sort(key=lambda project: (project.manifest.path_prefix,
+                                     project.relpath))
     return result
 
+  def ManifestList(self, opt):
+    """Yields all of the manifests to traverse.
+
+    Args:
+      opt: The command options.
+    """
+    top = self.outer_manifest
+    if not opt.outer_manifest or opt.this_manifest_only:
+      top = self.manifest
+    yield top
+    if not opt.this_manifest_only:
+      for child in top.all_children:
+        yield child
+
 
 class InteractiveCommand(Command):
   """Command which requires user interaction on the tty and

docs/internal-fs-layout.md

@@ -50,6 +50,10 @@ For example, if you want to change the manifest branch, you can simply run
 For more documentation on the manifest format, including the local_manifests
 support, see the [manifest-format.md] file.
 
+*   `submanifests/{submanifest.path}/`: The path prefix to the manifest state of
+    a submanifest included in a multi-manifest checkout.  The outermost manifest
+    manifest state is found adjacent to `submanifests/`.
+
 *   `manifests/`: A git checkout of the manifest project.  Its `.git/` state
     points to the `manifest.git` bare checkout (see below).  It tracks the git
     branch specified at `repo init` time via `--manifest-branch`.
@@ -163,6 +167,7 @@ User controlled settings are initialized when running `repo init`.
 | repo.clonefilter         | `--clone-filter`          | Filter setting when using [partial git clones] |
 | repo.depth               | `--depth`                 | Create shallow checkouts when cloning |
 | repo.dissociate          | `--dissociate`            | Dissociate from any reference/mirrors after initial clone |
+| repo.git-lfs             | `--git-lfs`               | Enable [Git LFS] support |
 | repo.mirror              | `--mirror`                | Checkout is a repo mirror |
 | repo.partialclone        | `--partial-clone`         | Create [partial git clones] |
 | repo.partialcloneexclude | `--partial-clone-exclude` | Comma-delimited list of project names (not paths) to exclude while using [partial git clones] |
@@ -254,6 +259,7 @@ Repo will create & maintain a few files in the user's home directory.
 
 
 [git-config]: https://git-scm.com/docs/git-config
+[Git LFS]: https://git-lfs.github.com/
 [git worktree]: https://git-scm.com/docs/git-worktree
 [gitsubmodules]: https://git-scm.com/docs/gitsubmodules
 [manifest-format.md]: ./manifest-format.md

docs/manifest-format.md

@@ -26,6 +26,7 @@ following DTD:
                       remote*,
                       default?,
                       manifest-server?,
+                      submanifest*?,
                       remove-project*,
                       project*,
                       extend-project*,
@@ -57,6 +58,16 @@ following DTD:
   <!ELEMENT manifest-server EMPTY>
   <!ATTLIST manifest-server url CDATA #REQUIRED>
 
+  <!ELEMENT submanifest EMPTY>
+  <!ATTLIST submanifest name           ID #REQUIRED>
+  <!ATTLIST submanifest remote         IDREF #IMPLIED>
+  <!ATTLIST submanifest project        CDATA #IMPLIED>
+  <!ATTLIST submanifest manifest-name  CDATA #IMPLIED>
+  <!ATTLIST submanifest revision       CDATA #IMPLIED>
+  <!ATTLIST submanifest path           CDATA #IMPLIED>
+  <!ATTLIST submanifest groups         CDATA #IMPLIED>
+  <!ATTLIST submanifest default-groups CDATA #IMPLIED>
+
   <!ELEMENT project (annotation*,
                      project*,
                      copyfile*,
@@ -94,6 +105,8 @@ following DTD:
   <!ATTLIST extend-project groups CDATA #IMPLIED>
   <!ATTLIST extend-project revision CDATA #IMPLIED>
   <!ATTLIST extend-project remote CDATA #IMPLIED>
+  <!ATTLIST extend-project dest-branch CDATA #IMPLIED>
+  <!ATTLIST extend-project upstream CDATA #IMPLIED>
 
   <!ELEMENT remove-project EMPTY>
   <!ATTLIST remove-project name  CDATA #REQUIRED>
@@ -236,6 +249,66 @@ the specified tag. This is used by repo sync when the --smart-tag option
 is given.
 
 
+### Element submanifest
+
+One or more submanifest elements may be specified.  Each element describes a
+single manifest to be checked out as a child.
+
+Attribute `name`: A unique name (within the current (sub)manifest) for this
+submanifest. It acts as a default for `revision` below.  The same name can be
+used for submanifests with different parent (sub)manifests.
+
+Attribute `remote`: Name of a previously defined remote element.
+If not supplied the remote given by the default element is used.
+
+Attribute `project`: The manifest project name.  The project's name is appended
+onto its remote's fetch URL to generate the actual URL to configure the Git
+remote with.  The URL gets formed as:
+
+    ${remote_fetch}/${project_name}.git
+
+where ${remote_fetch} is the remote's fetch attribute and
+${project_name} is the project's name attribute.  The suffix ".git"
+is always appended as repo assumes the upstream is a forest of
+bare Git repositories.  If the project has a parent element, its
+name will be prefixed by the parent's.
+
+The project name must match the name Gerrit knows, if Gerrit is
+being used for code reviews.
+
+`project` must not be empty, and may not be an absolute path or use "." or ".."
+path components.  It is always interpreted relative to the remote's fetch
+settings, so if a different base path is needed, declare a different remote
+with the new settings needed.
+
+If not supplied the remote and project for this manifest will be used: `remote`
+cannot be supplied.
+
+Projects from a submanifest and its submanifests are added to the
+submanifest::path:<path_prefix> group.
+
+Attribute `manifest-name`: The manifest filename in the manifest project.  If
+not supplied, `default.xml` is used.
+
+Attribute `revision`: Name of a Git branch (e.g. "main" or "refs/heads/main"),
+tag (e.g. "refs/tags/stable"), or a commit hash.  If not supplied, `name` is
+used.
+
+Attribute `path`: An optional path relative to the top directory
+of the repo client where the submanifest repo client top directory
+should be placed.  If not supplied, `revision` is used.
+
+`path` may not be an absolute path or use "." or ".." path components.
+
+Attribute `groups`: List of additional groups to which all projects
+in the included submanifest belong. This appends and recurses, meaning
+all projects in submanifests carry all parent submanifest groups.
+Same syntax as the corresponding element of `project`.
+
+Attribute `default-groups`: The list of manifest groups to sync if no
+`--groups=` parameter was specified at init.  When that list is empty, use this
+list instead of "default" as the list of groups to sync.
+
 ### Element project
 
 One or more project elements may be specified.  Each element
@@ -352,6 +425,12 @@ project.  Same syntax as the corresponding element of `project`.
 Attribute `remote`: If specified, overrides the remote of the original
 project.  Same syntax as the corresponding element of `project`.
 
+Attribute `dest-branch`: If specified, overrides the dest-branch of the original
+project.  Same syntax as the corresponding element of `project`.
+
+Attribute `upstream`: If specified, overrides the upstream of the original
+project.  Same syntax as the corresponding element of `project`.
+
 ### Element annotation
 
 Zero or more annotation elements may be specified as children of a
@@ -471,7 +550,7 @@ These restrictions are not enforced for [Local Manifests].
 
 Attribute `groups`: List of additional groups to which all projects
 in the included manifest belong. This appends and recurses, meaning
-all projects in sub-manifests carry all parent include groups.
+all projects in included manifests carry all parent include groups.
 Same syntax as the corresponding element of `project`.
 
 ## Local Manifests {#local-manifests}

git_command.py

@@ -158,6 +158,8 @@ def git_require(min_version, fail=False, msg=''):
 
 
 class GitCommand(object):
+  """Wrapper around a single git invocation."""
+
   def __init__(self,
                project,
                cmdv,
@@ -169,7 +171,8 @@ class GitCommand(object):
                disable_editor=False,
                ssh_proxy=None,
                cwd=None,
-               gitdir=None):
+               gitdir=None,
+               objdir=None):
     env = self._GetBasicEnv()
 
     if disable_editor:
@@ -194,13 +197,24 @@ class GitCommand(object):
         cwd = project.worktree
       if not gitdir:
         gitdir = project.gitdir
+    # Git on Windows wants its paths only using / for reliability.
+    if platform_utils.isWindows():
+      if objdir:
+        objdir = objdir.replace('\\', '/')
+      if gitdir:
+        gitdir = gitdir.replace('\\', '/')
+
+    if objdir:
+      # Set to the place we want to save the objects.
+      env['GIT_OBJECT_DIRECTORY'] = objdir
+      if gitdir:
+        # Allow git to search the original place in case of local or unique refs
+        # that git will attempt to resolve even if we aren't fetching them.
+        env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = gitdir + '/objects'
 
     command = [GIT]
     if bare:
       if gitdir:
-        # Git on Windows wants its paths only using / for reliability.
-        if platform_utils.isWindows():
-          gitdir = gitdir.replace('\\', '/')
         env[GIT_DIR] = gitdir
       cwd = None
     command.append(cmdv[0])
@@ -216,12 +230,11 @@ class GitCommand(object):
     stderr = (subprocess.STDOUT if merge_output else
               (subprocess.PIPE if capture_stderr else None))
 
+    dbg = ''
     if IsTrace():
       global LAST_CWD
       global LAST_GITDIR
 
-      dbg = ''
-
       if cwd and LAST_CWD != cwd:
         if LAST_GITDIR or LAST_CWD:
           dbg += '\n'
@@ -234,6 +247,11 @@ class GitCommand(object):
         dbg += ': export GIT_DIR=%s\n' % env[GIT_DIR]
         LAST_GITDIR = env[GIT_DIR]
 
+      if 'GIT_OBJECT_DIRECTORY' in env:
+        dbg += ': export GIT_OBJECT_DIRECTORY=%s\n' % env['GIT_OBJECT_DIRECTORY']
+      if 'GIT_ALTERNATE_OBJECT_DIRECTORIES' in env:
+        dbg += ': export GIT_ALTERNATE_OBJECT_DIRECTORIES=%s\n' % env['GIT_ALTERNATE_OBJECT_DIRECTORIES']
+
       dbg += ': '
       dbg += ' '.join(command)
       if stdin == subprocess.PIPE:
@@ -244,36 +262,31 @@ class GitCommand(object):
         dbg += ' 2>|'
       elif stderr == subprocess.STDOUT:
         dbg += ' 2>&1'
-      Trace('%s', dbg)
 
-    try:
-      p = subprocess.Popen(command,
-                           cwd=cwd,
-                           env=env,
-                           encoding='utf-8',
-                           errors='backslashreplace',
-                           stdin=stdin,
-                           stdout=stdout,
-                           stderr=stderr)
-    except Exception as e:
-      raise GitError('%s: %s' % (command[1], e))
+    with Trace('git command %s %s with debug: %s', LAST_GITDIR, command, dbg):
+      try:
+        p = subprocess.Popen(command,
+                            cwd=cwd,
+                            env=env,
+                            encoding='utf-8',
+                            errors='backslashreplace',
+                            stdin=stdin,
+                            stdout=stdout,
+                            stderr=stderr)
+      except Exception as e:
+        raise GitError('%s: %s' % (command[1], e))
 
-    if ssh_proxy:
-      ssh_proxy.add_client(p)
-
-    self.process = p
-    if input:
-      if isinstance(input, str):
-        input = input.encode('utf-8')
-      p.stdin.write(input)
-      p.stdin.close()
-
-    try:
-      self.stdout, self.stderr = p.communicate()
-    finally:
       if ssh_proxy:
-        ssh_proxy.remove_client(p)
-    self.rc = p.wait()
+        ssh_proxy.add_client(p)
+
+      self.process = p
+
+      try:
+        self.stdout, self.stderr = p.communicate(input=input)
+      finally:
+        if ssh_proxy:
+          ssh_proxy.remove_client(p)
+      self.rc = p.wait()
 
   @staticmethod
   def _GetBasicEnv():

git_config.py

@@ -219,8 +219,8 @@ class GitConfig(object):
     """Set the value(s) for a key.
        Only this configuration file is modified.
 
-       The supplied value should be either a string,
-       or a list of strings (to store multiple values).
+       The supplied value should be either a string, or a list of strings (to
+       store multiple values), or None (to delete the key).
     """
     key = _key(name)
 
@@ -349,9 +349,9 @@ class GitConfig(object):
     except OSError:
       return None
     try:
-      Trace(': parsing %s', self.file)
-      with open(self._json) as fd:
-        return json.load(fd)
+      with Trace(': parsing %s', self.file):
+        with open(self._json) as fd:
+          return json.load(fd)
     except (IOError, ValueError):
       platform_utils.remove(self._json, missing_ok=True)
       return None

git_refs.py

@@ -67,38 +67,37 @@ class GitRefs(object):
       self._LoadAll()
 
   def _NeedUpdate(self):
-    Trace(': scan refs %s', self._gitdir)
-
-    for name, mtime in self._mtime.items():
-      try:
-        if mtime != os.path.getmtime(os.path.join(self._gitdir, name)):
+    with Trace(': scan refs %s', self._gitdir):
+      for name, mtime in self._mtime.items():
+        try:
+          if mtime != os.path.getmtime(os.path.join(self._gitdir, name)):
+            return True
+        except OSError:
           return True
-      except OSError:
-        return True
-    return False
+      return False
 
   def _LoadAll(self):
-    Trace(': load refs %s', self._gitdir)
+    with Trace(': load refs %s', self._gitdir):
 
-    self._phyref = {}
-    self._symref = {}
-    self._mtime = {}
+      self._phyref = {}
+      self._symref = {}
+      self._mtime = {}
 
-    self._ReadPackedRefs()
-    self._ReadLoose('refs/')
-    self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD)
+      self._ReadPackedRefs()
+      self._ReadLoose('refs/')
+      self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD)
 
-    scan = self._symref
-    attempts = 0
-    while scan and attempts < 5:
-      scan_next = {}
-      for name, dest in scan.items():
-        if dest in self._phyref:
-          self._phyref[name] = self._phyref[dest]
-        else:
-          scan_next[name] = dest
-      scan = scan_next
-      attempts += 1
+      scan = self._symref
+      attempts = 0
+      while scan and attempts < 5:
+        scan_next = {}
+        for name, dest in scan.items():
+          if dest in self._phyref:
+            self._phyref[name] = self._phyref[dest]
+          else:
+            scan_next[name] = dest
+        scan = scan_next
+        attempts += 1
 
   def _ReadPackedRefs(self):
     path = os.path.join(self._gitdir, 'packed-refs')

git_superproject.py

@@ -18,7 +18,7 @@ For more information on superproject, check out:
 https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects
 
 Examples:
-  superproject = Superproject()
+  superproject = Superproject(manifest, name, remote, revision)
   UpdateProjectsResult = superproject.UpdateProjectsRevisionId(projects)
 """
 
@@ -32,7 +32,6 @@ from typing import NamedTuple
 from git_command import git_require, GitCommand
 from git_config import RepoConfig
 from git_refs import R_HEADS
-from manifest_xml import LOCAL_MANIFEST_GROUP_PREFIX
 
 _SUPERPROJECT_GIT_NAME = 'superproject.git'
 _SUPERPROJECT_MANIFEST_NAME = 'superproject_override.xml'
@@ -72,41 +71,50 @@ class Superproject(object):
   lookup of commit ids for all projects. It contains _project_commit_ids which
   is a dictionary with project/commit id entries.
   """
-  def __init__(self, manifest, repodir, git_event_log,
-               superproject_dir='exp-superproject', quiet=False, print_messages=False):
+  def __init__(self, manifest, name, remote, revision,
+               superproject_dir='exp-superproject'):
     """Initializes superproject.
 
     Args:
       manifest: A Manifest object that is to be written to a file.
-      repodir: Path to the .repo/ dir for holding all internal checkout state.
-          It must be in the top directory of the repo client checkout.
-      git_event_log: A git trace2 event log to log events.
-      superproject_dir: Relative path under |repodir| to checkout superproject.
-      quiet:  If True then only print the progress messages.
-      print_messages: if True then print error/warning messages.
+      name: The unique name of the superproject
+      remote: The RemoteSpec for the remote.
+      revision: The name of the git branch to track.
+      superproject_dir: Relative path under |manifest.subdir| to checkout
+          superproject.
     """
     self._project_commit_ids = None
     self._manifest = manifest
-    self._git_event_log = git_event_log
-    self._quiet = quiet
-    self._print_messages = print_messages
-    self._branch = manifest.branch
-    self._repodir = os.path.abspath(repodir)
+    self.name = name
+    self.remote = remote
+    self.revision = self._branch = revision
+    self._repodir = manifest.repodir
     self._superproject_dir = superproject_dir
-    self._superproject_path = os.path.join(self._repodir, superproject_dir)
+    self._superproject_path = manifest.SubmanifestInfoDir(manifest.path_prefix,
+                                                          superproject_dir)
     self._manifest_path = os.path.join(self._superproject_path,
                                        _SUPERPROJECT_MANIFEST_NAME)
-    git_name = ''
-    if self._manifest.superproject:
-      remote = self._manifest.superproject['remote']
-      git_name = hashlib.md5(remote.name.encode('utf8')).hexdigest() + '-'
-      self._branch = self._manifest.superproject['revision']
-      self._remote_url = remote.url
-    else:
-      self._remote_url = None
+    git_name = hashlib.md5(remote.name.encode('utf8')).hexdigest() + '-'
+    self._remote_url = remote.url
     self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME
     self._work_git = os.path.join(self._superproject_path, self._work_git_name)
 
+    # The following are command arguemnts, rather than superproject attributes,
+    # and were included here originally.  They should eventually become
+    # arguments that are passed down from the public methods, instead of being
+    # treated as attributes.
+    self._git_event_log = None
+    self._quiet = False
+    self._print_messages = False
+
+  def SetQuiet(self, value):
+    """Set the _quiet attribute."""
+    self._quiet = value
+
+  def SetPrintMessages(self, value):
+    """Set the _print_messages attribute."""
+    self._print_messages = value
+
   @property
   def project_commit_ids(self):
     """Returns a dictionary of projects and their commit ids."""
@@ -215,19 +223,23 @@ class Superproject(object):
                        f'return code: {retval}, stderr: {p.stderr}')
     return data
 
-  def Sync(self):
+  def Sync(self, git_event_log):
     """Gets a local copy of a superproject for the manifest.
 
+    Args:
+      git_event_log: an EventLog, for git tracing.
+
     Returns:
       SyncResult
     """
+    self._git_event_log = git_event_log
     if not self._manifest.superproject:
       self._LogWarning(f'superproject tag is not defined in manifest: '
                        f'{self._manifest.manifestFile}')
       return SyncResult(False, False)
 
-    print('NOTICE: --use-superproject is in beta; report any issues to the '
-          'address described in `repo version`', file=sys.stderr)
+    _PrintBetaNotice()
+
     should_exit = True
     if not self._remote_url:
       self._LogWarning(f'superproject URL is not defined in manifest: '
@@ -248,7 +260,7 @@ class Superproject(object):
     Returns:
       CommitIdsResult
     """
-    sync_result = self.Sync()
+    sync_result = self.Sync(self._git_event_log)
     if not sync_result.success:
       return CommitIdsResult(None, sync_result.fatal)
 
@@ -283,7 +295,8 @@ class Superproject(object):
     if not os.path.exists(self._superproject_path):
       self._LogWarning(f'missing superproject directory: {self._superproject_path}')
       return None
-    manifest_str = self._manifest.ToXml(groups=self._manifest.GetGroupsStr()).toxml()
+    manifest_str = self._manifest.ToXml(groups=self._manifest.GetGroupsStr(),
+                                        omit_local=True).toxml()
     manifest_path = self._manifest_path
     try:
       with open(manifest_path, 'w', encoding='utf-8') as fp:
@@ -311,17 +324,19 @@ class Superproject(object):
     if project.revisionId:
       return True
     # Skip the project if it comes from the local manifest.
-    return any(s.startswith(LOCAL_MANIFEST_GROUP_PREFIX) for s in project.groups)
+    return project.manifest.IsFromLocalManifest(project)
 
-  def UpdateProjectsRevisionId(self, projects):
+  def UpdateProjectsRevisionId(self, projects, git_event_log):
     """Update revisionId of every project in projects with the commit id.
 
     Args:
-      projects: List of projects whose revisionId needs to be updated.
+      projects: a list of projects whose revisionId needs to be updated.
+      git_event_log: an EventLog, for git tracing.
 
     Returns:
       UpdateProjectsResult
     """
+    self._git_event_log = git_event_log
     commit_ids_result = self._GetAllProjectsCommitIds()
     commit_ids = commit_ids_result.commit_ids
     if not commit_ids:
@@ -351,6 +366,13 @@ class Superproject(object):
     return UpdateProjectsResult(manifest_path, False)
 
 
+@functools.lru_cache(maxsize=10)
+def _PrintBetaNotice():
+  """Print the notice of beta status."""
+  print('NOTICE: --use-superproject is in beta; report any issues to the '
+        'address described in `repo version`', file=sys.stderr)
+
+
 @functools.lru_cache(maxsize=None)
 def _UseSuperprojectFromConfiguration():
   """Returns the user choice of whether to use superproject."""
@@ -395,21 +417,37 @@ def _UseSuperprojectFromConfiguration():
   return False
 
 
-def PrintMessages(opt, manifest):
-  """Returns a boolean if error/warning messages are to be printed."""
-  return opt.use_superproject is not None or manifest.superproject
+def PrintMessages(use_superproject, manifest):
+  """Returns a boolean if error/warning messages are to be printed.
+
+  Args:
+    use_superproject: option value from optparse.
+    manifest: manifest to use.
+  """
+  return use_superproject is not None or bool(manifest.superproject)
 
 
-def UseSuperproject(opt, manifest):
-  """Returns a boolean if use-superproject option is enabled."""
+def UseSuperproject(use_superproject, manifest):
+  """Returns a boolean if use-superproject option is enabled.
 
-  if opt.use_superproject is not None:
-    return opt.use_superproject
+  Args:
+    use_superproject: option value from optparse.
+    manifest: manifest to use.
+
+  Returns:
+    Whether the superproject should be used.
+  """
+
+  if not manifest.superproject:
+    # This (sub) manifest does not have a superproject definition.
+    return False
+  elif use_superproject is not None:
+    return use_superproject
   else:
-    client_value = manifest.manifestProject.config.GetBoolean('repo.superproject')
+    client_value = manifest.manifestProject.use_superproject
     if client_value is not None:
       return client_value
-    else:
-      if not manifest.superproject:
-        return False
+    elif manifest.superproject:
       return _UseSuperprojectFromConfiguration()
+    else:
+      return False

git_trace2_event_log.py

@@ -29,8 +29,10 @@ https://git-scm.com/docs/api-trace2#_the_event_format_target
 
 
 import datetime
+import errno
 import json
 import os
+import socket
 import sys
 import tempfile
 import threading
@@ -218,20 +220,39 @@ class EventLog(object):
           retval, p.stderr), file=sys.stderr)
     return path
 
+  def _WriteLog(self, write_fn):
+    """Writes the log out using a provided writer function.
+
+    Generate compact JSON output for each item in the log, and write it using
+    write_fn.
+
+    Args:
+      write_fn: A function that accepts byts and writes them to a destination.
+    """
+
+    for e in self._log:
+      # Dump in compact encoding mode.
+      # See 'Compact encoding' in Python docs:
+      # https://docs.python.org/3/library/json.html#module-json
+      write_fn(json.dumps(e, indent=None, separators=(',', ':')).encode('utf-8') + b'\n')
+
   def Write(self, path=None):
-    """Writes the log out to a file.
+    """Writes the log out to a file or socket.
 
     Log is only written if 'path' or 'git config --get trace2.eventtarget'
-    provide a valid path to write logs to.
+    provide a valid path (or socket) to write logs to.
 
     Logging filename format follows the git trace2 style of being a unique
     (exclusive writable) file.
 
     Args:
-      path: Path to where logs should be written.
+      path: Path to where logs should be written. The path may have a prefix of
+          the form "af_unix:[{stream|dgram}:]", in which case the path is
+          treated as a Unix domain socket. See
+          https://git-scm.com/docs/api-trace2#_enabling_a_target for details.
 
     Returns:
-      log_path: Path to the log file if log is written, otherwise None
+      log_path: Path to the log file or socket if log is written, otherwise None
     """
     log_path = None
     # If no logging path is specified, get the path from 'trace2.eventtarget'.
@@ -242,29 +263,66 @@ class EventLog(object):
     if path is None:
       return None
 
+    path_is_socket = False
+    socket_type = None
     if isinstance(path, str):
-      # Get absolute path.
-      path = os.path.abspath(os.path.expanduser(path))
+      parts = path.split(':', 1)
+      if parts[0] == 'af_unix' and len(parts) == 2:
+        path_is_socket = True
+        path = parts[1]
+        parts = path.split(':', 1)
+        if parts[0] == 'stream' and len(parts) == 2:
+          socket_type = socket.SOCK_STREAM
+          path = parts[1]
+        elif parts[0] == 'dgram' and len(parts) == 2:
+          socket_type = socket.SOCK_DGRAM
+          path = parts[1]
+      else:
+        # Get absolute path.
+        path = os.path.abspath(os.path.expanduser(path))
     else:
       raise TypeError('path: str required but got %s.' % type(path))
 
     # Git trace2 requires a directory to write log to.
 
     # TODO(https://crbug.com/gerrit/13706): Support file (append) mode also.
-    if not os.path.isdir(path):
+    if not (path_is_socket or os.path.isdir(path)):
       return None
+
+    if path_is_socket:
+      if socket_type == socket.SOCK_STREAM or socket_type is None:
+        try:
+          with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
+            sock.connect(path)
+            self._WriteLog(sock.sendall)
+          return f'af_unix:stream:{path}'
+        except OSError as err:
+          # If we tried to connect to a DGRAM socket using STREAM, ignore the
+          # attempt and continue to DGRAM below. Otherwise, issue a warning.
+          if err.errno != errno.EPROTOTYPE:
+            print(f'repo: warning: git trace2 logging failed: {err}', file=sys.stderr)
+            return None
+      if socket_type == socket.SOCK_DGRAM or socket_type is None:
+        try:
+          with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
+            self._WriteLog(lambda bs: sock.sendto(bs, path))
+            return f'af_unix:dgram:{path}'
+        except OSError as err:
+          print(f'repo: warning: git trace2 logging failed: {err}', file=sys.stderr)
+          return None
+      # Tried to open a socket but couldn't connect (SOCK_STREAM) or write
+      # (SOCK_DGRAM).
+      print('repo: warning: git trace2 logging failed: could not write to socket', file=sys.stderr)
+      return None
+
+    # Path is an absolute path
     # Use NamedTemporaryFile to generate a unique filename as required by git trace2.
     try:
-      with tempfile.NamedTemporaryFile(mode='x', prefix=self._sid, dir=path,
+      with tempfile.NamedTemporaryFile(mode='xb', prefix=self._sid, dir=path,
                                        delete=False) as f:
         # TODO(https://crbug.com/gerrit/13706): Support writing events as they
         # occur.
-        for e in self._log:
-          # Dump in compact encoding mode.
-          # See 'Compact encoding' in Python docs:
-          # https://docs.python.org/3/library/json.html#module-json
-          json.dump(e, f, indent=None, separators=(',', ':'))
-          f.write('\n')
+        self._WriteLog(f.write)
         log_path = f.name
     except FileExistsError as err:
       print('repo: warning: git trace2 logging failed: %r' % err,

hooks/commit-msg

@@ -1,5 +1,5 @@
 #!/bin/sh
-# From Gerrit Code Review 3.1.3
+# From Gerrit Code Review 3.6.1 c67916dbdc07555c44e32a68f92ffc484b9b34f0
 #
 # Part of Gerrit Code Review (https://www.gerritcodereview.com/)
 #
@@ -17,6 +17,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -u
+
 # avoid [[ which is not POSIX sh.
 if test "$#" != 1 ; then
   echo "$0 requires an argument."
@@ -29,15 +31,25 @@ if test ! -f "$1" ; then
 fi
 
 # Do not create a change id if requested
-if test "false" = "`git config --bool --get gerrit.createChangeId`" ; then
+if test "false" = "$(git config --bool --get gerrit.createChangeId)" ; then
   exit 0
 fi
 
-# $RANDOM will be undefined if not using bash, so don't use set -u
-random=$( (whoami ; hostname ; date; cat $1 ; echo $RANDOM) | git hash-object --stdin)
+# Do not create a change id for squash commits.
+if head -n1 "$1" | grep -q '^squash! '; then
+  exit 0
+fi
+
+if git rev-parse --verify HEAD >/dev/null 2>&1; then
+  refhash="$(git rev-parse HEAD)"
+else
+  refhash="$(git hash-object -t tree /dev/null)"
+fi
+
+random=$({ git var GIT_COMMITTER_IDENT ; echo "$refhash" ; cat "$1"; } | git hash-object --stdin)
 dest="$1.tmp.${random}"
 
-trap 'rm -f "${dest}"' EXIT
+trap 'rm -f "$dest" "$dest-2"' EXIT
 
 if ! git stripspace --strip-comments < "$1" > "${dest}" ; then
    echo "cannot strip comments from $1"
@@ -49,11 +61,40 @@ if test ! -s "${dest}" ; then
   exit 1
 fi
 
+reviewurl="$(git config --get gerrit.reviewUrl)"
+if test -n "${reviewurl}" ; then
+  token="Link"
+  value="${reviewurl%/}/id/I$random"
+  pattern=".*/id/I[0-9a-f]\{40\}$"
+else
+  token="Change-Id"
+  value="I$random"
+  pattern=".*"
+fi
+
+if git interpret-trailers --parse < "$1" | grep -q "^$token: $pattern$" ; then
+  exit 0
+fi
+
+# There must be a Signed-off-by trailer for the code below to work. Insert a
+# sentinel at the end to make sure there is one.
 # Avoid the --in-place option which only appeared in Git 2.8
-# Avoid the --if-exists option which only appeared in Git 2.15
-if ! git -c trailer.ifexists=doNothing interpret-trailers \
-      --trailer "Change-Id: I${random}" < "$1" > "${dest}" ; then
-  echo "cannot insert change-id line in $1"
+if ! git interpret-trailers \
+         --trailer "Signed-off-by: SENTINEL" < "$1" > "$dest-2" ; then
+  echo "cannot insert Signed-off-by sentinel line in $1"
+  exit 1
+fi
+
+# Make sure the trailer appears before any Signed-off-by trailers by inserting
+# it as if it was a Signed-off-by trailer and then use sed to remove the
+# Signed-off-by prefix and the Signed-off-by sentinel line.
+# Avoid the --in-place option which only appeared in Git 2.8
+# Avoid the --where option which only appeared in Git 2.15
+if ! git -c trailer.where=before interpret-trailers \
+         --trailer "Signed-off-by: $token: $value" < "$dest-2" |
+     sed -re "s/^Signed-off-by: ($token: )/\1/" \
+         -e "/^Signed-off-by: SENTINEL/d" > "$dest" ; then
+  echo "cannot insert $token line in $1"
   exit 1
 fi
 

main.py

@@ -37,7 +37,7 @@ except ImportError:
 
 from color import SetDefaultColoring
 import event_log
-from repo_trace import SetTrace
+from repo_trace import SetTrace, Trace, SetTraceToStderr
 from git_command import user_agent
 from git_config import RepoConfig
 from git_trace2_event_log import EventLog
@@ -109,6 +109,9 @@ global_options.add_option('--color',
 global_options.add_option('--trace',
                           dest='trace', action='store_true',
                           help='trace git command execution (REPO_TRACE=1)')
+global_options.add_option('--trace-to-stderr',
+                          dest='trace_to_stderr', action='store_true',
+                          help='trace outputs go to stderr in addition to .repo/TRACE_FILE')
 global_options.add_option('--trace-python',
                           dest='trace_python', action='store_true',
                           help='trace python command execution')
@@ -127,6 +130,8 @@ global_options.add_option('--event-log',
                           help='filename of event log to append timeline to')
 global_options.add_option('--git-trace2-event-log', action='store',
                           help='directory to write git trace2 event log to')
+global_options.add_option('--submanifest-path', action='store',
+                          metavar='REL_PATH', help='submanifest path')
 
 
 class _Repo(object):
@@ -196,9 +201,6 @@ class _Repo(object):
     """Execute the requested subcommand."""
     result = 0
 
-    if gopts.trace:
-      SetTrace()
-
     # Handle options that terminate quickly first.
     if gopts.help or gopts.help_all:
       self._PrintHelp(short=False, all_commands=gopts.help_all)
@@ -214,10 +216,30 @@ class _Repo(object):
       self._PrintHelp(short=True)
       return 1
 
+    run = lambda: self._RunLong(name, gopts, argv) or 0
+    with Trace('starting new command: %s', ', '.join([name] + argv),
+               first_trace=True):
+      if gopts.trace_python:
+        import trace
+        tracer = trace.Trace(count=False, trace=True, timing=True,
+                             ignoredirs=set(sys.path[1:]))
+        result = tracer.runfunc(run)
+      else:
+        result = run()
+    return result
+
+  def _RunLong(self, name, gopts, argv):
+    """Execute the (longer running) requested subcommand."""
+    result = 0
     SetDefaultColoring(gopts.color)
 
     git_trace2_event_log = EventLog()
-    repo_client = RepoClient(self.repodir)
+    outer_client = RepoClient(self.repodir)
+    repo_client = outer_client
+    if gopts.submanifest_path:
+      repo_client = RepoClient(self.repodir,
+                               submanifest_path=gopts.submanifest_path,
+                               outer_client=outer_client)
     gitc_manifest = None
     gitc_client_name = gitc_utils.parse_clientdir(os.getcwd())
     if gitc_client_name:
@@ -229,6 +251,8 @@ class _Repo(object):
           repodir=self.repodir,
           client=repo_client,
           manifest=repo_client.manifest,
+          outer_client=outer_client,
+          outer_manifest=outer_client.manifest,
           gitc_manifest=gitc_manifest,
           git_event_log=git_trace2_event_log)
     except KeyError:
@@ -283,7 +307,36 @@ class _Repo(object):
     try:
       cmd.CommonValidateOptions(copts, cargs)
       cmd.ValidateOptions(copts, cargs)
-      result = cmd.Execute(copts, cargs)
+
+      this_manifest_only = copts.this_manifest_only
+      outer_manifest = copts.outer_manifest
+      if cmd.MULTI_MANIFEST_SUPPORT or this_manifest_only:
+        result = cmd.Execute(copts, cargs)
+      elif outer_manifest and repo_client.manifest.is_submanifest:
+        # The command does not support multi-manifest, we are using a
+        # submanifest, and the command line is for the outermost manifest.
+        # Re-run using the outermost manifest, which will recurse through the
+        # submanifests.
+        gopts.submanifest_path = ''
+        result = self._Run(name, gopts, argv)
+      else:
+        # No multi-manifest support. Run the command in the current
+        # (sub)manifest, and then any child submanifests.
+        result = cmd.Execute(copts, cargs)
+        for submanifest in repo_client.manifest.submanifests.values():
+          spec = submanifest.ToSubmanifestSpec()
+          gopts.submanifest_path = submanifest.repo_client.path_prefix
+          child_argv = argv[:]
+          child_argv.append('--no-outer-manifest')
+          # Not all subcommands support the 3 manifest options, so only add them
+          # if the original command includes them.
+          if hasattr(copts, 'manifest_url'):
+            child_argv.extend(['--manifest-url', spec.manifestUrl])
+          if hasattr(copts, 'manifest_name'):
+            child_argv.extend(['--manifest-name', spec.manifestName])
+          if hasattr(copts, 'manifest_branch'):
+            child_argv.extend(['--manifest-branch', spec.revision])
+          result = self._Run(name, gopts, child_argv) or result
     except (DownloadError, ManifestInvalidRevisionError,
             NoManifestException) as e:
       print('error: in `%s`: %s' % (' '.join([name] + argv), str(e)),
@@ -614,17 +667,18 @@ def _Main(argv):
   Version.wrapper_path = opt.wrapper_path
 
   repo = _Repo(opt.repodir)
+
   try:
     init_http()
     name, gopts, argv = repo._ParseArgs(argv)
-    run = lambda: repo._Run(name, gopts, argv) or 0
-    if gopts.trace_python:
-      import trace
-      tracer = trace.Trace(count=False, trace=True, timing=True,
-                           ignoredirs=set(sys.path[1:]))
-      result = tracer.runfunc(run)
-    else:
-      result = run()
+
+    if gopts.trace:
+      SetTrace()
+
+    if gopts.trace_to_stderr:
+      SetTraceToStderr()
+
+    result = repo._Run(name, gopts, argv) or 0
   except KeyboardInterrupt:
     print('aborted by user', file=sys.stderr)
     result = 1

man/repo-abandon.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo abandon" "Repo Manual"
+.TH REPO "1" "July 2022" "repo abandon" "Repo Manual"
 .SH NAME
 repo \- repo abandon - manual page for repo abandon
 .SH SYNOPSIS
@@ -32,5 +32,18 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help abandon` to view the detailed manual.

man/repo-branches.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo branches" "Repo Manual"
+.TH REPO "1" "July 2022" "repo branches" "Repo Manual"
 .SH NAME
 repo \- repo branches - manual page for repo branches
 .SH SYNOPSIS
@@ -55,5 +55,18 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help branches` to view the detailed manual.

man/repo-checkout.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo checkout" "Repo Manual"
+.TH REPO "1" "July 2022" "repo checkout" "Repo Manual"
 .SH NAME
 repo \- repo checkout - manual page for repo checkout
 .SH SYNOPSIS
@@ -24,6 +24,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help checkout` to view the detailed manual.
 .SH DETAILS

man/repo-cherry-pick.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo cherry-pick" "Repo Manual"
+.TH REPO "1" "July 2022" "repo cherry-pick" "Repo Manual"
 .SH NAME
 repo \- repo cherry-pick - manual page for repo cherry-pick
 .SH SYNOPSIS
@@ -20,6 +20,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help cherry\-pick` to view the detailed manual.
 .SH DETAILS

man/repo-diff.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo diff" "Repo Manual"
+.TH REPO "1" "July 2022" "repo diff" "Repo Manual"
 .SH NAME
 repo \- repo diff - manual page for repo diff
 .SH SYNOPSIS
@@ -31,5 +31,18 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help diff` to view the detailed manual.

man/repo-diffmanifests.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo diffmanifests" "Repo Manual"
+.TH REPO "1" "July 2022" "repo diffmanifests" "Repo Manual"
 .SH NAME
 repo \- repo diffmanifests - manual page for repo diffmanifests
 .SH SYNOPSIS
@@ -29,6 +29,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help diffmanifests` to view the detailed manual.
 .SH DETAILS

man/repo-download.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo download" "Repo Manual"
+.TH REPO "1" "July 2022" "repo download" "Repo Manual"
 .SH NAME
 repo \- repo download - manual page for repo download
 .SH SYNOPSIS
@@ -35,6 +35,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help download` to view the detailed manual.
 .SH DETAILS

man/repo-forall.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo forall" "Repo Manual"
+.TH REPO "1" "July 2022" "repo forall" "Repo Manual"
 .SH NAME
 repo \- repo forall - manual page for repo forall
 .SH SYNOPSIS
@@ -54,6 +54,19 @@ only show errors
 .TP
 \fB\-p\fR
 show project headers before output
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help forall` to view the detailed manual.
 .SH DETAILS
@@ -93,6 +106,11 @@ REPO_PROJECT is set to the unique name of the project.
 .PP
 REPO_PATH is the path relative the the root of the client.
 .PP
+REPO_OUTERPATH is the path of the sub manifest's root relative to the root of
+the client.
+.PP
+REPO_INNERPATH is the path relative to the root of the sub manifest.
+.PP
 REPO_REMOTE is the name of the remote system from the manifest.
 .PP
 REPO_LREV is the name of the revision from the manifest, translated to a local

man/repo-gitc-delete.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo gitc-delete" "Repo Manual"
+.TH REPO "1" "July 2022" "repo gitc-delete" "Repo Manual"
 .SH NAME
 repo \- repo gitc-delete - manual page for repo gitc-delete
 .SH SYNOPSIS
@@ -23,6 +23,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help gitc\-delete` to view the detailed manual.
 .SH DETAILS

man/repo-gitc-init.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "November 2021" "repo gitc-init" "Repo Manual"
+.TH REPO "1" "October 2022" "repo gitc-init" "Repo Manual"
 .SH NAME
 repo \- repo gitc-init - manual page for repo gitc-init
 .SH SYNOPSIS
@@ -45,10 +45,15 @@ sync any submodules associated with the manifest repo
 \fB\-\-standalone\-manifest\fR
 download the manifest as a static file rather then
 create a git checkout of the manifest repo
+.TP
+\fB\-\-manifest\-depth\fR=\fI\,DEPTH\/\fR
+create a shallow clone of the manifest repo with given
+depth (0 for full clone); see git clone (default: 0)
 .SS Manifest (only) checkout options:
 .TP
 \fB\-\-current\-branch\fR
 fetch only current manifest branch from server
+(default)
 .TP
 \fB\-\-no\-current\-branch\fR
 fetch all manifest branches from server
@@ -109,6 +114,12 @@ not \fB\-\-partial\-clone\fR)
 \fB\-\-no\-clone\-bundle\fR
 disable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS (default if
 \fB\-\-partial\-clone\fR)
+.TP
+\fB\-\-git\-lfs\fR
+enable Git LFS support
+.TP
+\fB\-\-no\-git\-lfs\fR
+disable Git LFS support
 .SS repo Version options:
 .TP
 \fB\-\-repo\-url\fR=\fI\,URL\/\fR
@@ -130,6 +141,19 @@ Optional manifest file to use for this GITC client.
 .TP
 \fB\-c\fR GITC_CLIENT, \fB\-\-gitc\-client\fR=\fI\,GITC_CLIENT\/\fR
 Name of the gitc_client instance to create or modify.
+.SS Multi\-manifest:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help gitc\-init` to view the detailed manual.
 .SH DETAILS

man/repo-grep.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo grep" "Repo Manual"
+.TH REPO "1" "July 2022" "repo grep" "Repo Manual"
 .SH NAME
 repo \- repo grep - manual page for repo grep
 .SH SYNOPSIS
@@ -24,6 +24,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .SS Sources:
 .TP
 \fB\-\-cached\fR

man/repo-help.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo help" "Repo Manual"
+.TH REPO "1" "July 2022" "repo help" "Repo Manual"
 .SH NAME
 repo \- repo help - manual page for repo help
 .SH SYNOPSIS
@@ -26,6 +26,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help help` to view the detailed manual.
 .SH DETAILS

man/repo-info.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo info" "Repo Manual"
+.TH REPO "1" "July 2022" "repo info" "Repo Manual"
 .SH NAME
 repo \- repo info - manual page for repo info
 .SH SYNOPSIS
@@ -36,5 +36,18 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help info` to view the detailed manual.

man/repo-init.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "November 2021" "repo init" "Repo Manual"
+.TH REPO "1" "October 2022" "repo init" "Repo Manual"
 .SH NAME
 repo \- repo init - manual page for repo init
 .SH SYNOPSIS
@@ -45,10 +45,15 @@ sync any submodules associated with the manifest repo
 \fB\-\-standalone\-manifest\fR
 download the manifest as a static file rather then
 create a git checkout of the manifest repo
+.TP
+\fB\-\-manifest\-depth\fR=\fI\,DEPTH\/\fR
+create a shallow clone of the manifest repo with given
+depth (0 for full clone); see git clone (default: 0)
 .SS Manifest (only) checkout options:
 .TP
 \fB\-c\fR, \fB\-\-current\-branch\fR
 fetch only current manifest branch from server
+(default)
 .TP
 \fB\-\-no\-current\-branch\fR
 fetch all manifest branches from server
@@ -109,6 +114,12 @@ not \fB\-\-partial\-clone\fR)
 \fB\-\-no\-clone\-bundle\fR
 disable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS (default if
 \fB\-\-partial\-clone\fR)
+.TP
+\fB\-\-git\-lfs\fR
+enable Git LFS support
+.TP
+\fB\-\-no\-git\-lfs\fR
+disable Git LFS support
 .SS repo Version options:
 .TP
 \fB\-\-repo\-url\fR=\fI\,URL\/\fR
@@ -123,6 +134,19 @@ do not verify repo source code
 .TP
 \fB\-\-config\-name\fR
 Always prompt for name/e\-mail
+.SS Multi\-manifest:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help init` to view the detailed manual.
 .SH DETAILS

man/repo-list.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo list" "Repo Manual"
+.TH REPO "1" "July 2022" "repo list" "Repo Manual"
 .SH NAME
 repo \- repo list - manual page for repo list
 .SH SYNOPSIS
@@ -47,6 +47,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help list` to view the detailed manual.
 .SH DETAILS

man/repo-manifest.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "November 2021" "repo manifest" "Repo Manual"
+.TH REPO "1" "October 2022" "repo manifest" "Repo Manual"
 .SH NAME
 repo \- repo manifest - manual page for repo manifest
 .SH SYNOPSIS
@@ -40,7 +40,8 @@ format output for humans to read
 ignore local manifests
 .TP
 \fB\-o\fR \-|NAME.xml, \fB\-\-output\-file\fR=\fI\,\-\/\fR|NAME.xml
-file to save the manifest to
+file to save the manifest to. (Filename prefix for
+multi\-tree.)
 .SS Logging options:
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR
@@ -48,6 +49,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help manifest` to view the detailed manual.
 .SH DETAILS
@@ -88,6 +102,7 @@ A manifest XML file (e.g. `default.xml`) roughly conforms to the following DTD:
 remote*,
 default?,
 manifest\-server?,
+submanifest*?,
 remove\-project*,
 project*,
 extend\-project*,
@@ -118,6 +133,16 @@ include*)>
 .IP
 <!ELEMENT manifest\-server EMPTY>
 <!ATTLIST manifest\-server url CDATA #REQUIRED>
+.IP
+<!ELEMENT submanifest EMPTY>
+<!ATTLIST submanifest name           ID #REQUIRED>
+<!ATTLIST submanifest remote         IDREF #IMPLIED>
+<!ATTLIST submanifest project        CDATA #IMPLIED>
+<!ATTLIST submanifest manifest\-name  CDATA #IMPLIED>
+<!ATTLIST submanifest revision       CDATA #IMPLIED>
+<!ATTLIST submanifest path           CDATA #IMPLIED>
+<!ATTLIST submanifest groups         CDATA #IMPLIED>
+<!ATTLIST submanifest default\-groups CDATA #IMPLIED>
 .TP
 <!ELEMENT project (annotation*,
 project*,
@@ -165,6 +190,8 @@ CDATA #IMPLIED>
 <!ATTLIST extend\-project groups CDATA #IMPLIED>
 <!ATTLIST extend\-project revision CDATA #IMPLIED>
 <!ATTLIST extend\-project remote CDATA #IMPLIED>
+<!ATTLIST extend\-project dest\-branch CDATA #IMPLIED>
+<!ATTLIST extend\-project upstream CDATA #IMPLIED>
 .IP
 <!ELEMENT remove\-project EMPTY>
 <!ATTLIST remove\-project name  CDATA #REQUIRED>
@@ -295,6 +322,65 @@ GetManifest(tag)
 Return a manifest in which each project is pegged to the revision at the
 specified tag. This is used by repo sync when the \fB\-\-smart\-tag\fR option is given.
 .PP
+Element submanifest
+.PP
+One or more submanifest elements may be specified. Each element describes a
+single manifest to be checked out as a child.
+.PP
+Attribute `name`: A unique name (within the current (sub)manifest) for this
+submanifest. It acts as a default for `revision` below. The same name can be
+used for submanifests with different parent (sub)manifests.
+.PP
+Attribute `remote`: Name of a previously defined remote element. If not supplied
+the remote given by the default element is used.
+.PP
+Attribute `project`: The manifest project name. The project's name is appended
+onto its remote's fetch URL to generate the actual URL to configure the Git
+remote with. The URL gets formed as:
+.IP
+${remote_fetch}/${project_name}.git
+.PP
+where ${remote_fetch} is the remote's fetch attribute and ${project_name} is the
+project's name attribute. The suffix ".git" is always appended as repo assumes
+the upstream is a forest of bare Git repositories. If the project has a parent
+element, its name will be prefixed by the parent's.
+.PP
+The project name must match the name Gerrit knows, if Gerrit is being used for
+code reviews.
+.PP
+`project` must not be empty, and may not be an absolute path or use "." or ".."
+path components. It is always interpreted relative to the remote's fetch
+settings, so if a different base path is needed, declare a different remote with
+the new settings needed.
+.PP
+If not supplied the remote and project for this manifest will be used: `remote`
+cannot be supplied.
+.PP
+Projects from a submanifest and its submanifests are added to the
+submanifest::path:<path_prefix> group.
+.PP
+Attribute `manifest\-name`: The manifest filename in the manifest project. If not
+supplied, `default.xml` is used.
+.PP
+Attribute `revision`: Name of a Git branch (e.g. "main" or "refs/heads/main"),
+tag (e.g. "refs/tags/stable"), or a commit hash. If not supplied, `name` is
+used.
+.PP
+Attribute `path`: An optional path relative to the top directory of the repo
+client where the submanifest repo client top directory should be placed. If not
+supplied, `revision` is used.
+.PP
+`path` may not be an absolute path or use "." or ".." path components.
+.PP
+Attribute `groups`: List of additional groups to which all projects in the
+included submanifest belong. This appends and recurses, meaning all projects in
+submanifests carry all parent submanifest groups. Same syntax as the
+corresponding element of `project`.
+.PP
+Attribute `default\-groups`: The list of manifest groups to sync if no
+`\-\-groups=` parameter was specified at init. When that list is empty, use this
+list instead of "default" as the list of groups to sync.
+.PP
 Element project
 .PP
 One or more project elements may be specified. Each element describes a single
@@ -401,6 +487,12 @@ project. Same syntax as the corresponding element of `project`.
 Attribute `remote`: If specified, overrides the remote of the original project.
 Same syntax as the corresponding element of `project`.
 .PP
+Attribute `dest\-branch`: If specified, overrides the dest\-branch of the original
+project. Same syntax as the corresponding element of `project`.
+.PP
+Attribute `upstream`: If specified, overrides the upstream of the original
+project. Same syntax as the corresponding element of `project`.
+.PP
 Element annotation
 .PP
 Zero or more annotation elements may be specified as children of a project or
@@ -513,10 +605,10 @@ restrictions are not enforced for [Local Manifests].
 .PP
 Attribute `groups`: List of additional groups to which all projects in the
 included manifest belong. This appends and recurses, meaning all projects in
-sub\-manifests carry all parent include groups. Same syntax as the corresponding
-element of `project`.
+included manifests carry all parent include groups. Same syntax as the
+corresponding element of `project`.
 .PP
-Local Manifests
+Local Manifests 
 .PP
 Additional remotes and projects may be added through local manifest files stored
 in `$TOP_DIR/.repo/local_manifests/*.xml`.

man/repo-overview.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo overview" "Repo Manual"
+.TH REPO "1" "July 2022" "repo overview" "Repo Manual"
 .SH NAME
 repo \- repo overview - manual page for repo overview
 .SH SYNOPSIS
@@ -26,6 +26,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help overview` to view the detailed manual.
 .SH DETAILS

man/repo-prune.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo prune" "Repo Manual"
+.TH REPO "1" "July 2022" "repo prune" "Repo Manual"
 .SH NAME
 repo \- repo prune - manual page for repo prune
 .SH SYNOPSIS
@@ -24,5 +24,18 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help prune` to view the detailed manual.

man/repo-rebase.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo rebase" "Repo Manual"
+.TH REPO "1" "July 2022" "repo rebase" "Repo Manual"
 .SH NAME
 repo \- repo rebase - manual page for repo rebase
 .SH SYNOPSIS
@@ -46,6 +46,19 @@ only show errors
 .TP
 \fB\-i\fR, \fB\-\-interactive\fR
 interactive rebase (single project only)
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help rebase` to view the detailed manual.
 .SH DETAILS

man/repo-selfupdate.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo selfupdate" "Repo Manual"
+.TH REPO "1" "July 2022" "repo selfupdate" "Repo Manual"
 .SH NAME
 repo \- repo selfupdate - manual page for repo selfupdate
 .SH SYNOPSIS
@@ -20,6 +20,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .SS repo Version options:
 .TP
 \fB\-\-no\-repo\-verify\fR

man/repo-smartsync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "November 2021" "repo smartsync" "Repo Manual"
+.TH REPO "1" "November 2022" "repo smartsync" "Repo Manual"
 .SH NAME
 repo \- repo smartsync - manual page for repo smartsync
 .SH SYNOPSIS
@@ -20,11 +20,11 @@ number of CPU cores)
 .TP
 \fB\-\-jobs\-network\fR=\fI\,JOBS\/\fR
 number of network jobs to run in parallel (defaults to
-\fB\-\-jobs\fR)
+\fB\-\-jobs\fR or 1)
 .TP
 \fB\-\-jobs\-checkout\fR=\fI\,JOBS\/\fR
 number of local checkout jobs to run in parallel
-(defaults to \fB\-\-jobs\fR)
+(defaults to \fB\-\-jobs\fR or 8)
 .TP
 \fB\-f\fR, \fB\-\-force\-broken\fR
 obsolete option (to be deleted in the future)
@@ -105,6 +105,13 @@ delete refs that no longer exist on the remote
 .TP
 \fB\-\-no\-prune\fR
 do not delete refs that no longer exist on the remote
+.TP
+\fB\-\-auto\-gc\fR
+run garbage collection on all synced projects
+.TP
+\fB\-\-no\-auto\-gc\fR
+do not run garbage collection on any projects
+(default)
 .SS Logging options:
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR
@@ -112,6 +119,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .SS repo Version options:
 .TP
 \fB\-\-no\-repo\-verify\fR

man/repo-stage.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo stage" "Repo Manual"
+.TH REPO "1" "July 2022" "repo stage" "Repo Manual"
 .SH NAME
 repo \- repo stage - manual page for repo stage
 .SH SYNOPSIS
@@ -23,6 +23,19 @@ only show errors
 .TP
 \fB\-i\fR, \fB\-\-interactive\fR
 use interactive staging
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help stage` to view the detailed manual.
 .SH DETAILS

man/repo-start.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo start" "Repo Manual"
+.TH REPO "1" "July 2022" "repo start" "Repo Manual"
 .SH NAME
 repo \- repo start - manual page for repo start
 .SH SYNOPSIS
@@ -33,6 +33,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help start` to view the detailed manual.
 .SH DETAILS

man/repo-status.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo status" "Repo Manual"
+.TH REPO "1" "July 2022" "repo status" "Repo Manual"
 .SH NAME
 repo \- repo status - manual page for repo status
 .SH SYNOPSIS
@@ -28,6 +28,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help status` to view the detailed manual.
 .SH DETAILS

man/repo-sync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "November 2021" "repo sync" "Repo Manual"
+.TH REPO "1" "November 2022" "repo sync" "Repo Manual"
 .SH NAME
 repo \- repo sync - manual page for repo sync
 .SH SYNOPSIS
@@ -20,11 +20,11 @@ number of CPU cores)
 .TP
 \fB\-\-jobs\-network\fR=\fI\,JOBS\/\fR
 number of network jobs to run in parallel (defaults to
-\fB\-\-jobs\fR)
+\fB\-\-jobs\fR or 1)
 .TP
 \fB\-\-jobs\-checkout\fR=\fI\,JOBS\/\fR
 number of local checkout jobs to run in parallel
-(defaults to \fB\-\-jobs\fR)
+(defaults to \fB\-\-jobs\fR or 8)
 .TP
 \fB\-f\fR, \fB\-\-force\-broken\fR
 obsolete option (to be deleted in the future)
@@ -106,6 +106,13 @@ delete refs that no longer exist on the remote
 \fB\-\-no\-prune\fR
 do not delete refs that no longer exist on the remote
 .TP
+\fB\-\-auto\-gc\fR
+run garbage collection on all synced projects
+.TP
+\fB\-\-no\-auto\-gc\fR
+do not run garbage collection on any projects
+(default)
+.TP
 \fB\-s\fR, \fB\-\-smart\-sync\fR
 smart sync using manifest from the latest known good
 build
@@ -119,6 +126,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .SS repo Version options:
 .TP
 \fB\-\-no\-repo\-verify\fR
@@ -187,6 +207,9 @@ to a sha1 revision if the sha1 revision does not already exist locally.
 The \fB\-\-prune\fR option can be used to remove any refs that no longer exist on the
 remote.
 .PP
+The \fB\-\-auto\-gc\fR option can be used to trigger garbage collection on all projects.
+By default, repo does not run garbage collection.
+.PP
 SSH Connections
 .PP
 If at least one project remote URL uses an SSH connection (ssh://, git+ssh://,

man/repo-upload.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo upload" "Repo Manual"
+.TH REPO "1" "August 2022" "repo upload" "Repo Manual"
 .SH NAME
 repo \- repo upload - manual page for repo upload
 .SH SYNOPSIS
@@ -54,6 +54,9 @@ upload as a private change (deprecated; use \fB\-\-wip\fR)
 \fB\-w\fR, \fB\-\-wip\fR
 upload as a work\-in\-progress change
 .TP
+\fB\-r\fR, \fB\-\-ready\fR
+mark change as ready (clears work\-in\-progress setting)
+.TP
 \fB\-o\fR PUSH_OPTIONS, \fB\-\-push\-option\fR=\fI\,PUSH_OPTIONS\/\fR
 additional push options to transmit
 .TP
@@ -66,6 +69,12 @@ do everything except actually upload the CL
 \fB\-y\fR, \fB\-\-yes\fR
 answer yes to all safe prompts
 .TP
+\fB\-\-ignore\-untracked\-files\fR
+ignore untracked files in the working copy
+.TP
+\fB\-\-no\-ignore\-untracked\-files\fR
+always ask about untracked files in the working copy
+.TP
 \fB\-\-no\-cert\-checks\fR
 disable verifying ssl certs (unsafe)
 .SS Logging options:
@@ -75,6 +84,19 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .SS pre\-upload hooks:
 .TP
 \fB\-\-no\-verify\fR
@@ -105,6 +127,12 @@ respective list of users, and emails are sent to any new users. Users passed as
 \fB\-\-reviewers\fR must already be registered with the code review system, or the
 upload will fail.
 .PP
+While most normal Gerrit options have dedicated command line options, direct
+access to the Gerit options is available via \fB\-\-push\-options\fR. This is useful when
+Gerrit has newer functionality that repo upload doesn't yet support, or doesn't
+have plans to support. See the Push Options documentation for more details:
+https://gerrit\-review.googlesource.com/Documentation/user\-upload.html#push_options
+.PP
 Configuration
 .PP
 review.URL.autoupload:

man/repo-version.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo version" "Repo Manual"
+.TH REPO "1" "July 2022" "repo version" "Repo Manual"
 .SH NAME
 repo \- repo version - manual page for repo version
 .SH SYNOPSIS
@@ -20,5 +20,18 @@ show all output
 .TP
 \fB\-q\fR, \fB\-\-quiet\fR
 only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
 .PP
 Run `repo help version` to view the detailed manual.

man/repo.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "November 2021" "repo" "Repo Manual"
+.TH REPO "1" "November 2022" "repo" "Repo Manual"
 .SH NAME
 repo \- repository management tool built on top of git
 .SH SYNOPSIS
@@ -25,6 +25,10 @@ control color usage: auto, always, never
 \fB\-\-trace\fR
 trace git command execution (REPO_TRACE=1)
 .TP
+\fB\-\-trace-to-stderr\fR
+trace outputs go to stderr in addition to
+\&.repo/TRACE_FILE
+.TP
 \fB\-\-trace\-python\fR
 trace python command execution
 .TP
@@ -43,6 +47,9 @@ filename of event log to append timeline to
 .TP
 \fB\-\-git\-trace2\-event\-log\fR=\fI\,GIT_TRACE2_EVENT_LOG\/\fR
 directory to write git trace2 event log to
+.TP
+\fB\-\-submanifest\-path\fR=\fI\,REL_PATH\/\fR
+submanifest path
 .SS "The complete list of recognized repo commands is:"
 .TP
 abandon

pager.py

@@ -56,8 +56,11 @@ def _PipePager(pager):
   global pager_process, old_stdout, old_stderr
   assert pager_process is None, "Only one active pager process at a time"
   # Create pager process, piping stdout/err into its stdin
-  pager_process = subprocess.Popen([pager], stdin=subprocess.PIPE, stdout=sys.stdout,
-                                   stderr=sys.stderr)
+  try:
+    pager_process = subprocess.Popen([pager], stdin=subprocess.PIPE, stdout=sys.stdout,
+                                     stderr=sys.stderr)
+  except FileNotFoundError:
+    sys.exit(f'fatal: cannot start pager "{pager}"')
   old_stdout = sys.stdout
   old_stderr = sys.stderr
   sys.stdout = pager_process.stdin

progress.py

@@ -15,7 +15,7 @@
 import os
 import sys
 from time import time
-from repo_trace import IsTrace
+from repo_trace import IsTraceToStderr
 
 _NOT_TTY = not os.isatty(2)
 
@@ -24,6 +24,11 @@ _NOT_TTY = not os.isatty(2)
 # column 0.
 CSI_ERASE_LINE = '\x1b[2K'
 
+# This will erase all content in the current line after the cursor.  This is
+# useful for partial updates & progress messages as the terminal can display
+# it better.
+CSI_ERASE_LINE_AFTER = '\x1b[K'
+
 
 def duration_str(total):
   """A less noisy timedelta.__str__.
@@ -75,7 +80,7 @@ class Progress(object):
   def update(self, inc=1, msg=''):
     self._done += inc
 
-    if _NOT_TTY or IsTrace():
+    if _NOT_TTY or IsTraceToStderr():
       return
 
     if not self._show:
@@ -85,10 +90,10 @@ class Progress(object):
         return
 
     if self._total <= 0:
-      sys.stderr.write('%s\r%s: %d,' % (
-          CSI_ERASE_LINE,
+      sys.stderr.write('\r%s: %d,%s' % (
           self._title,
-          self._done))
+          self._done,
+          CSI_ERASE_LINE_AFTER))
       sys.stderr.flush()
     else:
       p = (100 * self._done) / self._total
@@ -96,36 +101,36 @@ class Progress(object):
         jobs = '[%d job%s] ' % (self._active, 's' if self._active > 1 else '')
       else:
         jobs = ''
-      sys.stderr.write('%s\r%s: %2d%% %s(%d%s/%d%s)%s%s%s' % (
-          CSI_ERASE_LINE,
+      sys.stderr.write('\r%s: %2d%% %s(%d%s/%d%s)%s%s%s%s' % (
           self._title,
           p,
           jobs,
           self._done, self._units,
           self._total, self._units,
           ' ' if msg else '', msg,
+          CSI_ERASE_LINE_AFTER,
           '\n' if self._print_newline else ''))
       sys.stderr.flush()
 
   def end(self):
-    if _NOT_TTY or IsTrace() or not self._show:
+    if _NOT_TTY or IsTraceToStderr() or not self._show:
       return
 
     duration = duration_str(time() - self._start)
     if self._total <= 0:
-      sys.stderr.write('%s\r%s: %d, done in %s\n' % (
-          CSI_ERASE_LINE,
+      sys.stderr.write('\r%s: %d, done in %s%s\n' % (
           self._title,
           self._done,
-          duration))
+          duration,
+          CSI_ERASE_LINE_AFTER))
       sys.stderr.flush()
     else:
       p = (100 * self._done) / self._total
-      sys.stderr.write('%s\r%s: %3d%% (%d%s/%d%s), done in %s\n' % (
-          CSI_ERASE_LINE,
+      sys.stderr.write('\r%s: %3d%% (%d%s/%d%s), done in %s%s\n' % (
           self._title,
           p,
           self._done, self._units,
           self._total, self._units,
-          duration))
+          duration,
+          CSI_ERASE_LINE_AFTER))
       sys.stderr.flush()

release/update-manpages

@@ -59,18 +59,26 @@ def main(argv):
   version = RepoSourceVersion()
   cmdlist = [['help2man', '-N', '-n', f'repo {cmd} - manual page for repo {cmd}',
     '-S', f'repo {cmd}', '-m', 'Repo Manual', f'--version-string={version}',
-    '-o', MANDIR.joinpath(f'repo-{cmd}.1.tmp'), TOPDIR.joinpath('repo'),
+    '-o', MANDIR.joinpath(f'repo-{cmd}.1.tmp'), './repo',
     '-h', f'help {cmd}'] for cmd in subcmds.all_commands]
   cmdlist.append(['help2man', '-N', '-n', 'repository management tool built on top of git',
     '-S', 'repo', '-m', 'Repo Manual', f'--version-string={version}',
-    '-o', MANDIR.joinpath('repo.1.tmp'), TOPDIR.joinpath('repo'),
+    '-o', MANDIR.joinpath('repo.1.tmp'), './repo',
     '-h', '--help-all'])
 
   with tempfile.TemporaryDirectory() as tempdir:
-    repo_dir = Path(tempdir) / '.repo'
+    tempdir = Path(tempdir)
+    repo_dir = tempdir / '.repo'
     repo_dir.mkdir()
     (repo_dir / 'repo').symlink_to(TOPDIR)
 
+    # Create a repo wrapper using the active Python executable.  We can't pass
+    # this directly to help2man as it's too simple, so insert it via shebang.
+    data = (TOPDIR / 'repo').read_text(encoding='utf-8')
+    tempbin = tempdir / 'repo'
+    tempbin.write_text(f'#!{sys.executable}\n' + data, encoding='utf-8')
+    tempbin.chmod(0o755)
+
     # Run all cmd in parallel, and wait for them to finish.
     with multiprocessing.Pool() as pool:
       pool.map(partial(worker, cwd=tempdir, check=True), cmdlist)

repo

@@ -149,7 +149,7 @@ if not REPO_REV:
 BUG_URL = 'https://bugs.chromium.org/p/gerrit/issues/entry?template=Repo+tool+issue'
 
 # increment this whenever we make important changes to this script
-VERSION = (2, 17)
+VERSION = (2, 29)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (2, 3)
@@ -316,6 +316,10 @@ def InitParser(parser, gitc_init=False):
                    help='download the manifest as a static file '
                         'rather then create a git checkout of '
                         'the manifest repo')
+  group.add_option('--manifest-depth', type='int', default=0, metavar='DEPTH',
+                   help='create a shallow clone of the manifest repo with '
+                        'given depth (0 for full clone); see git clone '
+                        '(default: %default)')
 
   # Options that only affect manifest project, and not any of the projects
   # specified in the manifest itself.
@@ -325,9 +329,9 @@ def InitParser(parser, gitc_init=False):
   # want -c, so try to satisfy both as best we can.
   if not gitc_init:
     cbr_opts += ['-c']
-  group.add_option(*cbr_opts,
+  group.add_option(*cbr_opts, default=True,
                    dest='current_branch_only', action='store_true',
-                   help='fetch only current manifest branch from server')
+                   help='fetch only current manifest branch from server (default)')
   group.add_option('--no-current-branch',
                    dest='current_branch_only', action='store_false',
                    help='fetch all manifest branches from server')
@@ -382,6 +386,11 @@ def InitParser(parser, gitc_init=False):
   group.add_option('--no-clone-bundle',
                    dest='clone_bundle', action='store_false',
                    help='disable use of /clone.bundle on HTTP/HTTPS (default if --partial-clone)')
+  group.add_option('--git-lfs', action='store_true',
+                   help='enable Git LFS support')
+  group.add_option('--no-git-lfs',
+                   dest='git_lfs', action='store_false',
+                   help='disable Git LFS support')
 
   # Tool.
   group = parser.add_option_group('repo Version options')
@@ -607,17 +616,23 @@ def _Init(args, gitc_init=False):
   try:
     if not opt.quiet:
       print('Downloading Repo source from', url)
-    dst = os.path.abspath(os.path.join(repodir, S_repo))
+    dst_final = os.path.abspath(os.path.join(repodir, S_repo))
+    dst = dst_final + '.tmp'
+    shutil.rmtree(dst, ignore_errors=True)
     _Clone(url, dst, opt.clone_bundle, opt.quiet, opt.verbose)
 
     remote_ref, rev = check_repo_rev(dst, rev, opt.repo_verify, quiet=opt.quiet)
     _Checkout(dst, remote_ref, rev, opt.quiet)
 
     if not os.path.isfile(os.path.join(dst, 'repo')):
-      print("warning: '%s' does not look like a git-repo repository, is "
-            "REPO_URL set correctly?" % url, file=sys.stderr)
+      print("fatal: '%s' does not look like a git-repo repository, is "
+            "--repo-url set correctly?" % url, file=sys.stderr)
+      raise CloneFailure()
+
+    os.rename(dst, dst_final)
 
   except CloneFailure:
+    print('fatal: double check your --repo-rev setting.', file=sys.stderr)
     if opt.quiet:
       print('fatal: repo init failed; run without --quiet to see why',
             file=sys.stderr)
@@ -1311,6 +1326,7 @@ def main(orig_args):
         print("fatal: cloning the git-repo repository failed, will remove "
               "'%s' " % path, file=sys.stderr)
         shutil.rmtree(path, ignore_errors=True)
+        shutil.rmtree(path + '.tmp', ignore_errors=True)
         sys.exit(1)
       repo_main, rel_repo_dir = _FindRepo()
     else:

repo_trace.py

@@ -15,26 +15,124 @@
 """Logic for tracing repo interactions.
 
 Activated via `repo --trace ...` or `REPO_TRACE=1 repo ...`.
+
+Temporary: Tracing is always on. Set `REPO_TRACE=0` to turn off.
+To also include trace outputs in stderr do `repo --trace_to_stderr ...`
 """
 
 import sys
 import os
+import time
+from contextlib import ContextDecorator
+
+import platform_utils
 
 # Env var to implicitly turn on tracing.
 REPO_TRACE = 'REPO_TRACE'
 
-_TRACE = os.environ.get(REPO_TRACE) == '1'
+# Temporarily set tracing to always on unless user expicitly sets to 0.
+_TRACE = os.environ.get(REPO_TRACE) != '0'
+_TRACE_TO_STDERR = False
+_TRACE_FILE = None
+_TRACE_FILE_NAME = 'TRACE_FILE'
+_MAX_SIZE = 70  # in mb
+_NEW_COMMAND_SEP = '+++++++++++++++NEW COMMAND+++++++++++++++++++'
+
+
+def IsTraceToStderr():
+  return _TRACE_TO_STDERR
 
 
 def IsTrace():
   return _TRACE
 
 
+def SetTraceToStderr():
+  global _TRACE_TO_STDERR
+  _TRACE_TO_STDERR = True
+
+
 def SetTrace():
   global _TRACE
   _TRACE = True
 
 
-def Trace(fmt, *args):
-  if IsTrace():
-    print(fmt % args, file=sys.stderr)
+def _SetTraceFile():
+  global _TRACE_FILE
+  _TRACE_FILE = _GetTraceFile()
+
+
+class Trace(ContextDecorator):
+
+    def _time(self):
+      """Generate nanoseconds of time in a py3.6 safe way"""
+      return int(time.time()*1e+9)
+
+    def __init__(self, fmt, *args, first_trace=False):
+      if not IsTrace():
+        return
+      self._trace_msg = fmt % args
+
+      if not _TRACE_FILE:
+        _SetTraceFile()
+
+      if first_trace:
+        _ClearOldTraces()
+        self._trace_msg = '%s %s' % (_NEW_COMMAND_SEP, self._trace_msg)
+
+
+    def __enter__(self):
+      if not IsTrace():
+        return self
+
+      print_msg = f'PID: {os.getpid()} START: {self._time()} :' + self._trace_msg + '\n'
+
+      with open(_TRACE_FILE, 'a') as f:
+        print(print_msg, file=f)
+
+      if _TRACE_TO_STDERR:
+        print(print_msg, file=sys.stderr)
+
+      return self
+
+    def __exit__(self, *exc):
+      if not IsTrace():
+        return False
+
+      print_msg = f'PID: {os.getpid()} END: {self._time()} :' + self._trace_msg + '\n'
+
+      with open(_TRACE_FILE, 'a') as f:
+        print(print_msg, file=f)
+
+      if _TRACE_TO_STDERR:
+        print(print_msg, file=sys.stderr)
+
+      return False
+
+
+def _GetTraceFile():
+  """Get the trace file or create one."""
+  # TODO: refactor to pass repodir to Trace.
+  repo_dir = os.path.dirname(os.path.dirname(__file__))
+  trace_file = os.path.join(repo_dir, _TRACE_FILE_NAME)
+  print('Trace outputs in %s' % trace_file, file=sys.stderr)
+  return trace_file
+
+def _ClearOldTraces():
+  """Clear the oldest commands if trace file is too big.
+
+  Note: If the trace file contains output from two `repo`
+        commands that were running at the same time, this
+        will not work precisely.
+  """
+  if os.path.isfile(_TRACE_FILE):
+    while os.path.getsize(_TRACE_FILE)/(1024*1024) > _MAX_SIZE:
+      temp_file = _TRACE_FILE + '.tmp'
+      with open(_TRACE_FILE, 'r', errors='ignore') as fin:
+        with open(temp_file, 'w') as tf:
+          trace_lines = fin.readlines()
+          for i , l in enumerate(trace_lines):
+            if 'END:' in l and _NEW_COMMAND_SEP in l:
+              tf.writelines(trace_lines[i+1:])
+              break
+      platform_utils.rename(temp_file, _TRACE_FILE)

run_tests

@@ -20,6 +20,7 @@ import os
 import shutil
 import subprocess
 import sys
+import repo_trace
 
 
 def find_pytest():

ssh.py

@@ -182,28 +182,29 @@ class ProxyManager:
     # be important because we can't tell that that 'git@myhost.com' is the same
     # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file.
     check_command = command_base + ['-O', 'check']
-    try:
-      Trace(': %s', ' '.join(check_command))
-      check_process = subprocess.Popen(check_command,
-                                       stdout=subprocess.PIPE,
-                                       stderr=subprocess.PIPE)
-      check_process.communicate()  # read output, but ignore it...
-      isnt_running = check_process.wait()
+    with Trace('Call to ssh (check call): %s', ' '.join(check_command)):
+      try:
+        check_process = subprocess.Popen(check_command,
+                                        stdout=subprocess.PIPE,
+                                        stderr=subprocess.PIPE)
+        check_process.communicate()  # read output, but ignore it...
+        isnt_running = check_process.wait()
 
-      if not isnt_running:
-        # Our double-check found that the master _was_ infact running.  Add to
-        # the list of keys.
-        self._master_keys[key] = True
-        return True
-    except Exception:
-      # Ignore excpetions.  We we will fall back to the normal command and print
-      # to the log there.
-      pass
+        if not isnt_running:
+          # Our double-check found that the master _was_ infact running.  Add to
+          # the list of keys.
+          self._master_keys[key] = True
+          return True
+      except Exception:
+        # Ignore excpetions.  We we will fall back to the normal command and
+        # print to the log there.
+        pass
 
     command = command_base[:1] + ['-M', '-N'] + command_base[1:]
+    p = None
     try:
-      Trace(': %s', ' '.join(command))
-      p = subprocess.Popen(command)
+      with Trace('Call to ssh: %s', ' '.join(command)):
+        p = subprocess.Popen(command)
     except Exception as e:
       self._master_broken.value = True
       print('\nwarn: cannot enable ssh control master for %s:%s\n%s'

subcmds/abandon.py

@@ -69,7 +69,8 @@ It is equivalent to "git branch -D <branchname>".
     nb = args[0]
     err = defaultdict(list)
     success = defaultdict(list)
-    all_projects = self.GetProjects(args[1:])
+    all_projects = self.GetProjects(args[1:], all_manifests=not opt.this_manifest_only)
+    _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
 
     def _ProcessResults(_pool, pm, states):
       for (results, project) in states:
@@ -94,7 +95,7 @@ It is equivalent to "git branch -D <branchname>".
         err_msg = "error: cannot abandon %s" % br
         print(err_msg, file=sys.stderr)
         for proj in err[br]:
-          print(' ' * len(err_msg) + " | %s" % proj.relpath, file=sys.stderr)
+          print(' ' * len(err_msg) + " | %s" % _RelPath(proj), file=sys.stderr)
       sys.exit(1)
     elif not success:
       print('error: no project has local branch(es) : %s' % nb,
@@ -110,5 +111,5 @@ It is equivalent to "git branch -D <branchname>".
           result = "all project"
         else:
           result = "%s" % (
-              ('\n' + ' ' * width + '| ').join(p.relpath for p in success[br]))
+              ('\n' + ' ' * width + '| ').join(_RelPath(p) for p in success[br]))
         print("%s%s| %s\n" % (br, ' ' * (width - len(br)), result))

subcmds/branches.py

@@ -98,7 +98,7 @@ is shown, then the branch appears in all projects.
   PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
   def Execute(self, opt, args):
-    projects = self.GetProjects(args)
+    projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
     out = BranchColoring(self.manifest.manifestProject.config)
     all_branches = {}
     project_cnt = len(projects)
@@ -147,26 +147,28 @@ is shown, then the branch appears in all projects.
       hdr('%c%c %-*s' % (current, published, width, name))
       out.write(' |')
 
+      _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
       if in_cnt < project_cnt:
         fmt = out.write
         paths = []
         non_cur_paths = []
-        if i.IsSplitCurrent or (in_cnt < project_cnt - in_cnt):
+        if i.IsSplitCurrent or (in_cnt <= project_cnt - in_cnt):
           in_type = 'in'
           for b in i.projects:
+            relpath = b.project.relpath
             if not i.IsSplitCurrent or b.current:
-              paths.append(b.project.relpath)
+              paths.append(_RelPath(b.project))
             else:
-              non_cur_paths.append(b.project.relpath)
+              non_cur_paths.append(_RelPath(b.project))
         else:
           fmt = out.notinproject
           in_type = 'not in'
           have = set()
           for b in i.projects:
-            have.add(b.project)
+            have.add(_RelPath(b.project))
           for p in projects:
-            if p not in have:
-              paths.append(p.relpath)
+            if _RelPath(p) not in have:
+              paths.append(_RelPath(p))
 
         s = ' %s %s' % (in_type, ', '.join(paths))
         if not i.IsSplitCurrent and (width + 7 + len(s) < 80):

subcmds/checkout.py

@@ -47,7 +47,7 @@ The command is equivalent to:
     nb = args[0]
     err = []
     success = []
-    all_projects = self.GetProjects(args[1:])
+    all_projects = self.GetProjects(args[1:], all_manifests=not opt.this_manifest_only)
 
     def _ProcessResults(_pool, pm, results):
       for status, project in results:

subcmds/cherry_pick.py

@@ -60,8 +60,10 @@ change id will be added.
                    capture_stderr=True)
     status = p.Wait()
 
-    print(p.stdout, file=sys.stdout)
-    print(p.stderr, file=sys.stderr)
+    if p.stdout:
+      print(p.stdout.strip(), file=sys.stdout)
+    if p.stderr:
+      print(p.stderr.strip(), file=sys.stderr)
 
     if status == 0:
       # The cherry-pick was applied correctly. We just need to edit the

subcmds/diff.py

@@ -35,22 +35,25 @@ to the Unix 'patch' command.
                  dest='absolute', action='store_true',
                  help='paths are relative to the repository root')
 
-  def _ExecuteOne(self, absolute, project):
+  def _ExecuteOne(self, absolute, local, project):
     """Obtains the diff for a specific project.
 
     Args:
       absolute: Paths are relative to the root.
+      local: a boolean, if True, the path is relative to the local
+             (sub)manifest.  If false, the path is relative to the
+             outermost manifest.
       project: Project to get status of.
 
     Returns:
       The status of the project.
     """
     buf = io.StringIO()
-    ret = project.PrintWorkTreeDiff(absolute, output_redir=buf)
+    ret = project.PrintWorkTreeDiff(absolute, output_redir=buf, local=local)
     return (ret, buf.getvalue())
 
   def Execute(self, opt, args):
-    all_projects = self.GetProjects(args)
+    all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     def _ProcessResults(_pool, _output, results):
       ret = 0
@@ -63,7 +66,7 @@ to the Unix 'patch' command.
 
     return self.ExecuteInParallel(
         opt.jobs,
-        functools.partial(self._ExecuteOne, opt.absolute),
+        functools.partial(self._ExecuteOne, opt.absolute, opt.this_manifest_only),
         all_projects,
         callback=_ProcessResults,
         ordered=True)

subcmds/diffmanifests.py

@@ -118,6 +118,16 @@ synced and their revisions won't be found.
         self.printRevision(project.revisionExpr)
         self.out.nl()
 
+    if diff['missing']:
+      self.out.nl()
+      self.printText('missing projects : \n')
+      self.out.nl()
+      for project in diff['missing']:
+        self.printProject('\t%s' % (project.relpath))
+        self.printText(' at revision ')
+        self.printRevision(project.revisionExpr)
+        self.out.nl()
+
     if diff['changed']:
       self.out.nl()
       self.printText('changed projects : \n')
@@ -179,6 +189,9 @@ synced and their revisions won't be found.
   def ValidateOptions(self, opt, args):
     if not args or len(args) > 2:
       self.OptionParser.error('missing manifests to diff')
+    if opt.this_manifest_only is False:
+      raise self.OptionParser.error(
+          '`diffmanifest` only supports the current tree')
 
   def Execute(self, opt, args):
     self.out = _Coloring(self.client.globalConfig)

subcmds/download.py

@@ -48,7 +48,7 @@ If no project is specified try to use current directory as a project.
                  dest='ffonly', action='store_true',
                  help="force fast-forward merge")
 
-  def _ParseChangeIds(self, args):
+  def _ParseChangeIds(self, opt, args):
     if not args:
       self.Usage()
 
@@ -77,7 +77,7 @@ If no project is specified try to use current directory as a project.
                 ps_id = max(int(match.group(1)), ps_id)
         to_get.append((project, chg_id, ps_id))
       else:
-        projects = self.GetProjects([a])
+        projects = self.GetProjects([a], all_manifests=not opt.this_manifest_only)
         if len(projects) > 1:
           # If the cwd is one of the projects, assume they want that.
           try:
@@ -88,8 +88,8 @@ If no project is specified try to use current directory as a project.
             print('error: %s matches too many projects; please re-run inside '
                   'the project checkout.' % (a,), file=sys.stderr)
             for project in projects:
-              print('  %s/ @ %s' % (project.relpath, project.revisionExpr),
-                    file=sys.stderr)
+              print('  %s/ @ %s' % (project.RelPath(local=opt.this_manifest_only),
+                                    project.revisionExpr), file=sys.stderr)
             sys.exit(1)
         else:
           project = projects[0]
@@ -105,7 +105,7 @@ If no project is specified try to use current directory as a project.
         self.OptionParser.error('-x and --ff are mutually exclusive options')
 
   def Execute(self, opt, args):
-    for project, change_id, ps_id in self._ParseChangeIds(args):
+    for project, change_id, ps_id in self._ParseChangeIds(opt, args):
       dl = project.DownloadPatchSet(change_id, ps_id)
       if not dl:
         print('[%s] change %d/%d not found'

subcmds/forall.py

@@ -84,6 +84,11 @@ REPO_PROJECT is set to the unique name of the project.
 
 REPO_PATH is the path relative the the root of the client.
 
+REPO_OUTERPATH is the path of the sub manifest's root relative to the root of
+the client.
+
+REPO_INNERPATH is the path relative to the root of the sub manifest.
+
 REPO_REMOTE is the name of the remote system from the manifest.
 
 REPO_LREV is the name of the revision from the manifest, translated
@@ -168,6 +173,7 @@ without iterating through the remaining projects.
 
   def Execute(self, opt, args):
     cmd = [opt.command[0]]
+    all_trees = not opt.this_manifest_only
 
     shell = True
     if re.compile(r'^[a-z0-9A-Z_/\.-]+$').match(cmd[0]):
@@ -213,11 +219,11 @@ without iterating through the remaining projects.
       self.manifest.Override(smart_sync_manifest_path)
 
     if opt.regex:
-      projects = self.FindProjects(args)
+      projects = self.FindProjects(args, all_manifests=all_trees)
     elif opt.inverse_regex:
-      projects = self.FindProjects(args, inverse=True)
+      projects = self.FindProjects(args, inverse=True, all_manifests=all_trees)
     else:
-      projects = self.GetProjects(args, groups=opt.groups)
+      projects = self.GetProjects(args, groups=opt.groups, all_manifests=all_trees)
 
     os.environ['REPO_COUNT'] = str(len(projects))
 
@@ -289,7 +295,9 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
     env[name] = val
 
   setenv('REPO_PROJECT', project.name)
-  setenv('REPO_PATH', project.relpath)
+  setenv('REPO_OUTERPATH', project.manifest.path_prefix)
+  setenv('REPO_INNERPATH', project.relpath)
+  setenv('REPO_PATH', project.RelPath(local=opt.this_manifest_only))
   setenv('REPO_REMOTE', project.remote.name)
   try:
     # If we aren't in a fully synced state and we don't have the ref the manifest
@@ -320,7 +328,7 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
     output = ''
     if ((opt.project_header and opt.verbose)
             or not opt.project_header):
-      output = 'skipping %s/' % project.relpath
+      output = 'skipping %s/' % project.RelPath(local=opt.this_manifest_only)
     return (1, output)
 
   if opt.verbose:
@@ -344,7 +352,7 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
       if mirror:
         project_header_path = project.name
       else:
-        project_header_path = project.relpath
+        project_header_path = project.RelPath(local=opt.this_manifest_only)
       out.project('project %s/' % project_header_path)
       out.nl()
       buf.write(output)

subcmds/gitc_init.py

@@ -24,6 +24,7 @@ import wrapper
 
 class GitcInit(init.Init, GitcAvailableCommand):
   COMMON = True
+  MULTI_MANIFEST_SUPPORT = False
   helpSummary = "Initialize a GITC Client."
   helpUsage = """
 %prog [options] [client name]
@@ -67,7 +68,8 @@ use for this GITC client.
         sys.exit(1)
       manifest_file = opt.manifest_file
 
-    manifest = GitcManifest(self.repodir, gitc_client)
+    manifest = GitcManifest(self.repodir, os.path.join(self.client_dir,
+                                                       '.manifest'))
     manifest.Override(manifest_file)
     gitc_utils.generate_gitc_manifest(None, manifest)
     print('Please run `cd %s` to view your GITC client.' %

subcmds/grep.py

@@ -172,15 +172,16 @@ contain a line that matches both expressions:
     return (project, p.Wait(), p.stdout, p.stderr)
 
   @staticmethod
-  def _ProcessResults(full_name, have_rev, _pool, out, results):
+  def _ProcessResults(full_name, have_rev, opt, _pool, out, results):
     git_failed = False
     bad_rev = False
     have_match = False
+    _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
 
     for project, rc, stdout, stderr in results:
       if rc < 0:
         git_failed = True
-        out.project('--- project %s ---' % project.relpath)
+        out.project('--- project %s ---' % _RelPath(project))
         out.nl()
         out.fail('%s', stderr)
         out.nl()
@@ -192,7 +193,7 @@ contain a line that matches both expressions:
           if have_rev and 'fatal: ambiguous argument' in stderr:
             bad_rev = True
           else:
-            out.project('--- project %s ---' % project.relpath)
+            out.project('--- project %s ---' % _RelPath(project))
             out.nl()
             out.fail('%s', stderr.strip())
             out.nl()
@@ -208,13 +209,13 @@ contain a line that matches both expressions:
           rev, line = line.split(':', 1)
           out.write("%s", rev)
           out.write(':')
-          out.project(project.relpath)
+          out.project(_RelPath(project))
           out.write('/')
           out.write("%s", line)
           out.nl()
       elif full_name:
         for line in r:
-          out.project(project.relpath)
+          out.project(_RelPath(project))
           out.write('/')
           out.write("%s", line)
           out.nl()
@@ -239,7 +240,7 @@ contain a line that matches both expressions:
       cmd_argv.append(args[0])
       args = args[1:]
 
-    projects = self.GetProjects(args)
+    projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     full_name = False
     if len(projects) > 1:
@@ -259,7 +260,7 @@ contain a line that matches both expressions:
         opt.jobs,
         functools.partial(self._ExecuteOne, cmd_argv),
         projects,
-        callback=functools.partial(self._ProcessResults, full_name, have_rev),
+        callback=functools.partial(self._ProcessResults, full_name, have_rev, opt),
         output=out,
         ordered=True)
 

subcmds/info.py

@@ -61,10 +61,11 @@ class Info(PagedCommand):
 
     self.opt = opt
 
+    if not opt.this_manifest_only:
+      self.manifest = self.manifest.outer_client
     manifestConfig = self.manifest.manifestProject.config
     mergeBranch = manifestConfig.GetBranch("default").merge
-    manifestGroups = (manifestConfig.GetString('manifest.groups')
-                      or 'all,-notdefault')
+    manifestGroups = self.manifest.GetGroupsStr()
 
     self.heading("Manifest branch: ")
     if self.manifest.default.revisionExpr:
@@ -80,17 +81,17 @@ class Info(PagedCommand):
     self.printSeparator()
 
     if not opt.overview:
-      self.printDiffInfo(args)
+      self._printDiffInfo(opt, args)
     else:
-      self.printCommitOverview(args)
+      self._printCommitOverview(opt, args)
 
   def printSeparator(self):
     self.text("----------------------------")
     self.out.nl()
 
-  def printDiffInfo(self, args):
+  def _printDiffInfo(self, opt, args):
     # We let exceptions bubble up to main as they'll be well structured.
-    projs = self.GetProjects(args)
+    projs = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     for p in projs:
       self.heading("Project: ")
@@ -179,9 +180,9 @@ class Info(PagedCommand):
       self.text(" ".join(split[1:]))
       self.out.nl()
 
-  def printCommitOverview(self, args):
+  def _printCommitOverview(self, opt, args):
     all_branches = []
-    for project in self.GetProjects(args):
+    for project in self.GetProjects(args, all_manifests=not opt.this_manifest_only):
       br = [project.GetUploadableBranch(x)
             for x in project.GetBranches()]
       br = [x for x in br if x]
@@ -200,7 +201,7 @@ class Info(PagedCommand):
       if project != branch.project:
         project = branch.project
         self.out.nl()
-        self.headtext(project.relpath)
+        self.headtext(project.RelPath(local=opt.this_manifest_only))
         self.out.nl()
 
       commits = branch.commits

subcmds/init.py

@@ -24,14 +24,12 @@ from error import ManifestParseError
 from project import SyncBuffer
 from git_config import GitConfig
 from git_command import git_require, MIN_GIT_VERSION_SOFT, MIN_GIT_VERSION_HARD
-import fetch
-import git_superproject
-import platform_utils
 from wrapper import Wrapper
 
 
 class Init(InteractiveCommand, MirrorSafeCommand):
   COMMON = True
+  MULTI_MANIFEST_SUPPORT = True
   helpSummary = "Initialize a repo client checkout in the current directory"
   helpUsage = """
 %prog [options] [manifest url]
@@ -90,257 +88,55 @@ to update the working directory files.
 
   def _Options(self, p, gitc_init=False):
     Wrapper().InitParser(p, gitc_init=gitc_init)
+    m = p.add_option_group('Multi-manifest')
+    m.add_option('--outer-manifest', action='store_true', default=True,
+                 help='operate starting at the outermost manifest')
+    m.add_option('--no-outer-manifest', dest='outer_manifest',
+                 action='store_false', help='do not operate on outer manifests')
+    m.add_option('--this-manifest-only', action='store_true', default=None,
+                 help='only operate on this (sub)manifest')
+    m.add_option('--no-this-manifest-only', '--all-manifests',
+                 dest='this_manifest_only', action='store_false',
+                 help='operate on this manifest and its submanifests')
 
   def _RegisteredEnvironmentOptions(self):
     return {'REPO_MANIFEST_URL': 'manifest_url',
             'REPO_MIRROR_LOCATION': 'reference'}
 
-  def _CloneSuperproject(self, opt):
-    """Clone the superproject based on the superproject's url and branch.
+  def _SyncManifest(self, opt):
+    """Call manifestProject.Sync with arguments from opt.
 
     Args:
-      opt: Program options returned from optparse.  See _Options().
+      opt: options from optparse.
     """
-    superproject = git_superproject.Superproject(self.manifest,
-                                                 self.repodir,
-                                                 self.git_event_log,
-                                                 quiet=opt.quiet)
-    sync_result = superproject.Sync()
-    if not sync_result.success:
-      print('warning: git update of superproject failed, repo sync will not '
-            'use superproject to fetch source; while this error is not fatal, '
-            'and you can continue to run repo sync, please run repo init with '
-            'the --no-use-superproject option to stop seeing this warning',
-            file=sys.stderr)
-      if sync_result.fatal and opt.use_superproject is not None:
-        sys.exit(1)
-
-  def _SyncManifest(self, opt):
-    m = self.manifest.manifestProject
-    is_new = not m.Exists
-
-    # If repo has already been initialized, we take -u with the absence of
-    # --standalone-manifest to mean "transition to a standard repo set up",
-    # which necessitates starting fresh.
-    # If --standalone-manifest is set, we always tear everything down and start
-    # anew.
-    if not is_new:
-      was_standalone_manifest = m.config.GetString('manifest.standalone')
-      if was_standalone_manifest and not opt.manifest_url:
-        print('fatal: repo was initialized with a standlone manifest, '
-              'cannot be re-initialized without --manifest-url/-u')
-        sys.exit(1)
-
-      if opt.standalone_manifest or (was_standalone_manifest and
-                                     opt.manifest_url):
-        m.config.ClearCache()
-        if m.gitdir and os.path.exists(m.gitdir):
-          platform_utils.rmtree(m.gitdir)
-        if m.worktree and os.path.exists(m.worktree):
-          platform_utils.rmtree(m.worktree)
-
-    is_new = not m.Exists
-    if is_new:
-      if not opt.manifest_url:
-        print('fatal: manifest url is required.', file=sys.stderr)
-        sys.exit(1)
-
-      if not opt.quiet:
-        print('Downloading manifest from %s' %
-              (GitConfig.ForUser().UrlInsteadOf(opt.manifest_url),),
-              file=sys.stderr)
-
-      # The manifest project object doesn't keep track of the path on the
-      # server where this git is located, so let's save that here.
-      mirrored_manifest_git = None
-      if opt.reference:
-        manifest_git_path = urllib.parse.urlparse(opt.manifest_url).path[1:]
-        mirrored_manifest_git = os.path.join(opt.reference, manifest_git_path)
-        if not mirrored_manifest_git.endswith(".git"):
-          mirrored_manifest_git += ".git"
-        if not os.path.exists(mirrored_manifest_git):
-          mirrored_manifest_git = os.path.join(opt.reference,
-                                               '.repo/manifests.git')
-
-      m._InitGitDir(mirror_git=mirrored_manifest_git)
-
-    # If standalone_manifest is set, mark the project as "standalone" -- we'll
-    # still do much of the manifests.git set up, but will avoid actual syncs to
-    # a remote.
-    standalone_manifest = False
-    if opt.standalone_manifest:
-      standalone_manifest = True
-      m.config.SetString('manifest.standalone', opt.manifest_url)
-    elif not opt.manifest_url and not opt.manifest_branch:
-      # If -u is set and --standalone-manifest is not, then we're not in
-      # standalone mode. Otherwise, use config to infer what we were in the last
-      # init.
-      standalone_manifest = bool(m.config.GetString('manifest.standalone'))
-    if not standalone_manifest:
-      m.config.SetString('manifest.standalone', None)
-
-    self._ConfigureDepth(opt)
-
-    # Set the remote URL before the remote branch as we might need it below.
-    if opt.manifest_url:
-      r = m.GetRemote(m.remote.name)
-      r.url = opt.manifest_url
-      r.ResetFetch()
-      r.Save()
-
-    if not standalone_manifest:
-      if opt.manifest_branch:
-        if opt.manifest_branch == 'HEAD':
-          opt.manifest_branch = m.ResolveRemoteHead()
-          if opt.manifest_branch is None:
-            print('fatal: unable to resolve HEAD', file=sys.stderr)
-            sys.exit(1)
-        m.revisionExpr = opt.manifest_branch
-      else:
-        if is_new:
-          default_branch = m.ResolveRemoteHead()
-          if default_branch is None:
-            # If the remote doesn't have HEAD configured, default to master.
-            default_branch = 'refs/heads/master'
-          m.revisionExpr = default_branch
-        else:
-          m.PreSync()
-
-    groups = re.split(r'[,\s]+', opt.groups)
-    all_platforms = ['linux', 'darwin', 'windows']
-    platformize = lambda x: 'platform-' + x
-    if opt.platform == 'auto':
-      if (not opt.mirror and
-              not m.config.GetString('repo.mirror') == 'true'):
-        groups.append(platformize(platform.system().lower()))
-    elif opt.platform == 'all':
-      groups.extend(map(platformize, all_platforms))
-    elif opt.platform in all_platforms:
-      groups.append(platformize(opt.platform))
-    elif opt.platform != 'none':
-      print('fatal: invalid platform flag', file=sys.stderr)
-      sys.exit(1)
-
-    groups = [x for x in groups if x]
-    groupstr = ','.join(groups)
-    if opt.platform == 'auto' and groupstr == self.manifest.GetDefaultGroupsStr():
-      groupstr = None
-    m.config.SetString('manifest.groups', groupstr)
-
-    if opt.reference:
-      m.config.SetString('repo.reference', opt.reference)
-
-    if opt.dissociate:
-      m.config.SetBoolean('repo.dissociate', opt.dissociate)
-
-    if opt.worktree:
-      if opt.mirror:
-        print('fatal: --mirror and --worktree are incompatible',
-              file=sys.stderr)
-        sys.exit(1)
-      if opt.submodules:
-        print('fatal: --submodules and --worktree are incompatible',
-              file=sys.stderr)
-        sys.exit(1)
-      m.config.SetBoolean('repo.worktree', opt.worktree)
-      if is_new:
-        m.use_git_worktrees = True
-      print('warning: --worktree is experimental!', file=sys.stderr)
-
-    if opt.archive:
-      if is_new:
-        m.config.SetBoolean('repo.archive', opt.archive)
-      else:
-        print('fatal: --archive is only supported when initializing a new '
-              'workspace.', file=sys.stderr)
-        print('Either delete the .repo folder in this workspace, or initialize '
-              'in another location.', file=sys.stderr)
-        sys.exit(1)
-
-    if opt.mirror:
-      if is_new:
-        m.config.SetBoolean('repo.mirror', opt.mirror)
-      else:
-        print('fatal: --mirror is only supported when initializing a new '
-              'workspace.', file=sys.stderr)
-        print('Either delete the .repo folder in this workspace, or initialize '
-              'in another location.', file=sys.stderr)
-        sys.exit(1)
-
-    if opt.partial_clone is not None:
-      if opt.mirror:
-        print('fatal: --mirror and --partial-clone are mutually exclusive',
-              file=sys.stderr)
-        sys.exit(1)
-      m.config.SetBoolean('repo.partialclone', opt.partial_clone)
-      if opt.clone_filter:
-        m.config.SetString('repo.clonefilter', opt.clone_filter)
-    elif m.config.GetBoolean('repo.partialclone'):
-      opt.clone_filter = m.config.GetString('repo.clonefilter')
-    else:
-      opt.clone_filter = None
-
-    if opt.partial_clone_exclude is not None:
-      m.config.SetString('repo.partialcloneexclude', opt.partial_clone_exclude)
-
-    if opt.clone_bundle is None:
-      opt.clone_bundle = False if opt.partial_clone else True
-    else:
-      m.config.SetBoolean('repo.clonebundle', opt.clone_bundle)
-
-    if opt.submodules:
-      m.config.SetBoolean('repo.submodules', opt.submodules)
-
-    if opt.use_superproject is not None:
-      m.config.SetBoolean('repo.superproject', opt.use_superproject)
-
-    if standalone_manifest:
-      if is_new:
-        manifest_name = 'default.xml'
-        manifest_data = fetch.fetch_file(opt.manifest_url, verbose=opt.verbose)
-        dest = os.path.join(m.worktree, manifest_name)
-        os.makedirs(os.path.dirname(dest), exist_ok=True)
-        with open(dest, 'wb') as f:
-          f.write(manifest_data)
-      return
-
-    if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet, verbose=opt.verbose,
-                              clone_bundle=opt.clone_bundle,
-                              current_branch_only=opt.current_branch_only,
-                              tags=opt.tags, submodules=opt.submodules,
-                              clone_filter=opt.clone_filter,
-                              partial_clone_exclude=self.manifest.PartialCloneExclude):
-      r = m.GetRemote(m.remote.name)
-      print('fatal: cannot obtain manifest %s' % r.url, file=sys.stderr)
-
-      # Better delete the manifest git dir if we created it; otherwise next
-      # time (when user fixes problems) we won't go through the "is_new" logic.
-      if is_new:
-        platform_utils.rmtree(m.gitdir)
-      sys.exit(1)
-
-    if opt.manifest_branch:
-      m.MetaBranchSwitch(submodules=opt.submodules)
-
-    syncbuf = SyncBuffer(m.config)
-    m.Sync_LocalHalf(syncbuf, submodules=opt.submodules)
-    syncbuf.Finish()
-
-    if is_new or m.CurrentBranch is None:
-      if not m.StartBranch('default'):
-        print('fatal: cannot create default in manifest', file=sys.stderr)
-        sys.exit(1)
-
-  def _LinkManifest(self, name):
-    if not name:
-      print('fatal: manifest name (-m) is required.', file=sys.stderr)
-      sys.exit(1)
-
-    try:
-      self.manifest.Link(name)
-    except ManifestParseError as e:
-      print("fatal: manifest '%s' not available" % name, file=sys.stderr)
-      print('fatal: %s' % str(e), file=sys.stderr)
+    # Normally this value is set when instantiating the project, but the
+    # manifest project is special and is created when instantiating the
+    # manifest which happens before we parse options.
+    self.manifest.manifestProject.clone_depth = opt.manifest_depth
+    if not self.manifest.manifestProject.Sync(
+        manifest_url=opt.manifest_url,
+        manifest_branch=opt.manifest_branch,
+        standalone_manifest=opt.standalone_manifest,
+        groups=opt.groups,
+        platform=opt.platform,
+        mirror=opt.mirror,
+        dissociate=opt.dissociate,
+        reference=opt.reference,
+        worktree=opt.worktree,
+        submodules=opt.submodules,
+        archive=opt.archive,
+        partial_clone=opt.partial_clone,
+        clone_filter=opt.clone_filter,
+        partial_clone_exclude=opt.partial_clone_exclude,
+        clone_bundle=opt.clone_bundle,
+        git_lfs=opt.git_lfs,
+        use_superproject=opt.use_superproject,
+        verbose=opt.verbose,
+        current_branch_only=opt.current_branch_only,
+        tags=opt.tags,
+        depth=opt.depth,
+        git_event_log=self.git_event_log,
+        manifest_name=opt.manifest_name):
       sys.exit(1)
 
   def _Prompt(self, prompt, value):
@@ -352,7 +148,7 @@ to update the working directory files.
       return value
     return a
 
-  def _ShouldConfigureUser(self, opt):
+  def _ShouldConfigureUser(self, opt, existing_checkout):
     gc = self.client.globalConfig
     mp = self.manifest.manifestProject
 
@@ -364,7 +160,7 @@ to update the working directory files.
       mp.config.SetString('user.name', gc.GetString('user.name'))
       mp.config.SetString('user.email', gc.GetString('user.email'))
 
-    if not opt.quiet:
+    if not opt.quiet and not existing_checkout or opt.verbose:
       print()
       print('Your identity is: %s <%s>' % (mp.config.GetString('user.name'),
                                            mp.config.GetString('user.email')))
@@ -434,25 +230,6 @@ to update the working directory files.
     if a in ('y', 'yes', 't', 'true', 'on'):
       gc.SetString('color.ui', 'auto')
 
-  def _ConfigureDepth(self, opt):
-    """Configure the depth we'll sync down.
-
-    Args:
-      opt: Options from optparse.  We care about opt.depth.
-    """
-    # Opt.depth will be non-None if user actually passed --depth to repo init.
-    if opt.depth is not None:
-      if opt.depth > 0:
-        # Positive values will set the depth.
-        depth = str(opt.depth)
-      else:
-        # Negative numbers will clear the depth; passing None to SetString
-        # will do that.
-        depth = None
-
-      # We store the depth in the main manifest project.
-      self.manifest.manifestProject.config.SetString('repo.depth', depth)
-
   def _DisplayResult(self, opt):
     if self.manifest.IsMirror:
       init_type = 'mirror '
@@ -468,7 +245,7 @@ to update the working directory files.
     if current_dir != self.manifest.topdir:
       print('If this is not the directory in which you want to initialize '
             'repo, please run:')
-      print('   rm -r %s/.repo' % self.manifest.topdir)
+      print('   rm -r %s' % os.path.join(self.manifest.topdir, '.repo'))
       print('and try again.')
 
   def ValidateOptions(self, opt, args):
@@ -484,6 +261,9 @@ to update the working directory files.
       if opt.use_superproject is not None:
         self.OptionParser.error('--mirror and --use-superproject cannot be '
                                 'used together.')
+    if opt.archive and opt.use_superproject is not None:
+      self.OptionParser.error('--archive and --use-superproject cannot be used '
+                              'together.')
 
     if opt.standalone_manifest and (opt.manifest_branch or
                                     opt.manifest_name != 'default.xml'):
@@ -520,8 +300,12 @@ to update the working directory files.
     # Handle new --repo-rev requests.
     if opt.repo_rev:
       wrapper = Wrapper()
-      remote_ref, rev = wrapper.check_repo_rev(
-          rp.gitdir, opt.repo_rev, repo_verify=opt.repo_verify, quiet=opt.quiet)
+      try:
+        remote_ref, rev = wrapper.check_repo_rev(
+            rp.gitdir, opt.repo_rev, repo_verify=opt.repo_verify, quiet=opt.quiet)
+      except wrapper.CloneFailure:
+        print('fatal: double check your --repo-rev setting.', file=sys.stderr)
+        sys.exit(1)
       branch = rp.GetBranch('default')
       branch.merge = remote_ref
       rp.work_git.reset('--hard', rev)
@@ -531,14 +315,17 @@ to update the working directory files.
       # Older versions of git supported worktree, but had dangerous gc bugs.
       git_require((2, 15, 0), fail=True, msg='git gc worktree corruption')
 
-    self._SyncManifest(opt)
-    self._LinkManifest(opt.manifest_name)
+    # Provide a short notice that we're reinitializing an existing checkout.
+    # Sometimes developers might not realize that they're in one, or that
+    # repo doesn't do nested checkouts.
+    existing_checkout = self.manifest.manifestProject.Exists
+    if not opt.quiet and existing_checkout:
+      print('repo: reusing existing repo client checkout in', self.manifest.topdir)
 
-    if self.manifest.manifestProject.config.GetBoolean('repo.superproject'):
-      self._CloneSuperproject(opt)
+    self._SyncManifest(opt)
 
     if os.isatty(0) and os.isatty(1) and not self.manifest.IsMirror:
-      if opt.config_name or self._ShouldConfigureUser(opt):
+      if opt.config_name or self._ShouldConfigureUser(opt, existing_checkout):
         self._ConfigureUser(opt)
       self._ConfigureColor()
 

subcmds/list.py

@@ -77,16 +77,17 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
       args: Positional args.  Can be a list of projects to list, or empty.
     """
     if not opt.regex:
-      projects = self.GetProjects(args, groups=opt.groups, missing_ok=opt.all)
+      projects = self.GetProjects(args, groups=opt.groups, missing_ok=opt.all,
+                                  all_manifests=not opt.this_manifest_only)
     else:
-      projects = self.FindProjects(args)
+      projects = self.FindProjects(args, all_manifests=not opt.this_manifest_only)
 
     def _getpath(x):
       if opt.fullpath:
         return x.worktree
       if opt.relative_to:
         return os.path.relpath(x.worktree, opt.relative_to)
-      return x.relpath
+      return x.RelPath(local=opt.this_manifest_only)
 
     lines = []
     for project in projects:

subcmds/manifest.py

@@ -15,6 +15,7 @@
 import json
 import os
 import sys
+import optparse
 
 from command import PagedCommand
 
@@ -75,7 +76,7 @@ to indicate the remote ref to push changes to via 'repo upload'.
     p.add_option('-o', '--output-file',
                  dest='output_file',
                  default='-',
-                 help='file to save the manifest to',
+                 help='file to save the manifest to. (Filename prefix for multi-tree.)',
                  metavar='-|NAME.xml')
 
   def _Output(self, opt):
@@ -83,36 +84,45 @@ to indicate the remote ref to push changes to via 'repo upload'.
     if opt.manifest_name:
       self.manifest.Override(opt.manifest_name, False)
 
-    if opt.output_file == '-':
-      fd = sys.stdout
-    else:
-      fd = open(opt.output_file, 'w')
+    for manifest in self.ManifestList(opt):
+      output_file = opt.output_file
+      if output_file == '-':
+        fd = sys.stdout
+      else:
+        if manifest.path_prefix:
+          output_file = f'{opt.output_file}:{manifest.path_prefix.replace("/", "%2f")}'
+        fd = open(output_file, 'w')
 
-    self.manifest.SetUseLocalManifests(not opt.ignore_local_manifests)
+      manifest.SetUseLocalManifests(not opt.ignore_local_manifests)
 
-    if opt.json:
-      print('warning: --json is experimental!', file=sys.stderr)
-      doc = self.manifest.ToDict(peg_rev=opt.peg_rev,
-                                 peg_rev_upstream=opt.peg_rev_upstream,
-                                 peg_rev_dest_branch=opt.peg_rev_dest_branch)
+      if opt.json:
+        print('warning: --json is experimental!', file=sys.stderr)
+        doc = manifest.ToDict(peg_rev=opt.peg_rev,
+                                   peg_rev_upstream=opt.peg_rev_upstream,
+                                   peg_rev_dest_branch=opt.peg_rev_dest_branch)
+
+        json_settings = {
+            # JSON style guide says Uunicode characters are fully allowed.
+            'ensure_ascii': False,
+            # We use 2 space indent to match JSON style guide.
+            'indent': 2 if opt.pretty else None,
+            'separators': (',', ': ') if opt.pretty else (',', ':'),
+            'sort_keys': True,
+        }
+        fd.write(json.dumps(doc, **json_settings))
+      else:
+        manifest.Save(fd,
+                      peg_rev=opt.peg_rev,
+                      peg_rev_upstream=opt.peg_rev_upstream,
+                      peg_rev_dest_branch=opt.peg_rev_dest_branch)
+      if output_file != '-':
+        fd.close()
+        if manifest.path_prefix:
+          print(f'Saved {manifest.path_prefix} submanifest to {output_file}',
+                file=sys.stderr)
+        else:
+          print(f'Saved manifest to {output_file}', file=sys.stderr)
 
-      json_settings = {
-          # JSON style guide says Uunicode characters are fully allowed.
-          'ensure_ascii': False,
-          # We use 2 space indent to match JSON style guide.
-          'indent': 2 if opt.pretty else None,
-          'separators': (',', ': ') if opt.pretty else (',', ':'),
-          'sort_keys': True,
-      }
-      fd.write(json.dumps(doc, **json_settings))
-    else:
-      self.manifest.Save(fd,
-                         peg_rev=opt.peg_rev,
-                         peg_rev_upstream=opt.peg_rev_upstream,
-                         peg_rev_dest_branch=opt.peg_rev_dest_branch)
-    fd.close()
-    if opt.output_file != '-':
-      print('Saved manifest to %s' % opt.output_file, file=sys.stderr)
 
   def ValidateOptions(self, opt, args):
     if args:

subcmds/overview.py

@@ -47,7 +47,7 @@ are displayed.
 
   def Execute(self, opt, args):
     all_branches = []
-    for project in self.GetProjects(args):
+    for project in self.GetProjects(args, all_manifests=not opt.this_manifest_only):
       br = [project.GetUploadableBranch(x)
             for x in project.GetBranches()]
       br = [x for x in br if x]
@@ -76,7 +76,7 @@ are displayed.
       if project != branch.project:
         project = branch.project
         out.nl()
-        out.project('project %s/' % project.relpath)
+        out.project('project %s/' % project.RelPath(local=opt.this_manifest_only))
         out.nl()
 
       commits = branch.commits

subcmds/prune.py

@@ -31,7 +31,7 @@ class Prune(PagedCommand):
     return project.PruneHeads()
 
   def Execute(self, opt, args):
-    projects = self.GetProjects(args)
+    projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     # NB: Should be able to refactor this module to display summary as results
     # come back from children.
@@ -63,7 +63,7 @@ class Prune(PagedCommand):
       if project != branch.project:
         project = branch.project
         out.nl()
-        out.project('project %s/' % project.relpath)
+        out.project('project %s/' % project.RelPath(local=opt.this_manifest_only))
         out.nl()
 
       print('%s %-33s ' % (

subcmds/rebase.py

@@ -69,7 +69,7 @@ branch but need to incorporate new upstream changes "underneath" them.
                       'consistent if you previously synced to a manifest)')
 
   def Execute(self, opt, args):
-    all_projects = self.GetProjects(args)
+    all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
     one_project = len(all_projects) == 1
 
     if opt.interactive and not one_project:
@@ -98,6 +98,7 @@ branch but need to incorporate new upstream changes "underneath" them.
     config = self.manifest.manifestProject.config
     out = RebaseColoring(config)
     out.redirect(sys.stdout)
+    _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
 
     ret = 0
     for project in all_projects:
@@ -107,7 +108,7 @@ branch but need to incorporate new upstream changes "underneath" them.
       cb = project.CurrentBranch
       if not cb:
         if one_project:
-          print("error: project %s has a detached HEAD" % project.relpath,
+          print("error: project %s has a detached HEAD" % _RelPath(project),
                 file=sys.stderr)
           return 1
         # ignore branches with detatched HEADs
@@ -117,7 +118,7 @@ branch but need to incorporate new upstream changes "underneath" them.
       if not upbranch.LocalMerge:
         if one_project:
           print("error: project %s does not track any remote branches"
-                % project.relpath, file=sys.stderr)
+                % _RelPath(project), file=sys.stderr)
           return 1
         # ignore branches without remotes
         continue
@@ -130,7 +131,7 @@ branch but need to incorporate new upstream changes "underneath" them.
       args.append(upbranch.LocalMerge)
 
       out.project('project %s: rebasing %s -> %s',
-                  project.relpath, cb, upbranch.LocalMerge)
+                  _RelPath(project), cb, upbranch.LocalMerge)
       out.nl()
       out.flush()
 

subcmds/selfupdate.py

@@ -51,7 +51,7 @@ need to be performed by an end-user.
       _PostRepoUpgrade(self.manifest)
 
     else:
-      if not rp.Sync_NetworkHalf():
+      if not rp.Sync_NetworkHalf().success:
         print("error: can't update repo", file=sys.stderr)
         sys.exit(1)
 

subcmds/stage.py

@@ -50,7 +50,9 @@ The '%prog' command stages files to prepare the next commit.
       self.Usage()
 
   def _Interactive(self, opt, args):
-    all_projects = [p for p in self.GetProjects(args) if p.IsDirty()]
+    all_projects = [
+        p for p in self.GetProjects(args, all_manifests=not opt.this_manifest_only)
+        if p.IsDirty()]
     if not all_projects:
       print('no projects have uncommitted modifications', file=sys.stderr)
       return
@@ -62,7 +64,8 @@ The '%prog' command stages files to prepare the next commit.
 
       for i in range(len(all_projects)):
         project = all_projects[i]
-        out.write('%3d:    %s', i + 1, project.relpath + '/')
+        out.write('%3d:    %s', i + 1,
+                  project.RelPath(local=opt.this_manifest_only) + '/')
         out.nl()
       out.nl()
 
@@ -72,6 +75,7 @@ The '%prog' command stages files to prepare the next commit.
       out.nl()
 
       out.prompt('project> ')
+      out.flush()
       try:
         a = sys.stdin.readline()
       except KeyboardInterrupt:
@@ -99,7 +103,9 @@ The '%prog' command stages files to prepare the next commit.
           _AddI(all_projects[a_index - 1])
           continue
 
-      projects = [p for p in all_projects if a in [p.name, p.relpath]]
+      projects = [
+          p for p in all_projects
+          if a in [p.name, p.RelPath(local=opt.this_manifest_only)]]
       if len(projects) == 1:
         _AddI(projects[0])
         continue

subcmds/start.py

@@ -84,7 +84,8 @@ revision specified in the manifest.
         projects = ['.']  # start it in the local project by default
 
     all_projects = self.GetProjects(projects,
-                                    missing_ok=bool(self.gitc_manifest))
+                                    missing_ok=bool(self.gitc_manifest),
+                                    all_manifests=not opt.this_manifest_only)
 
     # This must happen after we find all_projects, since GetProjects may need
     # the local directory, which will disappear once we save the GITC manifest.
@@ -137,6 +138,6 @@ revision specified in the manifest.
 
     if err:
       for p in err:
-        print("error: %s/: cannot start %s" % (p.relpath, nb),
+        print("error: %s/: cannot start %s" % (p.RelPath(local=opt.this_manifest_only), nb),
               file=sys.stderr)
       sys.exit(1)

subcmds/status.py

@@ -83,7 +83,7 @@ the following meanings:
                  dest='orphans', action='store_true',
                  help="include objects in working directory outside of repo projects")
 
-  def _StatusHelper(self, quiet, project):
+  def _StatusHelper(self, quiet, local, project):
     """Obtains the status for a specific project.
 
     Obtains the status for a project, redirecting the output to
@@ -91,13 +91,17 @@ the following meanings:
 
     Args:
       quiet: Where to output the status.
+      local: a boolean, if True, the path is relative to the local
+             (sub)manifest.  If false, the path is relative to the
+             outermost manifest.
       project: Project to get status of.
 
     Returns:
       The status of the project.
     """
     buf = io.StringIO()
-    ret = project.PrintWorkTreeStatus(quiet=quiet, output_redir=buf)
+    ret = project.PrintWorkTreeStatus(quiet=quiet, output_redir=buf,
+                                      local=local)
     return (ret, buf.getvalue())
 
   def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring):
@@ -117,7 +121,7 @@ the following meanings:
       outstring.append(''.join([status_header, item, '/']))
 
   def Execute(self, opt, args):
-    all_projects = self.GetProjects(args)
+    all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     def _ProcessResults(_pool, _output, results):
       ret = 0
@@ -130,7 +134,7 @@ the following meanings:
 
     counter = self.ExecuteInParallel(
         opt.jobs,
-        functools.partial(self._StatusHelper, opt.quiet),
+        functools.partial(self._StatusHelper, opt.quiet, opt.this_manifest_only),
         all_projects,
         callback=_ProcessResults,
         ordered=True)
@@ -141,9 +145,10 @@ the following meanings:
     if opt.orphans:
       proj_dirs = set()
       proj_dirs_parents = set()
-      for project in self.GetProjects(None, missing_ok=True):
-        proj_dirs.add(project.relpath)
-        (head, _tail) = os.path.split(project.relpath)
+      for project in self.GetProjects(None, missing_ok=True, all_manifests=not opt.this_manifest_only):
+        relpath = project.RelPath(local=opt.this_manifest_only)
+        proj_dirs.add(relpath)
+        (head, _tail) = os.path.split(relpath)
         while head != "":
           proj_dirs_parents.add(head)
           (head, _tail) = os.path.split(head)

subcmds/upload.py

@@ -78,6 +78,13 @@ added to the respective list of users, and emails are sent to any
 new users.  Users passed as --reviewers must already be registered
 with the code review system, or the upload will fail.
 
+While most normal Gerrit options have dedicated command line options,
+direct access to the Gerit options is available via --push-options.
+This is useful when Gerrit has newer functionality that %prog doesn't
+yet support, or doesn't have plans to support.  See the Push Options
+documentation for more details:
+https://gerrit-review.googlesource.com/Documentation/user-upload.html#push_options
+
 # Configuration
 
 review.URL.autoupload:
@@ -190,6 +197,9 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     p.add_option('-w', '--wip',
                  action='store_true', dest='wip', default=False,
                  help='upload as a work-in-progress change')
+    p.add_option('-r', '--ready',
+                 action='store_true', default=False,
+                 help='mark change as ready (clears work-in-progress setting)')
     p.add_option('-o', '--push-option',
                  type='string', action='append', dest='push_options',
                  default=[],
@@ -204,6 +214,12 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     p.add_option('-y', '--yes',
                  default=False, action='store_true',
                  help='answer yes to all safe prompts')
+    p.add_option('--ignore-untracked-files',
+                 action='store_true', default=False,
+                 help='ignore untracked files in the working copy')
+    p.add_option('--no-ignore-untracked-files',
+                 dest='ignore_untracked_files', action='store_false',
+                 help='always ask about untracked files in the working copy')
     p.add_option('--no-cert-checks',
                  dest='validate_certs', action='store_false', default=True,
                  help='disable verifying ssl certs (unsafe)')
@@ -226,7 +242,8 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
 
       destination = opt.dest_branch or project.dest_branch or project.revisionExpr
       print('Upload project %s/ to remote branch %s%s:' %
-            (project.relpath, destination, ' (private)' if opt.private else ''))
+            (project.RelPath(local=opt.this_manifest_only), destination,
+             ' (private)' if opt.private else ''))
       print('  branch %s (%2d commit%s, %s):' % (
           name,
           len(commit_list),
@@ -245,7 +262,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
         answer = sys.stdin.readline().strip().lower()
         answer = answer in ('y', 'yes', '1', 'true', 't')
 
-    if answer:
+    if not opt.yes and answer:
       if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD:
         answer = _ConfirmManyUploads()
 
@@ -262,7 +279,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     script.append('# Uncomment the branches to upload:')
     for project, avail in pending:
       script.append('#')
-      script.append('# project %s/:' % project.relpath)
+      script.append('# project %s/:' % project.RelPath(local=opt.this_manifest_only))
 
       b = {}
       for branch in avail:
@@ -285,7 +302,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
           script.append('#         %s' % commit)
         b[name] = branch
 
-      projects[project.relpath] = project
+      projects[project.RelPath(local=opt.this_manifest_only)] = project
       branches[project.name] = b
     script.append('')
 
@@ -313,19 +330,20 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
           _die('project for branch %s not in script', name)
         branch = branches[project.name].get(name)
         if not branch:
-          _die('branch %s not in %s', name, project.relpath)
+          _die('branch %s not in %s', name, project.RelPath(local=opt.this_manifest_only))
         todo.append(branch)
     if not todo:
       _die("nothing uncommented for upload")
 
-    many_commits = False
-    for branch in todo:
-      if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD:
-        many_commits = True
-        break
-    if many_commits:
-      if not _ConfirmManyUploads(multiple_branches=True):
-        _die("upload aborted by user")
+    if not opt.yes:
+      many_commits = False
+      for branch in todo:
+        if len(branch.commits) > UNUSUAL_COMMIT_THRESHOLD:
+          many_commits = True
+          break
+      if many_commits:
+        if not _ConfirmManyUploads(multiple_branches=True):
+          _die("upload aborted by user")
 
     self._UploadAndReport(opt, todo, people)
 
@@ -369,6 +387,10 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
 
         # Check if there are local changes that may have been forgotten
         changes = branch.project.UncommitedFiles()
+        if opt.ignore_untracked_files:
+          untracked = set(branch.project.UntrackedFiles())
+          changes = [x for x in changes if x not in untracked]
+
         if changes:
           key = 'review.%s.autoupload' % branch.project.remote.review
           answer = branch.project.config.GetBoolean(key)
@@ -420,12 +442,6 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
         labels = set(_ExpandCommaList(branch.project.config.GetString(key)))
         for label in opt.labels:
           labels.update(_ExpandCommaList(label))
-        # Basic sanity check on label syntax.
-        for label in labels:
-          if not re.match(r'^.+[+-][0-9]+$', label):
-            print('repo: error: invalid label syntax "%s": labels use forms '
-                  'like CodeReview+1 or Verified-1' % (label,), file=sys.stderr)
-            sys.exit(1)
 
         # Handle e-mail notifications.
         if opt.notify is False:
@@ -460,6 +476,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
                                private=opt.private,
                                notify=notify,
                                wip=opt.wip,
+                               ready=opt.ready,
                                dest_branch=destination,
                                validate_certs=opt.validate_certs,
                                push_options=opt.push_options)
@@ -481,7 +498,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
           else:
             fmt = '\n       (%s)'
           print(('[FAILED] %-15s %-15s' + fmt) % (
-              branch.project.relpath + '/',
+              branch.project.RelPath(local=opt.this_manifest_only) + '/',
               branch.name,
               str(branch.error)),
               file=sys.stderr)
@@ -490,7 +507,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     for branch in todo:
       if branch.uploaded:
         print('[OK    ] %-15s %s' % (
-            branch.project.relpath + '/',
+            branch.project.RelPath(local=opt.this_manifest_only) + '/',
             branch.name),
             file=sys.stderr)
 
@@ -524,7 +541,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     return (project, avail)
 
   def Execute(self, opt, args):
-    projects = self.GetProjects(args)
+    projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only)
 
     def _ProcessResults(_pool, _out, results):
       pending = []
@@ -534,7 +551,8 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
           print('repo: error: %s: Unable to upload branch "%s". '
                 'You might be able to fix the branch by running:\n'
                 '  git branch --set-upstream-to m/%s' %
-                (project.relpath, project.CurrentBranch, self.manifest.branch),
+                (project.RelPath(local=opt.this_manifest_only), project.CurrentBranch,
+                 project.manifest.branch),
                 file=sys.stderr)
         elif avail:
           pending.append(result)
@@ -554,15 +572,23 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
               (opt.branch,), file=sys.stderr)
       return 1
 
-    pending_proj_names = [project.name for (project, available) in pending]
-    pending_worktrees = [project.worktree for (project, available) in pending]
-    hook = RepoHook.FromSubcmd(
-        hook_type='pre-upload', manifest=self.manifest,
-        opt=opt, abort_if_user_denies=True)
-    if not hook.Run(
-        project_list=pending_proj_names,
-        worktree_list=pending_worktrees):
-      return 1
+    manifests = {project.manifest.topdir: project.manifest
+                 for (project, available) in pending}
+    ret = 0
+    for manifest in manifests.values():
+      pending_proj_names = [project.name for (project, available) in pending
+                            if project.manifest.topdir == manifest.topdir]
+      pending_worktrees = [project.worktree for (project, available) in pending
+                           if project.manifest.topdir == manifest.topdir]
+      hook = RepoHook.FromSubcmd(
+          hook_type='pre-upload', manifest=manifest,
+          opt=opt, abort_if_user_denies=True)
+      if not hook.Run(
+          project_list=pending_proj_names,
+          worktree_list=pending_worktrees):
+        ret = 1
+    if ret:
+      return ret
 
     reviewers = _SplitEmails(opt.reviewers) if opt.reviewers else []
     cc = _SplitEmails(opt.cc) if opt.cc else []

subcmds/version.py

@@ -33,7 +33,7 @@ class Version(Command, MirrorSafeCommand):
 
   def Execute(self, opt, args):
     rp = self.manifest.repoProject
-    rem = rp.GetRemote(rp.remote.name)
+    rem = rp.GetRemote()
     branch = rp.GetBranch('default')
 
     # These might not be the same.  Report them both.

tests/test_git_config.py

@@ -19,6 +19,7 @@ import tempfile
 import unittest
 
 import git_config
+import repo_trace
 
 
 def fixture(*paths):
@@ -33,9 +34,16 @@ class GitConfigReadOnlyTests(unittest.TestCase):
   def setUp(self):
     """Create a GitConfig object using the test.gitconfig fixture.
     """
+
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    repo_trace._TRACE_FILE = os.path.join(self.tempdirobj.name, 'TRACE_FILE_from_test')
+
     config_fixture = fixture('test.gitconfig')
     self.config = git_config.GitConfig(config_fixture)
 
+  def tearDown(self):
+    self.tempdirobj.cleanup()
+
   def test_GetString_with_empty_config_values(self):
     """
     Test config entries with no value.
@@ -109,9 +117,15 @@ class GitConfigReadWriteTests(unittest.TestCase):
   """Read/write tests of the GitConfig class."""
 
   def setUp(self):
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    repo_trace._TRACE_FILE = os.path.join(self.tempdirobj.name, 'TRACE_FILE_from_test')
+
     self.tmpfile = tempfile.NamedTemporaryFile()
     self.config = self.get_config()
 
+  def tearDown(self):
+    self.tempdirobj.cleanup()
+
   def get_config(self):
     """Get a new GitConfig instance."""
     return git_config.GitConfig(self.tmpfile.name)

tests/test_git_superproject.py

@@ -24,7 +24,7 @@ from unittest import mock
 import git_superproject
 import git_trace2_event_log
 import manifest_xml
-import platform_utils
+import repo_trace
 from test_manifest_xml import sort_attributes
 
 
@@ -38,7 +38,9 @@ class SuperprojectTestCase(unittest.TestCase):
 
   def setUp(self):
     """Set up superproject every time."""
-    self.tempdir = tempfile.mkdtemp(prefix='repo_tests')
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    self.tempdir = self.tempdirobj.name
+    repo_trace._TRACE_FILE = os.path.join(self.tempdir, 'TRACE_FILE_from_test')
     self.repodir = os.path.join(self.tempdir, '.repo')
     self.manifest_file = os.path.join(
         self.repodir, manifest_xml.MANIFEST_FILE_NAME)
@@ -68,12 +70,14 @@ class SuperprojectTestCase(unittest.TestCase):
   <project path="art" name="platform/art" groups="notdefault,platform-""" + self.platform + """
   " /></manifest>
 """)
-    self._superproject = git_superproject.Superproject(manifest, self.repodir,
-                                                       self.git_event_log)
+    self._superproject = git_superproject.Superproject(
+        manifest, name='superproject',
+        remote=manifest.remotes.get('default-remote').ToRemoteSpec('superproject'),
+        revision='refs/heads/main')
 
   def tearDown(self):
     """Tear down superproject every time."""
-    platform_utils.rmtree(self.tempdir)
+    self.tempdirobj.cleanup()
 
   def getXmlManifest(self, data):
     """Helper to initialize a manifest for testing."""
@@ -125,12 +129,7 @@ class SuperprojectTestCase(unittest.TestCase):
 <manifest>
 </manifest>
 """)
-    superproject = git_superproject.Superproject(manifest, self.repodir, self.git_event_log)
-    # Test that exit condition is false when there is no superproject tag.
-    sync_result = superproject.Sync()
-    self.assertFalse(sync_result.success)
-    self.assertFalse(sync_result.fatal)
-    self.verifyErrorEvent()
+    self.assertIsNone(manifest.superproject)
 
   def test_superproject_get_superproject_invalid_url(self):
     """Test with an invalid url."""
@@ -141,8 +140,11 @@ class SuperprojectTestCase(unittest.TestCase):
   <superproject name="superproject"/>
 </manifest>
 """)
-    superproject = git_superproject.Superproject(manifest, self.repodir, self.git_event_log)
-    sync_result = superproject.Sync()
+    superproject = git_superproject.Superproject(
+        manifest, name='superproject',
+        remote=manifest.remotes.get('test-remote').ToRemoteSpec('superproject'),
+        revision='refs/heads/main')
+    sync_result = superproject.Sync(self.git_event_log)
     self.assertFalse(sync_result.success)
     self.assertTrue(sync_result.fatal)
 
@@ -155,17 +157,19 @@ class SuperprojectTestCase(unittest.TestCase):
   <superproject name="superproject"/>
 </manifest>
 """)
-    self._superproject = git_superproject.Superproject(manifest, self.repodir,
-                                                       self.git_event_log)
+    self._superproject = git_superproject.Superproject(
+        manifest, name='superproject',
+        remote=manifest.remotes.get('test-remote').ToRemoteSpec('superproject'),
+        revision='refs/heads/main')
     with mock.patch.object(self._superproject, '_branch', 'junk'):
-      sync_result = self._superproject.Sync()
+      sync_result = self._superproject.Sync(self.git_event_log)
       self.assertFalse(sync_result.success)
       self.assertTrue(sync_result.fatal)
 
   def test_superproject_get_superproject_mock_init(self):
     """Test with _Init failing."""
     with mock.patch.object(self._superproject, '_Init', return_value=False):
-      sync_result = self._superproject.Sync()
+      sync_result = self._superproject.Sync(self.git_event_log)
       self.assertFalse(sync_result.success)
       self.assertTrue(sync_result.fatal)
 
@@ -174,7 +178,7 @@ class SuperprojectTestCase(unittest.TestCase):
     with mock.patch.object(self._superproject, '_Init', return_value=True):
       os.mkdir(self._superproject._superproject_path)
       with mock.patch.object(self._superproject, '_Fetch', return_value=False):
-        sync_result = self._superproject.Sync()
+        sync_result = self._superproject.Sync(self.git_event_log)
         self.assertFalse(sync_result.success)
         self.assertTrue(sync_result.fatal)
 
@@ -230,7 +234,7 @@ class SuperprojectTestCase(unittest.TestCase):
                                return_value=data):
           # Create temporary directory so that it can write the file.
           os.mkdir(self._superproject._superproject_path)
-          update_result = self._superproject.UpdateProjectsRevisionId(projects)
+          update_result = self._superproject.UpdateProjectsRevisionId(projects, self.git_event_log)
           self.assertIsNotNone(update_result.manifest_path)
           self.assertFalse(update_result.fatal)
           with open(update_result.manifest_path, 'r') as fp:
@@ -256,22 +260,13 @@ class SuperprojectTestCase(unittest.TestCase):
 </manifest>
 """)
     self.maxDiff = None
-    self._superproject = git_superproject.Superproject(manifest, self.repodir,
-                                                       self.git_event_log)
-    self.assertEqual(len(self._superproject._manifest.projects), 1)
-    projects = self._superproject._manifest.projects
-    project = projects[0]
-    project.SetRevisionId('ABCDEF')
-    update_result = self._superproject.UpdateProjectsRevisionId(projects)
-    self.assertIsNone(update_result.manifest_path)
-    self.assertFalse(update_result.fatal)
-    self.verifyErrorEvent()
+    self.assertIsNone(manifest.superproject)
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
         '<remote fetch="http://localhost" name="default-remote"/>'
         '<default remote="default-remote" revision="refs/heads/main"/>'
-        '<project name="test-name" revision="ABCDEF" upstream="refs/heads/main"/>'
+        '<project name="test-name"/>'
         '</manifest>')
 
   def test_superproject_update_project_revision_id_from_local_manifest_group(self):
@@ -290,8 +285,10 @@ class SuperprojectTestCase(unittest.TestCase):
   " /></manifest>
 """)
     self.maxDiff = None
-    self._superproject = git_superproject.Superproject(manifest, self.repodir,
-                                                       self.git_event_log)
+    self._superproject = git_superproject.Superproject(
+        manifest, name='superproject',
+        remote=manifest.remotes.get('default-remote').ToRemoteSpec('superproject'),
+        revision='refs/heads/main')
     self.assertEqual(len(self._superproject._manifest.projects), 2)
     projects = self._superproject._manifest.projects
     data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00')
@@ -302,7 +299,7 @@ class SuperprojectTestCase(unittest.TestCase):
                                return_value=data):
           # Create temporary directory so that it can write the file.
           os.mkdir(self._superproject._superproject_path)
-          update_result = self._superproject.UpdateProjectsRevisionId(projects)
+          update_result = self._superproject.UpdateProjectsRevisionId(projects, self.git_event_log)
           self.assertIsNotNone(update_result.manifest_path)
           self.assertFalse(update_result.fatal)
           with open(update_result.manifest_path, 'r') as fp:
@@ -317,9 +314,6 @@ class SuperprojectTestCase(unittest.TestCase):
               '<project groups="notdefault,platform-' + self.platform + '" '
               'name="platform/art" path="art" '
               'revision="2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea" upstream="refs/heads/main"/>'
-              '<project clone-depth="1" groups="' + local_group + '" '
-              'name="platform/vendor/x" path="vendor/x" remote="goog" '
-              'revision="master-with-vendor"/>'
               '<superproject name="superproject"/>'
               '</manifest>')
 
@@ -337,8 +331,10 @@ class SuperprojectTestCase(unittest.TestCase):
   " /></manifest>
 """)
     self.maxDiff = None
-    self._superproject = git_superproject.Superproject(manifest, self.repodir,
-                                                       self.git_event_log)
+    self._superproject = git_superproject.Superproject(
+        manifest, name='superproject',
+        remote=manifest.remotes.get('default-remote').ToRemoteSpec('superproject'),
+        revision='refs/heads/main')
     self.assertEqual(len(self._superproject._manifest.projects), 3)
     projects = self._superproject._manifest.projects
     data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00'
@@ -350,7 +346,7 @@ class SuperprojectTestCase(unittest.TestCase):
                                return_value=data):
           # Create temporary directory so that it can write the file.
           os.mkdir(self._superproject._superproject_path)
-          update_result = self._superproject.UpdateProjectsRevisionId(projects)
+          update_result = self._superproject.UpdateProjectsRevisionId(projects, self.git_event_log)
           self.assertIsNotNone(update_result.manifest_path)
           self.assertFalse(update_result.fatal)
           with open(update_result.manifest_path, 'r') as fp:

tests/test_git_trace2_event_log.py

@@ -16,11 +16,42 @@
 
 import json
 import os
+import socket
 import tempfile
+import threading
 import unittest
 from unittest import mock
 
 import git_trace2_event_log
+import platform_utils
+
+
+def serverLoggingThread(socket_path, server_ready, received_traces):
+  """Helper function to receive logs over a Unix domain socket.
+
+  Appends received messages on the provided socket and appends to received_traces.
+
+  Args:
+    socket_path: path to a Unix domain socket on which to listen for traces
+    server_ready: a threading.Condition used to signal to the caller that this thread is ready to
+        accept connections
+    received_traces: a list to which received traces will be appended (after decoding to a utf-8
+        string).
+  """
+  platform_utils.remove(socket_path, missing_ok=True)
+  data = b''
+  with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
+    sock.bind(socket_path)
+    sock.listen(0)
+    with server_ready:
+      server_ready.notify()
+    with sock.accept()[0] as conn:
+      while True:
+        recved = conn.recv(4096)
+        if not recved:
+          break
+        data += recved
+  received_traces.extend(data.decode('utf-8').splitlines())
 
 
 class EventLogTestCase(unittest.TestCase):
@@ -324,6 +355,37 @@ class EventLogTestCase(unittest.TestCase):
     with self.assertRaises(TypeError):
       self._event_log_module.Write(path=1234)
 
+  def test_write_socket(self):
+    """Test Write() with Unix domain socket for |path| and validate received traces."""
+    received_traces = []
+    with tempfile.TemporaryDirectory(prefix='test_server_sockets') as tempdir:
+      socket_path = os.path.join(tempdir, "server.sock")
+      server_ready = threading.Condition()
+      # Start "server" listening on Unix domain socket at socket_path.
+      try:
+        server_thread = threading.Thread(
+            target=serverLoggingThread,
+            args=(socket_path, server_ready, received_traces))
+        server_thread.start()
+
+        with server_ready:
+          server_ready.wait()
+
+        self._event_log_module.StartEvent()
+        path = self._event_log_module.Write(path=f'af_unix:{socket_path}')
+      finally:
+        server_thread.join(timeout=5)
+
+    self.assertEqual(path, f'af_unix:stream:{socket_path}')
+    self.assertEqual(len(received_traces), 2)
+    version_event = json.loads(received_traces[0])
+    start_event = json.loads(received_traces[1])
+    self.verifyCommonKeys(version_event, expected_event_name='version')
+    self.verifyCommonKeys(start_event, expected_event_name='start')
+    # Check for 'start' event specific fields.
+    self.assertIn('argv', start_event)
+    self.assertIsInstance(start_event['argv'], list)
+
 
 if __name__ == '__main__':
   unittest.main()

tests/test_manifest_xml.py

@@ -17,13 +17,13 @@
 import os
 import platform
 import re
-import shutil
 import tempfile
 import unittest
 import xml.dom.minidom
 
 import error
 import manifest_xml
+import repo_trace
 
 
 # Invalid paths that we don't want in the filesystem.
@@ -92,7 +92,9 @@ class ManifestParseTestCase(unittest.TestCase):
   """TestCase for parsing manifests."""
 
   def setUp(self):
-    self.tempdir = tempfile.mkdtemp(prefix='repo_tests')
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    self.tempdir = self.tempdirobj.name
+    repo_trace._TRACE_FILE = os.path.join(self.tempdir, 'TRACE_FILE_from_test')
     self.repodir = os.path.join(self.tempdir, '.repo')
     self.manifest_dir = os.path.join(self.repodir, 'manifests')
     self.manifest_file = os.path.join(
@@ -111,7 +113,7 @@ class ManifestParseTestCase(unittest.TestCase):
 """)
 
   def tearDown(self):
-    shutil.rmtree(self.tempdir, ignore_errors=True)
+    self.tempdirobj.cleanup()
 
   def getXmlManifest(self, data):
     """Helper to initialize a manifest for testing."""
@@ -252,6 +254,37 @@ class XmlManifestTests(ManifestParseTestCase):
         '<manifest></manifest>')
     self.assertEqual(manifest.ToDict(), {})
 
+  def test_toxml_omit_local(self):
+    """Does not include local_manifests projects when omit_local=True."""
+    manifest = self.getXmlManifest(
+        '<?xml version="1.0" encoding="UTF-8"?><manifest>'
+        '<remote name="a" fetch=".."/><default remote="a" revision="r"/>'
+        '<project name="p" groups="local::me"/>'
+        '<project name="q"/>'
+        '<project name="r" groups="keep"/>'
+        '</manifest>')
+    self.assertEqual(
+        sort_attributes(manifest.ToXml(omit_local=True).toxml()),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch=".." name="a"/><default remote="a" revision="r"/>'
+        '<project name="q"/><project groups="keep" name="r"/></manifest>')
+
+  def test_toxml_with_local(self):
+    """Does include local_manifests projects when omit_local=False."""
+    manifest = self.getXmlManifest(
+        '<?xml version="1.0" encoding="UTF-8"?><manifest>'
+        '<remote name="a" fetch=".."/><default remote="a" revision="r"/>'
+        '<project name="p" groups="local::me"/>'
+        '<project name="q"/>'
+        '<project name="r" groups="keep"/>'
+        '</manifest>')
+    self.assertEqual(
+        sort_attributes(manifest.ToXml(omit_local=False).toxml()),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch=".." name="a"/><default remote="a" revision="r"/>'
+        '<project groups="local::me" name="p"/>'
+        '<project name="q"/><project groups="keep" name="r"/></manifest>')
+
   def test_repo_hooks(self):
     """Check repo-hooks settings."""
     manifest = self.getXmlManifest("""
@@ -289,8 +322,8 @@ class XmlManifestTests(ManifestParseTestCase):
   <x-custom-tag>X tags are always ignored</x-custom-tag>
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
+    self.assertEqual(manifest.superproject.name, 'superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'test-remote')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -569,10 +602,10 @@ class SuperProjectElementTests(ManifestParseTestCase):
   <superproject name="superproject"/>
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
-    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
-    self.assertEqual(manifest.superproject['revision'], 'refs/heads/main')
+    self.assertEqual(manifest.superproject.name, 'superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'test-remote')
+    self.assertEqual(manifest.superproject.remote.url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject.revision, 'refs/heads/main')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -591,10 +624,10 @@ class SuperProjectElementTests(ManifestParseTestCase):
   <superproject name="superproject" revision="refs/heads/stable" />
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
-    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
-    self.assertEqual(manifest.superproject['revision'], 'refs/heads/stable')
+    self.assertEqual(manifest.superproject.name, 'superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'test-remote')
+    self.assertEqual(manifest.superproject.remote.url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject.revision, 'refs/heads/stable')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -613,10 +646,10 @@ class SuperProjectElementTests(ManifestParseTestCase):
   <superproject name="superproject" revision="refs/heads/stable" />
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
-    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
-    self.assertEqual(manifest.superproject['revision'], 'refs/heads/stable')
+    self.assertEqual(manifest.superproject.name, 'superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'test-remote')
+    self.assertEqual(manifest.superproject.remote.url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject.revision, 'refs/heads/stable')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -635,10 +668,10 @@ class SuperProjectElementTests(ManifestParseTestCase):
   <superproject name="superproject" revision="refs/heads/stable" />
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
-    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
-    self.assertEqual(manifest.superproject['revision'], 'refs/heads/stable')
+    self.assertEqual(manifest.superproject.name, 'superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'test-remote')
+    self.assertEqual(manifest.superproject.remote.url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject.revision, 'refs/heads/stable')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -657,10 +690,10 @@ class SuperProjectElementTests(ManifestParseTestCase):
   <superproject name="platform/superproject" remote="superproject-remote"/>
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'platform/superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'superproject-remote')
-    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/platform/superproject')
-    self.assertEqual(manifest.superproject['revision'], 'refs/heads/main')
+    self.assertEqual(manifest.superproject.name, 'platform/superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'superproject-remote')
+    self.assertEqual(manifest.superproject.remote.url, 'http://localhost/platform/superproject')
+    self.assertEqual(manifest.superproject.revision, 'refs/heads/main')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -679,9 +712,9 @@ class SuperProjectElementTests(ManifestParseTestCase):
   <superproject name="superproject" remote="default-remote"/>
 </manifest>
 """)
-    self.assertEqual(manifest.superproject['name'], 'superproject')
-    self.assertEqual(manifest.superproject['remote'].name, 'default-remote')
-    self.assertEqual(manifest.superproject['revision'], 'refs/heads/main')
+    self.assertEqual(manifest.superproject.name, 'superproject')
+    self.assertEqual(manifest.superproject.remote.name, 'default-remote')
+    self.assertEqual(manifest.superproject.revision, 'refs/heads/main')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -843,3 +876,27 @@ class ExtendProjectElementTests(ManifestParseTestCase):
     else:
       self.assertEqual(manifest.projects[0].relpath, 'bar')
       self.assertEqual(manifest.projects[1].relpath, 'y')
+
+  def test_extend_project_dest_branch(self):
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" dest-branch="foo" />
+  <project name="myproject" />
+  <extend-project name="myproject" dest-branch="bar" />
+</manifest>
+""")
+    self.assertEqual(len(manifest.projects), 1)
+    self.assertEqual(manifest.projects[0].dest_branch, 'bar')
+
+  def test_extend_project_upstream(self):
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="myproject" />
+  <extend-project name="myproject" upstream="bar" />
+</manifest>
+""")
+    self.assertEqual(len(manifest.projects), 1)
+    self.assertEqual(manifest.projects[0].upstream, 'bar')

tests/test_project.py

@@ -17,7 +17,6 @@
 import contextlib
 import os
 from pathlib import Path
-import shutil
 import subprocess
 import tempfile
 import unittest
@@ -27,16 +26,13 @@ import git_command
 import git_config
 import platform_utils
 import project
+import repo_trace
 
 
 @contextlib.contextmanager
 def TempGitTree():
   """Create a new empty git checkout for testing."""
-  # TODO(vapier): Convert this to tempfile.TemporaryDirectory once we drop
-  # Python 2 support entirely.
-  try:
-    tempdir = tempfile.mkdtemp(prefix='repo-tests')
-
+  with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
     # Tests need to assume, that main is default branch at init,
     # which is not supported in config until 2.28.
     cmd = ['git', 'init']
@@ -50,8 +46,6 @@ def TempGitTree():
       cmd += ['--template', templatedir]
     subprocess.check_call(cmd, cwd=tempdir)
     yield tempdir
-  finally:
-    platform_utils.rmtree(tempdir)
 
 
 class FakeProject(object):
@@ -71,6 +65,13 @@ class FakeProject(object):
 class ReviewableBranchTests(unittest.TestCase):
   """Check ReviewableBranch behavior."""
 
+  def setUp(self):
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    repo_trace._TRACE_FILE = os.path.join(self.tempdirobj.name, 'TRACE_FILE_from_test')
+
+  def tearDown(self):
+    self.tempdirobj.cleanup()
+
   def test_smoke(self):
     """A quick run through everything."""
     with TempGitTree() as tempdir:
@@ -124,14 +125,15 @@ class CopyLinkTestCase(unittest.TestCase):
   """
 
   def setUp(self):
-    self.tempdir = tempfile.mkdtemp(prefix='repo_tests')
+    self.tempdirobj = tempfile.TemporaryDirectory(prefix='repo_tests')
+    self.tempdir = self.tempdirobj.name
     self.topdir = os.path.join(self.tempdir, 'checkout')
     self.worktree = os.path.join(self.topdir, 'git-project')
     os.makedirs(self.topdir)
     os.makedirs(self.worktree)
 
   def tearDown(self):
-    shutil.rmtree(self.tempdir, ignore_errors=True)
+    self.tempdirobj.cleanup()
 
   @staticmethod
   def touch(path):
@@ -347,6 +349,10 @@ class MigrateWorkTreeTests(unittest.TestCase):
   }
   _FILES = {
       'COMMIT_EDITMSG', 'FETCH_HEAD', 'HEAD', 'index', 'ORIG_HEAD',
+      'unknown-file-should-be-migrated',
+  }
+  _CLEAN_FILES = {
+      'a-vim-temp-file~', '#an-emacs-temp-file#',
   }
 
   @classmethod
@@ -365,10 +371,9 @@ class MigrateWorkTreeTests(unittest.TestCase):
       dotgit.mkdir(parents=True)
       for name in cls._SYMLINKS:
         (dotgit / name).symlink_to(f'../../../.repo/projects/src/test.git/{name}')
-      for name in cls._FILES:
+      for name in cls._FILES | cls._CLEAN_FILES:
         (dotgit / name).write_text(name)
 
-      subprocess.run(['tree', '-a', str(dotgit)])
       yield tempdir
 
   def test_standard(self):
@@ -379,9 +384,30 @@ class MigrateWorkTreeTests(unittest.TestCase):
 
       # Make sure the dir was transformed into a symlink.
       self.assertTrue(dotgit.is_symlink())
-      self.assertEqual(str(dotgit.readlink()), '../../.repo/projects/src/test.git')
+      self.assertEqual(os.readlink(dotgit), '../../.repo/projects/src/test.git')
 
       # Make sure files were moved over.
       gitdir = tempdir / '.repo/projects/src/test.git'
       for name in self._FILES:
         self.assertEqual(name, (gitdir / name).read_text())
+      # Make sure files were removed.
+      for name in self._CLEAN_FILES:
+        self.assertFalse((gitdir / name).exists())
+
+  def test_unknown(self):
+    """A checkout with unknown files should abort."""
+    with self._simple_layout() as tempdir:
+      dotgit = tempdir / 'src/test/.git'
+      (tempdir / '.repo/projects/src/test.git/random-file').write_text('one')
+      (dotgit / 'random-file').write_text('two')
+      with self.assertRaises(error.GitError):
+        project.Project._MigrateOldWorkTreeGitDir(str(dotgit))
+
+      # Make sure no content was actually changed.
+      self.assertTrue(dotgit.is_dir())
+      for name in self._FILES:
+        self.assertTrue((dotgit / name).is_file())
+      for name in self._CLEAN_FILES:
+        self.assertTrue((dotgit / name).is_file())
+      for name in self._SYMLINKS:
+        self.assertTrue((dotgit / name).is_symlink())

tests/test_subcmds_sync.py

@@ -0,0 +1,86 @@
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unittests for the subcmds/sync.py module."""
+
+import unittest
+from unittest import mock
+
+import pytest
+
+from subcmds import sync
+
+
+@pytest.mark.parametrize('use_superproject, cli_args, result', [
+    (True, ['--current-branch'], True),
+    (True, ['--no-current-branch'], True),
+    (True, [], True),
+    (False, ['--current-branch'], True),
+    (False, ['--no-current-branch'], False),
+    (False, [], None),
+])
+def test_get_current_branch_only(use_superproject, cli_args, result):
+  """Test Sync._GetCurrentBranchOnly logic.
+
+  Sync._GetCurrentBranchOnly should return True if a superproject is requested,
+  and otherwise the value of the current_branch_only option.
+  """
+  cmd = sync.Sync()
+  opts, _ = cmd.OptionParser.parse_args(cli_args)
+
+  with mock.patch('git_superproject.UseSuperproject',
+                  return_value=use_superproject):
+    assert cmd._GetCurrentBranchOnly(opts, cmd.manifest) == result
+
+
+class GetPreciousObjectsState(unittest.TestCase):
+  """Tests for _GetPreciousObjectsState."""
+
+  def setUp(self):
+    """Common setup."""
+    self.cmd = sync.Sync()
+    self.project = p = mock.MagicMock(use_git_worktrees=False,
+                                      UseAlternates=False)
+    p.manifest.GetProjectsWithName.return_value = [p]
+
+    self.opt = mock.Mock(spec_set=['this_manifest_only'])
+    self.opt.this_manifest_only = False
+
+  def test_worktrees(self):
+    """False for worktrees."""
+    self.project.use_git_worktrees = True
+    self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt))
+
+  def test_not_shared(self):
+    """Singleton project."""
+    self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt))
+
+  def test_shared(self):
+    """Shared project."""
+    self.project.manifest.GetProjectsWithName.return_value = [
+        self.project, self.project
+    ]
+    self.assertTrue(self.cmd._GetPreciousObjectsState(self.project, self.opt))
+
+  def test_shared_with_alternates(self):
+    """Shared project, with alternates."""
+    self.project.manifest.GetProjectsWithName.return_value = [
+        self.project, self.project
+    ]
+    self.project.UseAlternates = True
+    self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt))
+
+  def test_not_found(self):
+    """Project not found in manifest."""
+    self.project.manifest.GetProjectsWithName.return_value = []
+    self.assertFalse(self.cmd._GetPreciousObjectsState(self.project, self.opt))

tests/test_wrapper.py

@@ -14,11 +14,9 @@
 
 """Unittests for the wrapper.py module."""
 
-import contextlib
 from io import StringIO
 import os
 import re
-import shutil
 import sys
 import tempfile
 import unittest
@@ -26,22 +24,9 @@ from unittest import mock
 
 import git_command
 import main
-import platform_utils
 import wrapper
 
 
-@contextlib.contextmanager
-def TemporaryDirectory():
-  """Create a new empty git checkout for testing."""
-  # TODO(vapier): Convert this to tempfile.TemporaryDirectory once we drop
-  # Python 2 support entirely.
-  try:
-    tempdir = tempfile.mkdtemp(prefix='repo-tests')
-    yield tempdir
-  finally:
-    platform_utils.rmtree(tempdir)
-
-
 def fixture(*paths):
   """Return a path relative to tests/fixtures.
   """
@@ -336,19 +321,19 @@ class NeedSetupGnuPG(RepoWrapperTestCase):
 
   def test_missing_dir(self):
     """The ~/.repoconfig tree doesn't exist yet."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = os.path.join(tempdir, 'foo')
       self.assertTrue(self.wrapper.NeedSetupGnuPG())
 
   def test_missing_keyring(self):
     """The keyring-version file doesn't exist yet."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       self.assertTrue(self.wrapper.NeedSetupGnuPG())
 
   def test_empty_keyring(self):
     """The keyring-version file exists, but is empty."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       with open(os.path.join(tempdir, 'keyring-version'), 'w'):
         pass
@@ -356,7 +341,7 @@ class NeedSetupGnuPG(RepoWrapperTestCase):
 
   def test_old_keyring(self):
     """The keyring-version file exists, but it's old."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       with open(os.path.join(tempdir, 'keyring-version'), 'w') as fp:
         fp.write('1.0\n')
@@ -364,7 +349,7 @@ class NeedSetupGnuPG(RepoWrapperTestCase):
 
   def test_new_keyring(self):
     """The keyring-version file exists, and is up-to-date."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       with open(os.path.join(tempdir, 'keyring-version'), 'w') as fp:
         fp.write('1000.0\n')
@@ -376,7 +361,7 @@ class SetupGnuPG(RepoWrapperTestCase):
 
   def test_full(self):
     """Make sure it works completely."""
-    with TemporaryDirectory() as tempdir:
+    with tempfile.TemporaryDirectory(prefix='repo-tests') as tempdir:
       self.wrapper.home_dot_repo = tempdir
       self.wrapper.gpg_dir = os.path.join(self.wrapper.home_dot_repo, 'gnupg')
       self.assertTrue(self.wrapper.SetupGnuPG(True))
@@ -426,7 +411,8 @@ class GitCheckoutTestCase(RepoWrapperTestCase):
   @classmethod
   def setUpClass(cls):
     # Create a repo to operate on, but do it once per-class.
-    cls.GIT_DIR = tempfile.mkdtemp(prefix='repo-rev-tests')
+    cls.tempdirobj = tempfile.TemporaryDirectory(prefix='repo-rev-tests')
+    cls.GIT_DIR = cls.tempdirobj.name
     run_git = wrapper.Wrapper().run_git
 
     remote = os.path.join(cls.GIT_DIR, 'remote')
@@ -455,10 +441,10 @@ class GitCheckoutTestCase(RepoWrapperTestCase):
 
   @classmethod
   def tearDownClass(cls):
-    if not cls.GIT_DIR:
+    if not cls.tempdirobj:
       return
 
-    shutil.rmtree(cls.GIT_DIR)
+    cls.tempdirobj.cleanup()
 
 
 class ResolveRepoRev(GitCheckoutTestCase):