project: migrate worktree .git/ dirs to symlinks

Historically we created a .git/ subdir in each source checkout and
symlinked individual files to the .repo/projects/ paths.  This layer
of indirection isn't actually needed: the .repo/projects/ paths are
guaranteed to only ever have a 1-to-1 mapping with the actual git
checkout.  So we don't need to worry about having files in .git/ be
isolated.

To that end, change how we manage the actual project checkouts from
a dir full of symlinks (and a few files) to a symlink to the internal
.repo/projects/ dir.  This makes the code simpler & faster.

The directory structure we have today is:
.repo/
  project-objects/chromiumos/third_party/kernel.git/
    <paths omitted as not relevant to this change>
  projects/src/third_party/kernel/
    v3.8.git/
      config
      description   -> …/project-objects/…/config
      FETCH_HEAD
      HEAD
      hooks/        -> …/project-objects/…/hooks/
      info/         -> …/project-objects/…/info/
      logs/
      objects/      -> …/project-objects/…/objects/
      packed-refs
      refs/
      rr-cache/     -> …/project-objects/…/rr-cache/
src/third_party/kernel/
  v3.8/
    .git/
      config        -> …/projects/…/v3.8.git/config
      description   -> …/project-objects/…/v3.8.git/description
      HEAD
      hooks/        -> …/project-objects/…/v3.8.git/hooks/
      index
      info/         -> …/project-objects/…/v3.8.git/info/
      logs/         -> …/projects/…/v3.8.git/logs/
      objects/      -> …/project-objects/…/v3.8.git/objects/
      packed-refs   -> …/projects/…/v3.8.git/packed-refs
      refs/         -> …/projects/…/v3.8.git/refs/
      rr-cache/     -> …/project-objects/…/v3.8.git/rr-cache/

The directory structure we have after this commit:
.repo/
  <nothing changes>
src/third_party/kernel/
  v3.8/
    .git            -> …/projects/…/v3.8.git

Bug: https://crbug.com/gerrit/15273
Change-Id: I9dd8def23fbfb2f4cb209a93f8b1b2b24002a444
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/323695
Reviewed-by: Mike Nichols <mikenichols@google.com>
Reviewed-by: Xin Li <delphij@google.com>
Tested-by: Mike Frysinger <vapier@google.com>

.gitreview

@@ -0,0 +1,5 @@
+[gerrit]
+host=gerrit-review.googlesource.com
+scheme=https
+project=git-repo.git
+defaultbranch=main

SUBMITTING_PATCHES.md

@@ -3,7 +3,7 @@
 # Short Version
 
  - Make small logical changes.
- - Provide a meaningful commit message.
+ - [Provide a meaningful commit message][commit-message-style].
  - Check for coding errors and style nits with flake8.
  - Make sure all code is under the Apache License, 2.0.
  - Publish your changes for review.
@@ -26,10 +26,11 @@ yourself with the following relevant bits.
 
 ## Make separate commits for logically separate changes.
 
-Unless your patch is really trivial, you should not be sending
+Unless your patch is really trivial, you should not be sending out a patch that
-out a patch that was generated between your working tree and your
+was generated between your working tree and your commit head.
-commit head.  Instead, always make a commit with complete commit
+Instead, always make a commit with a complete
-message and generate a series of patches from your repository.
+[commit message][commit-message-style] and generate a series of patches from
+your repository.
 It is a good discipline.
 
 Describe the technical detail of the change(s).
@@ -171,3 +172,6 @@ After you receive a Code-Review+2 from the maintainer, select the Verified
 button on the gerrit page for the change. This verifies that you have tested
 your changes and notifies the maintainer that they are ready to be submitted.
 The maintainer will then submit your changes to the repository.
+
+
+[commit-message-style]: https://chris.beams.io/posts/git-commit/

command.py

@@ -24,6 +24,10 @@ from error import InvalidProjectGroupsError
 import progress
 
 
+# Are we generating man-pages?
+GENERATE_MANPAGES = os.environ.get('_REPO_GENERATE_MANPAGES_') == ' indeed! '
+
+
 # Number of projects to submit to a single worker process at a time.
 # This number represents a tradeoff between the overhead of IPC and finer
 # grained opportunity for parallelism. This particular value was chosen by
@@ -122,10 +126,14 @@ class Command(object):
                  help='only show errors')
 
     if self.PARALLEL_JOBS is not None:
+      default = 'based on number of CPU cores'
+      if not GENERATE_MANPAGES:
+        # Only include active cpu count if we aren't generating man pages.
+        default = f'%default; {default}'
       p.add_option(
           '-j', '--jobs',
           type=int, default=self.PARALLEL_JOBS,
-          help='number of jobs to run in parallel (default: %s)' % self.PARALLEL_JOBS)
+          help=f'number of jobs to run in parallel (default: {default})')
 
   def _Options(self, p):
     """Initialize the option parser with subcommand-specific options."""

completion.bash

@@ -14,6 +14,9 @@
 
 # Programmable bash completion.  https://github.com/scop/bash-completion
 
+# TODO: Handle interspersed options.  We handle `repo h<tab>`, but not
+# `repo --time h<tab>`.
+
 # Complete the list of repo subcommands.
 __complete_repo_list_commands() {
   local repo=${COMP_WORDS[0]}
@@ -37,6 +40,7 @@ __complete_repo_list_branches() {
 __complete_repo_list_projects() {
   local repo=${COMP_WORDS[0]}
   "${repo}" list -n 2>/dev/null
+  "${repo}" list -p --relative-to=. 2>/dev/null
 }
 
 # Complete the repo <command> argument.
@@ -66,6 +70,48 @@ __complete_repo_command_projects() {
   COMPREPLY=($(compgen -W "$(__complete_repo_list_projects)" -- "${current}"))
 }
 
+# Complete `repo help`.
+__complete_repo_command_help() {
+  local current=$1
+  # CWORD=1 is "start".
+  # CWORD=2 is the <subcommand> which we complete here.
+  if [[ ${COMP_CWORD} -eq 2 ]]; then
+    COMPREPLY=(
+      $(compgen -W "$(__complete_repo_list_commands)" -- "${current}")
+    )
+  fi
+}
+
+# Complete `repo forall`.
+__complete_repo_command_forall() {
+  local current=$1
+  # CWORD=1 is "forall".
+  # CWORD=2+ are <projects> *until* we hit the -c option.
+  local i
+  for (( i = 0; i < COMP_CWORD; ++i )); do
+    if [[ "${COMP_WORDS[i]}" == "-c" ]]; then
+      return 0
+    fi
+  done
+
+  COMPREPLY=(
+    $(compgen -W "$(__complete_repo_list_projects)" -- "${current}")
+  )
+}
+
+# Complete `repo start`.
+__complete_repo_command_start() {
+  local current=$1
+  # CWORD=1 is "start".
+  # CWORD=2 is the <branch> which we don't complete.
+  # CWORD=3+ are <projects> which we complete here.
+  if [[ ${COMP_CWORD} -gt 2 ]]; then
+    COMPREPLY=(
+      $(compgen -W "$(__complete_repo_list_projects)" -- "${current}")
+    )
+  fi
+}
+
 # Complete the repo subcommand arguments.
 __complete_repo_arg() {
   if [[ ${COMP_CWORD} -le 1 ]]; then
@@ -86,21 +132,8 @@ __complete_repo_arg() {
     return 0
     ;;
 
-  help)
+  help|start|forall)
-    if [[ ${COMP_CWORD} -eq 2 ]]; then
+    __complete_repo_command_${command} "${current}"
-      COMPREPLY=(
-        $(compgen -W "$(__complete_repo_list_commands)" -- "${current}")
-      )
-    fi
-    return 0
-    ;;
-
-  start)
-    if [[ ${COMP_CWORD} -gt 2 ]]; then
-      COMPREPLY=(
-        $(compgen -W "$(__complete_repo_list_projects)" -- "${current}")
-      )
-    fi
     return 0
     ;;
 
@@ -118,4 +151,6 @@ __complete_repo() {
   return 0
 }
 
-complete -F __complete_repo repo
+# Fallback to the default complete methods if we aren't able to provide anything
+# useful.  This will allow e.g. local paths to be used when it makes sense.
+complete -F __complete_repo -o bashdefault -o default repo

docs/internal-fs-layout.md

@@ -146,12 +146,18 @@ Instead, you should use standard Git workflows like [git worktree] or
 
 The `.repo/manifests.git/config` file is used to track settings for the entire
 repo client checkout.
+
 Most settings use the `[repo]` section to avoid conflicts with git.
+
+Everything under `[repo.syncstate.*]` is used to keep track of sync details for logging
+purposes.
+
 User controlled settings are initialized when running `repo init`.
 
 | Setting                  | `repo init` Option        | Use/Meaning |
 |-------------------       |---------------------------|-------------|
 | manifest.groups          | `--groups` & `--platform` | The manifest groups to sync |
+| manifest.standalone      | `--standalone-manifest`   | Download manifest as static file instead of creating checkout |
 | repo.archive             | `--archive`               | Use `git archive` for checkouts |
 | repo.clonebundle         | `--clone-bundle`          | Whether the initial sync used clone.bundle explicitly |
 | repo.clonefilter         | `--clone-filter`          | Filter setting when using [partial git clones] |

docs/manifest-format.md

@@ -36,7 +36,7 @@ following DTD:
 
   <!ELEMENT notice (#PCDATA)>
 
-  <!ELEMENT remote EMPTY>
+  <!ELEMENT remote (annotation*)>
   <!ATTLIST remote name         ID    #REQUIRED>
   <!ATTLIST remote alias        CDATA #IMPLIED>
   <!ATTLIST remote fetch        CDATA #REQUIRED>
@@ -90,6 +90,7 @@ following DTD:
   <!ELEMENT extend-project EMPTY>
   <!ATTLIST extend-project name CDATA #REQUIRED>
   <!ATTLIST extend-project path CDATA #IMPLIED>
+  <!ATTLIST extend-project dest-path CDATA #IMPLIED>
   <!ATTLIST extend-project groups CDATA #IMPLIED>
   <!ATTLIST extend-project revision CDATA #IMPLIED>
   <!ATTLIST extend-project remote CDATA #IMPLIED>
@@ -103,8 +104,9 @@ following DTD:
   <!ATTLIST repo-hooks enabled-list CDATA #REQUIRED>
 
   <!ELEMENT superproject EMPTY>
-  <!ATTLIST superproject name    CDATA #REQUIRED>
+  <!ATTLIST superproject name     CDATA #REQUIRED>
-  <!ATTLIST superproject remote  IDREF #IMPLIED>
+  <!ATTLIST superproject remote   IDREF #IMPLIED>
+  <!ATTLIST superproject revision CDATA #IMPLIED>
 
   <!ELEMENT contactinfo EMPTY>
   <!ATTLIST contactinfo bugurl  CDATA #REQUIRED>
@@ -336,6 +338,11 @@ against changes to the original manifest.
 Attribute `path`: If specified, limit the change to projects checked out
 at the specified path, rather than all projects with the given name.
 
+Attribute `dest-path`: If specified, a path relative to the top directory
+of the repo client where the Git working directory for this project
+should be placed.  This is used to move a project in the checkout by
+overriding the existing `path` setting.
+
 Attribute `groups`: List of additional groups to which this project
 belongs.  Same syntax as the corresponding element of `project`.
 
@@ -348,12 +355,12 @@ project.  Same syntax as the corresponding element of `project`.
 ### Element annotation
 
 Zero or more annotation elements may be specified as children of a
-project element. Each element describes a name-value pair that will be
+project or remote element. Each element describes a name-value pair.
-exported into each project's environment during a 'forall' command,
+For projects, this name-value pair will be exported into each project's
-prefixed with REPO__.  In addition, there is an optional attribute
+environment during a 'forall' command, prefixed with `REPO__`.  In addition,
-"keep" which accepts the case insensitive values "true" (default) or
+there is an optional attribute "keep" which accepts the case insensitive values
-"false".  This attribute determines whether or not the annotation will
+"true" (default) or "false".  This attribute determines whether or not the
-be kept when exported with the manifest subcommand.
+annotation will be kept when exported with the manifest subcommand.
 
 ### Element copyfile
 
@@ -432,6 +439,11 @@ same meaning as project's name attribute. See the
 Attribute `remote`: Name of a previously defined remote element.
 If not supplied the remote given by the default element is used.
 
+Attribute `revision`: Name of the Git branch the manifest wants
+to track for this superproject. If not supplied the revision given
+by the remote element is used if applicable, else the default
+element is used.
+
 ### Element contactinfo
 
 ***

docs/release-process.md

@@ -208,80 +208,132 @@ Things in bold indicate stuff to take note of, but does not guarantee that we
 still support them.
 Things in italics are things we used to care about but probably don't anymore.
 
-|   Date   |     EOL      | [Git][rel-g] | [Python][rel-p] | [Ubuntu][rel-u] / [Debian][rel-d] | Git | Python |
+|   Date   |     EOL      | [Git][rel-g] | [Python][rel-p] | [SSH][rel-o] | [Ubuntu][rel-u] / [Debian][rel-d] | Git | Python | SSH |
-|:--------:|:------------:|--------------|-----------------|-----------------------------------|-----|--------|
+|:--------:|:------------:|:------------:|:---------------:|:------------:|-----------------------------------|-----|--------|-----|
-| Oct 2008 | *Oct 2013*   |              | 2.6.0           | *10.04 Lucid* - 10.10 Maverick / *Squeeze* |
+| Apr 2008 |              |              |                 | 5.0          |
+| Jun 2008 |              |              |                 | 5.1          |
+| Oct 2008 | *Oct 2013*   |              | 2.6.0           |              | *10.04 Lucid* - 10.10 Maverick / *Squeeze* |
 | Dec 2008 | *Feb 2009*   |              | 3.0.0           |
-| Feb 2009 | *Mar 2012*   |              |                 | Debian 5 Lenny       | 1.5.6.5 | 2.5.2 |
+| Feb 2009 |              |              |                 | 5.2          |
-| Jun 2009 | *Jun 2016*   |              | 3.1.0           | *10.04 Lucid* - 10.10 Maverick / *Squeeze* |
+| Feb 2009 | *Mar 2012*   |              |                 |              | Debian 5 Lenny       | 1.5.6.5 | 2.5.2 |
-| Feb 2010 | *Oct 2012*   | 1.7.0        |                 | *10.04 Lucid* - *12.04 Precise* - 12.10 Quantal |
+| Jun 2009 | *Jun 2016*   |              | 3.1.0           |              | *10.04 Lucid* - 10.10 Maverick / *Squeeze* |
-| Apr 2010 | *Apr 2015*   |              |                 | *10.04 Lucid*        | 1.7.0.4  | 2.6.5 3.1.2  |
+| Sep 2009 |              |              |                 | 5.3          | *10.04 Lucid* |
-| Jul 2010 | *Dec 2019*   |              | **2.7.0**       | 11.04 Natty - **<current>** |
+| Feb 2010 | *Oct 2012*   | 1.7.0        |                 |              | *10.04 Lucid* - *12.04 Precise* - 12.10 Quantal |
-| Oct 2010 |              |              |                 | 10.10 Maverick       | 1.7.1    | 2.6.6 3.1.3  |
+| Mar 2010 |              |              |                 | 5.4          |
-| Feb 2011 | *Feb 2016*   |              |                 | Debian 6 Squeeze     | 1.7.2.5  | 2.6.6 3.1.3  |
+| Apr 2010 |              |              |                 | 5.5          | 10.10 Maverick |
-| Apr 2011 |              |              |                 | 11.04 Natty          | 1.7.4    | 2.7.1 3.2.0  |
+| Apr 2010 | *Apr 2015*   |              |                 |              | *10.04 Lucid*        | 1.7.0.4  | 2.6.5 3.1.2  | 5.3 |
-| Oct 2011 | *Feb 2016*   |              | 3.2.0           | 11.04 Natty - 12.10 Quantal |
+| Jul 2010 | *Dec 2019*   |              | *2.7.0*         |              | 11.04 Natty - *<current>* |
-| Oct 2011 |              |              |                 | 11.10 Ocelot         | 1.7.5.4  | 2.7.2 3.2.2  |
+| Aug 2010 |              |              |                 | 5.6          |
-| Apr 2012 | *Apr 2019*   |              |                 | *12.04 Precise*      | 1.7.9.5  | 2.7.3 3.2.3  |
+| Oct 2010 |              |              |                 |              | 10.10 Maverick       | 1.7.1    | 2.6.6 3.1.3  | 5.5 |
-| Sep 2012 | *Sep 2017*   |              | 3.3.0           | 13.04 Raring - 13.10 Saucy |
+| Jan 2011 |              |              |                 | 5.7          |
-| Oct 2012 | *Dec 2014*   | 1.8.0        |                 | 13.04 Raring - 13.10 Saucy |
+| Feb 2011 |              |              |                 | 5.8          | 11.04 Natty |
-| Oct 2012 |              |              |                 | 12.10 Quantal        | 1.7.10.4 | 2.7.3 3.2.3  |
+| Feb 2011 | *Feb 2016*   |              |                 |              | Debian 6 Squeeze     | 1.7.2.5  | 2.6.6 3.1.3  |
-| Apr 2013 |              |              |                 | 13.04 Raring         | 1.8.1.2  | 2.7.4 3.3.1  |
+| Apr 2011 |              |              |                 |              | 11.04 Natty          | 1.7.4    | 2.7.1 3.2.0  | 5.8 |
-| May 2013 | *May 2018*   |              |                 | Debian 7 Wheezy      | 1.7.10.4 | 2.7.3 3.2.3  |
+| Sep 2011 |              |              |                 | 5.9          | *12.04 Precise* |
-| Oct 2013 |              |              |                 | 13.10 Saucy          | 1.8.3.2  | 2.7.5 3.3.2  |
+| Oct 2011 | *Feb 2016*   |              | 3.2.0           |              | 11.04 Natty - 12.10 Quantal |
-| Feb 2014 | *Dec 2014*   | **1.9.0**    |                 | **14.04 Trusty** |
+| Oct 2011 |              |              |                 |              | 11.10 Ocelot         | 1.7.5.4  | 2.7.2 3.2.2  | 5.8 |
-| Mar 2014 | *Mar 2019*   |              | **3.4.0**       | **14.04 Trusty** - 15.10 Wily / **Jessie** |
+| Apr 2012 |              |              |                 | 6.0          | 12.10 Quantal |
-| Apr 2014 | **Apr 2022** |              |                 | **14.04 Trusty**     | 1.9.1    | 2.7.5 3.4.0  |
+| Apr 2012 | *Apr 2019*   |              |                 |              | *12.04 Precise*      | 1.7.9.5  | 2.7.3 3.2.3  | 5.9 |
+| Aug 2012 |              |              |                 | 6.1          | 13.04 Raring |
+| Sep 2012 | *Sep 2017*   |              | 3.3.0           |              | 13.04 Raring - 13.10 Saucy |
+| Oct 2012 | *Dec 2014*   | 1.8.0        |                 |              | 13.04 Raring - 13.10 Saucy |
+| Oct 2012 |              |              |                 |              | 12.10 Quantal        | 1.7.10.4 | 2.7.3 3.2.3  | 6.0 |
+| Mar 2013 |              |              |                 | 6.2          | 13.10 Saucy |
+| Apr 2013 |              |              |                 |              | 13.04 Raring         | 1.8.1.2  | 2.7.4 3.3.1  | 6.1 |
+| May 2013 | *May 2018*   |              |                 |              | Debian 7 Wheezy      | 1.7.10.4 | 2.7.3 3.2.3  |
+| Sep 2013 |              |              |                 | 6.3          |
+| Oct 2013 |              |              |                 |              | 13.10 Saucy          | 1.8.3.2  | 2.7.5 3.3.2  | 6.2 |
+| Nov 2013 |              |              |                 | 6.4          |
+| Jan 2014 |              |              |                 | 6.5          |
+| Feb 2014 | *Dec 2014*   | **1.9.0**    |                 |              | *14.04 Trusty* |
+| Mar 2014 | *Mar 2019*   |              | *3.4.0*         |              | *14.04 Trusty* - 15.10 Wily / *Jessie* |
+| Mar 2014 |              |              |                 | 6.6          | *14.04 Trusty* - 14.10 Utopic |
+| Apr 2014 | *Apr 2022*   |              |                 |              | *14.04 Trusty*       | 1.9.1    | 2.7.5 3.4.0  | 6.6 |
 | May 2014 | *Dec 2014*   | 2.0.0        |
-| Aug 2014 | *Dec 2014*   | **2.1.0**    |                 | 14.10 Utopic - 15.04 Vivid / **Jessie** |
+| Aug 2014 | *Dec 2014*   | *2.1.0*      |                 |              | 14.10 Utopic - 15.04 Vivid / *Jessie* |
-| Oct 2014 |              |              |                 | 14.10 Utopic         | 2.1.0    | 2.7.8 3.4.2  |
+| Oct 2014 |              |              |                 | 6.7          | 15.04 Vivid |
+| Oct 2014 |              |              |                 |              | 14.10 Utopic         | 2.1.0    | 2.7.8 3.4.2  | 6.6 |
 | Nov 2014 | *Sep 2015*   | 2.2.0        |
 | Feb 2015 | *Sep 2015*   | 2.3.0        |
+| Mar 2015 |              |              |                 | 6.8          |
 | Apr 2015 | *May 2017*   | 2.4.0        |
-| Apr 2015 | **Jun 2020** |              |                 | **Debian 8 Jessie**  | 2.1.4    | 2.7.9 3.4.2  |
+| Apr 2015 | *Jun 2020*   |              |                 |              | *Debian 8 Jessie*    | 2.1.4    | 2.7.9 3.4.2  |
-| Apr 2015 |              |              |                 | 15.04 Vivid          | 2.1.4    | 2.7.9 3.4.3  |
+| Apr 2015 |              |              |                 |              | 15.04 Vivid          | 2.1.4    | 2.7.9 3.4.3  | 6.7 |
-| Jul 2015 | *May 2017*   | 2.5.0        |                 | 15.10 Wily |
+| Jul 2015 | *May 2017*   | 2.5.0        |                 |              | 15.10 Wily |
+| Jul 2015 |              |              |                 | 6.9          | 15.10 Wily |
+| Aug 2015 |              |              |                 | 7.0          |
+| Aug 2015 |              |              |                 | 7.1          |
 | Sep 2015 | *May 2017*   | 2.6.0        |
-| Sep 2015 | **Sep 2020** |              | **3.5.0**       | **16.04 Xenial** - 17.04 Zesty / **Stretch** |
+| Sep 2015 | *Sep 2020*   |              | *3.5.0*         |              | *16.04 Xenial* - 17.04 Zesty / *Stretch* |
-| Oct 2015 |              |              |                 | 15.10 Wily           | 2.5.0    | 2.7.9 3.4.3  |
+| Oct 2015 |              |              |                 |              | 15.10 Wily           | 2.5.0    | 2.7.9 3.4.3  | 6.9 |
-| Jan 2016 | *Jul 2017*   | **2.7.0**    |                 | **16.04 Xenial** |
+| Jan 2016 | *Jul 2017*   | *2.7.0*      |                 |              | *16.04 Xenial* |
+| Feb 2016 |              |              |                 | 7.2          | *16.04 Xenial* |
 | Mar 2016 | *Jul 2017*   | 2.8.0        |
-| Apr 2016 | **Apr 2024** |              |                 | **16.04 Xenial**     | 2.7.4    | 2.7.11 3.5.1 |
+| Apr 2016 | *Apr 2024*   |              |                 |              | *16.04 Xenial*       | 2.7.4    | 2.7.11 3.5.1 | 7.2 |
-| Jun 2016 | *Jul 2017*   | 2.9.0        |                 | 16.10 Yakkety |
+| Jun 2016 | *Jul 2017*   | 2.9.0        |                 |              | 16.10 Yakkety |
+| Jul 2016 |              |              |                 | 7.3          | 16.10 Yakkety |
 | Sep 2016 | *Sep 2017*   | 2.10.0       |
-| Oct 2016 |              |              |                 | 16.10 Yakkety        | 2.9.3    | 2.7.11 3.5.1 |
+| Oct 2016 |              |              |                 |              | 16.10 Yakkety        | 2.9.3    | 2.7.11 3.5.1 | 7.3 |
-| Nov 2016 | *Sep 2017*   | **2.11.0**   |                 | 17.04 Zesty / **Stretch** |
+| Nov 2016 | *Sep 2017*   | *2.11.0*     |                 |              | 17.04 Zesty / *Stretch* |
-| Dec 2016 | **Dec 2021** |              | **3.6.0**       | 17.10 Artful - **18.04 Bionic** - 18.10 Cosmic |
+| Dec 2016 | **Dec 2021** |              | **3.6.0**       |              | 17.10 Artful - **18.04 Bionic** - 18.10 Cosmic |
+| Dec 2016 |              |              |                 | 7.4          | 17.04 Zesty / *Debian 9 Stretch* |
 | Feb 2017 | *Sep 2017*   | 2.12.0       |
-| Apr 2017 |              |              |                 | 17.04 Zesty          | 2.11.0   | 2.7.13 3.5.3 |
+| Mar 2017 |              |              |                 | 7.5          | 17.10 Artful |
+| Apr 2017 |              |              |                 |              | 17.04 Zesty          | 2.11.0   | 2.7.13 3.5.3 | 7.4 |
 | May 2017 | *May 2018*   | 2.13.0       |
-| Jun 2017 | **Jun 2022** |              |                 | **Debian 9 Stretch** | 2.11.0   | 2.7.13 3.5.3 |
+| Jun 2017 | *Jun 2022*   |              |                 |              | *Debian 9 Stretch*   | 2.11.0   | 2.7.13 3.5.3 | 7.4 |
-| Aug 2017 | *Dec 2019*   | 2.14.0       |                 | 17.10 Artful |
+| Aug 2017 | *Dec 2019*   | 2.14.0       |                 |              | 17.10 Artful |
 | Oct 2017 | *Dec 2019*   | 2.15.0       |
-| Oct 2017 |              |              |                 | 17.10 Artful         | 2.14.1   | 2.7.14 3.6.3 |
+| Oct 2017 |              |              |                 | 7.6          | **18.04 Bionic** |
+| Oct 2017 |              |              |                 |              | 17.10 Artful         | 2.14.1   | 2.7.14 3.6.3 | 7.5 |
 | Jan 2018 | *Dec 2019*   | 2.16.0       |
-| Apr 2018 | *Dec 2019*   | 2.17.0       |                 | **18.04 Bionic**     |
+| Apr 2018 | *Mar 2021*   | **2.17.0**   |                 |              | **18.04 Bionic**     |
-| Apr 2018 | **Apr 2028** |              |                 | **18.04 Bionic**     | 2.17.0   | 2.7.15 3.6.5 |
+| Apr 2018 |              |              |                 | 7.7          | 18.10 Cosmic |
-| Jun 2018 | *Dec 2019*   | 2.18.0       |
+| Apr 2018 | **Apr 2028** |              |                 |              | **18.04 Bionic**     | 2.17.0   | 2.7.15 3.6.5 | 7.6 |
-| Jun 2018 | **Jun 2023** |              | 3.7.0           | 19.04 Disco - **20.04 Focal** / **Buster** |
+| Jun 2018 | *Mar 2021*   | 2.18.0       |
-| Sep 2018 | *Dec 2019*   | 2.19.0       |                 | 18.10 Cosmic |
+| Jun 2018 | **Jun 2023** |              | 3.7.0           |              | 19.04 Disco - **20.04 Focal** / **Buster** |
-| Oct 2018 |              |              |                 | 18.10 Cosmic         | 2.19.1   | 2.7.15 3.6.6 |
+| Aug 2018 |              |              |                 | 7.8          |
-| Dec 2018 | *Dec 2019*   | **2.20.0**   |                 | 19.04 Disco / **Buster** |
+| Sep 2018 | *Mar 2021*   | 2.19.0       |                 |              | 18.10 Cosmic |
-| Feb 2019 | *Dec 2019*   | 2.21.0       |
+| Oct 2018 |              |              |                 | 7.9          | 19.04 Disco / **Buster** |
-| Apr 2019 |              |              |                 | 19.04 Disco          | 2.20.1   | 2.7.16 3.7.3 |
+| Oct 2018 |              |              |                 |              | 18.10 Cosmic         | 2.19.1   | 2.7.15 3.6.6 | 7.7 |
+| Dec 2018 | *Mar 2021*   | **2.20.0**   |                 |              | 19.04 Disco - 19.10 Eoan / **Buster** |
+| Feb 2019 | *Mar 2021*   | 2.21.0       |
+| Apr 2019 |              |              |                 | 8.0          | 19.10 Eoan |
+| Apr 2019 |              |              |                 |              | 19.04 Disco          | 2.20.1   | 2.7.16 3.7.3 | 7.9 |
 | Jun 2019 |              | 2.22.0       |
-| Jul 2019 | **Jul 2024** |              |                 | **Debian 10 Buster** | 2.20.1   | 2.7.16 3.7.3 |
+| Jul 2019 | **Jul 2024** |              |                 |              | **Debian 10 Buster** | 2.20.1   | 2.7.16 3.7.3 | 7.9 |
-| Aug 2019 |              | 2.23.0       |
+| Aug 2019 | *Mar 2021*   | 2.23.0       |
-| Oct 2019 | **Oct 2024** |              | 3.8.0           |
+| Oct 2019 | **Oct 2024** |              | 3.8.0           |              | **20.04 Focal** - 20.10 Groovy |
-| Oct 2019 |              |              |                 | 19.10 Eoan           | 2.20.1   | 2.7.17 3.7.5 |
+| Oct 2019 |              |              |                 | 8.1          |
-| Nov 2019 |              | 2.24.0       |
+| Oct 2019 |              |              |                 |              | 19.10 Eoan           | 2.20.1   | 2.7.17 3.7.5 | 8.0 |
-| Jan 2020 |              | 2.25.0       |                 | **20.04 Focal** |
+| Nov 2019 | *Mar 2021*   | 2.24.0       |
-| Apr 2020 | **Apr 2030** |              |                 | **20.04 Focal**      | 2.25.0   | 2.7.17 3.7.5 |
+| Jan 2020 | *Mar 2021*   | 2.25.0       |                 |              | **20.04 Focal** |
+| Feb 2020 |              |              |                 | 8.2          | **20.04 Focal** |
+| Mar 2020 | *Mar 2021*   | 2.26.0       |
+| Apr 2020 | **Apr 2030** |              |                 |              | **20.04 Focal**      | 2.25.1   | 2.7.17 3.8.2 | 8.2 |
+| May 2020 | *Mar 2021*   | 2.27.0       |                 |              | 20.10 Groovy |
+| May 2020 |              |              |                 | 8.3          |
+| Jul 2020 | *Mar 2021*   | 2.28.0       |
+| Sep 2020 |              |              |                 | 8.4          | 21.04 Hirsute / **Bullseye** |
+| Oct 2020 | *Mar 2021*   | 2.29.0       |
+| Oct 2020 |              |              |                 |              | 20.10 Groovy         | 2.27.0   | 2.7.18 3.8.6 | 8.3 |
+| Oct 2020 | **Oct 2025** |              | 3.9.0           |              | 21.04 Hirsute / **Bullseye** |
+| Dec 2020 | *Mar 2021*   | 2.30.0       |                 |              | 21.04 Hirsute / **Bullseye** |
+| Mar 2021 |              | 2.31.0       |
+| Mar 2021 |              |              |                 | 8.5          |
+| Apr 2021 |              |              |                 | 8.6          |
+| Apr 2021 | *Jan 2022*   |              |                 |              | 21.04 Hirsute        | 2.30.2   | 2.7.18 3.9.4 | 8.4 |
+| Jun 2021 |              | 2.32.0       |
+| Aug 2021 |              | 2.33.0       |
+| Aug 2021 |              |              |                 | 8.7          |
+| Aug 2021 | **Aug 2026** |              |                 |              | **Debian 11 Bullseye** | 2.30.2 | 2.7.18 3.9.2 | 8.4 |
+| **Date** |   **EOL**    | **[Git][rel-g]** | **[Python][rel-p]** | **[SSH][rel-o]** | **[Ubuntu][rel-u] / [Debian][rel-d]** | **Git** | **Python** | **SSH** |
 
 
 [contact]: ../README.md#contact
 [rel-d]: https://en.wikipedia.org/wiki/Debian_version_history
 [rel-g]: https://en.wikipedia.org/wiki/Git#Releases
+[rel-o]: https://www.openssh.com/releasenotes.html
 [rel-p]: https://en.wikipedia.org/wiki/History_of_Python#Table_of_versions
 [rel-u]: https://en.wikipedia.org/wiki/Ubuntu_version_history#Table_of_versions
 [example announcement]: https://groups.google.com/d/topic/repo-discuss/UGBNismWo1M/discussion

fetch.py

@@ -0,0 +1,45 @@
+# Copyright (C) 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This module contains functions used to fetch files from various sources."""
+
+import subprocess
+import sys
+from urllib.parse import urlparse
+from urllib.request import urlopen
+
+
+def fetch_file(url, verbose=False):
+  """Fetch a file from the specified source using the appropriate protocol.
+
+  Returns:
+    The contents of the file as bytes.
+  """
+  scheme = urlparse(url).scheme
+  if scheme == 'gs':
+    cmd = ['gsutil', 'cat', url]
+    try:
+      result = subprocess.run(
+          cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+          check=True)
+      if result.stderr and verbose:
+        print('warning: non-fatal error running "gsutil": %s' % result.stderr,
+              file=sys.stderr)
+      return result.stdout
+    except subprocess.CalledProcessError as e:
+      print('fatal: error running "gsutil": %s' % e.stderr,
+            file=sys.stderr)
+    sys.exit(1)
+  with urlopen(url) as f:
+    return f.read()

git_config.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import contextlib
+import datetime
 import errno
 from http.client import HTTPException
 import json
@@ -30,6 +31,10 @@ from repo_trace import Trace
 from git_command import GitCommand
 from git_refs import R_CHANGES, R_HEADS, R_TAGS
 
+# Prefix that is prepended to all the keys of SyncAnalysisState's data
+# that is saved in the config.
+SYNC_STATE_PREFIX = 'repo.syncstate.'
+
 ID_RE = re.compile(r'^[0-9a-f]{40}$')
 
 REVIEW_CACHE = dict()
@@ -99,6 +104,10 @@ class GitConfig(object):
           os.path.dirname(self.file),
           '.repo_' + os.path.basename(self.file) + '.json')
 
+  def ClearCache(self):
+    """Clear the in-memory cache of config."""
+    self._cache_dict = None
+
   def Has(self, name, include_defaults=True):
     """Return true if this configuration file has the key.
     """
@@ -262,6 +271,22 @@ class GitConfig(object):
       self._branches[b.name] = b
     return b
 
+  def GetSyncAnalysisStateData(self):
+    """Returns data to be logged for the analysis of sync performance."""
+    return {k: v for k, v in self.DumpConfigDict().items() if k.startswith(SYNC_STATE_PREFIX)}
+
+  def UpdateSyncAnalysisState(self, options, superproject_logging_data):
+    """Update Config's SYNC_STATE_PREFIX* data with the latest sync data.
+
+    Args:
+      options: Options passed to sync returned from optparse. See _Options().
+      superproject_logging_data: A dictionary of superproject data that is to be logged.
+
+    Returns:
+      SyncAnalysisState object.
+    """
+    return SyncAnalysisState(self, options, superproject_logging_data)
+
   def GetSubSections(self, section):
     """List all subsection names matching $section.*.*
     """
@@ -328,7 +353,7 @@ class GitConfig(object):
       with open(self._json) as fd:
         return json.load(fd)
     except (IOError, ValueError):
-      platform_utils.remove(self._json)
+      platform_utils.remove(self._json, missing_ok=True)
       return None
 
   def _SaveJson(self, cache):
@@ -336,8 +361,7 @@ class GitConfig(object):
       with open(self._json, 'w') as fd:
         json.dump(cache, fd, indent=2)
     except (IOError, TypeError):
-      if os.path.exists(self._json):
+      platform_utils.remove(self._json, missing_ok=True)
-        platform_utils.remove(self._json)
 
   def _ReadGit(self):
     """
@@ -347,9 +371,10 @@ class GitConfig(object):
 
     """
     c = {}
-    d = self._do('--null', '--list')
+    if not os.path.exists(self.file):
-    if d is None:
       return c
+
+    d = self._do('--null', '--list')
     for line in d.rstrip('\0').split('\0'):
       if '\n' in line:
         key, val = line.split('\n', 1)
@@ -378,7 +403,7 @@ class GitConfig(object):
     if p.Wait() == 0:
       return p.stdout
     else:
-      GitError('git config %s: %s' % (str(args), p.stderr))
+      raise GitError('git config %s: %s' % (str(args), p.stderr))
 
 
 class RepoConfig(GitConfig):
@@ -717,3 +742,70 @@ class Branch(object):
   def _Get(self, key, all_keys=False):
     key = 'branch.%s.%s' % (self.name, key)
     return self._config.GetString(key, all_keys=all_keys)
+
+
+class SyncAnalysisState:
+  """Configuration options related to logging of sync state for analysis.
+
+  This object is versioned.
+  """
+  def __init__(self, config, options, superproject_logging_data):
+    """Initializes SyncAnalysisState.
+
+    Saves the following data into the |config| object.
+    - sys.argv, options, superproject's logging data.
+    - repo.*, branch.* and remote.* parameters from config object.
+    - Current time as synctime.
+    - Version number of the object.
+
+    All the keys saved by this object are prepended with SYNC_STATE_PREFIX.
+
+    Args:
+      config: GitConfig object to store all options.
+      options: Options passed to sync returned from optparse. See _Options().
+      superproject_logging_data: A dictionary of superproject data that is to be logged.
+    """
+    self._config = config
+    now = datetime.datetime.utcnow()
+    self._Set('main.synctime', now.isoformat() + 'Z')
+    self._Set('main.version', '1')
+    self._Set('sys.argv', sys.argv)
+    for key, value in superproject_logging_data.items():
+      self._Set(f'superproject.{key}', value)
+    for key, value in options.__dict__.items():
+      self._Set(f'options.{key}', value)
+    config_items = config.DumpConfigDict().items()
+    EXTRACT_NAMESPACES = {'repo', 'branch', 'remote'}
+    self._SetDictionary({k: v for k, v in config_items
+                         if not k.startswith(SYNC_STATE_PREFIX) and
+                         k.split('.', 1)[0] in EXTRACT_NAMESPACES})
+
+  def _SetDictionary(self, data):
+    """Save all key/value pairs of |data| dictionary.
+
+    Args:
+      data: A dictionary whose key/value are to be saved.
+    """
+    for key, value in data.items():
+      self._Set(key, value)
+
+  def _Set(self, key, value):
+    """Set the |value| for a |key| in the |_config| member.
+
+    |key| is prepended with the value of SYNC_STATE_PREFIX constant.
+
+    Args:
+      key: Name of the key.
+      value: |value| could be of any type. If it is 'bool', it will be saved
+             as a Boolean and for all other types, it will be saved as a String.
+    """
+    if value is None:
+      return
+    sync_key = f'{SYNC_STATE_PREFIX}{key}'
+    sync_key = sync_key.replace('_', '')
+    if isinstance(value, str):
+      self._config.SetString(sync_key, value)
+    elif isinstance(value, bool):
+      self._config.SetBoolean(sync_key, value)
+    else:
+      self._config.SetString(sync_key, str(value))

git_superproject.py

@@ -59,7 +59,7 @@ class CommitIdsResult(NamedTuple):
 class UpdateProjectsResult(NamedTuple):
   """Return the overriding manifest file and whether caller should exit."""
 
-  # Path name of the overriding manfiest file if successful, otherwise None.
+  # Path name of the overriding manifest file if successful, otherwise None.
   manifest_path: str
   # Whether the caller should exit.
   fatal: bool
@@ -73,7 +73,7 @@ class Superproject(object):
   is a dictionary with project/commit id entries.
   """
   def __init__(self, manifest, repodir, git_event_log,
-               superproject_dir='exp-superproject', quiet=False):
+               superproject_dir='exp-superproject', quiet=False, print_messages=False):
     """Initializes superproject.
 
     Args:
@@ -83,12 +83,14 @@ class Superproject(object):
       git_event_log: A git trace2 event log to log events.
       superproject_dir: Relative path under |repodir| to checkout superproject.
       quiet:  If True then only print the progress messages.
+      print_messages: if True then print error/warning messages.
     """
     self._project_commit_ids = None
     self._manifest = manifest
     self._git_event_log = git_event_log
     self._quiet = quiet
-    self._branch = self._GetBranch()
+    self._print_messages = print_messages
+    self._branch = manifest.branch
     self._repodir = os.path.abspath(repodir)
     self._superproject_dir = superproject_dir
     self._superproject_path = os.path.join(self._repodir, superproject_dir)
@@ -96,8 +98,12 @@ class Superproject(object):
                                        _SUPERPROJECT_MANIFEST_NAME)
     git_name = ''
     if self._manifest.superproject:
-      remote_name = self._manifest.superproject['remote'].name
+      remote = self._manifest.superproject['remote']
-      git_name = hashlib.md5(remote_name.encode('utf8')).hexdigest() + '-'
+      git_name = hashlib.md5(remote.name.encode('utf8')).hexdigest() + '-'
+      self._branch = self._manifest.superproject['revision']
+      self._remote_url = remote.url
+    else:
+      self._remote_url = None
     self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME
     self._work_git = os.path.join(self._superproject_path, self._work_git_name)
 
@@ -106,21 +112,28 @@ class Superproject(object):
     """Returns a dictionary of projects and their commit ids."""
     return self._project_commit_ids
 
-  def _GetBranch(self):
+  @property
-    """Returns the branch name for getting the approved manifest."""
+  def manifest_path(self):
-    p = self._manifest.manifestProject
+    """Returns the manifest path if the path exists or None."""
-    b = p.GetBranch(p.CurrentBranch)
+    return self._manifest_path if os.path.exists(self._manifest_path) else None
-    if not b:
+
-      return None
+  def _LogMessage(self, message):
-    branch = b.merge
+    """Logs message to stderr and _git_event_log."""
-    if branch and branch.startswith(R_HEADS):
+    if self._print_messages:
-      branch = branch[len(R_HEADS):]
+      print(message, file=sys.stderr)
-    return branch
+    self._git_event_log.ErrorEvent(message, f'{message}')
+
+  def _LogMessagePrefix(self):
+    """Returns the prefix string to be logged in each log message"""
+    return f'repo superproject branch: {self._branch} url: {self._remote_url}'
 
   def _LogError(self, message):
-    """Logs message to stderr and _git_event_log."""
+    """Logs error message to stderr and _git_event_log."""
-    print(message, file=sys.stderr)
+    self._LogMessage(f'{self._LogMessagePrefix()} error: {message}')
-    self._git_event_log.ErrorEvent(message, '')
+
+  def _LogWarning(self, message):
+    """Logs warning message to stderr and _git_event_log."""
+    self._LogMessage(f'{self._LogMessagePrefix()} warning: {message}')
 
   def _Init(self):
     """Sets up a local Git repository to get a copy of a superproject.
@@ -141,27 +154,25 @@ class Superproject(object):
                    capture_stderr=True)
     retval = p.Wait()
     if retval:
-      self._LogError(f'repo: error: git init call failed, command: git {cmd}, '
+      self._LogWarning(f'git init call failed, command: git {cmd}, '
-                     f'return code: {retval}, stderr: {p.stderr}')
+                       f'return code: {retval}, stderr: {p.stderr}')
       return False
     return True
 
-  def _Fetch(self, url):
+  def _Fetch(self):
-    """Fetches a local copy of a superproject for the manifest based on url.
+    """Fetches a local copy of a superproject for the manifest based on |_remote_url|.
-
-    Args:
-      url: superproject's url.
 
     Returns:
       True if fetch is successful, or False.
     """
     if not os.path.exists(self._work_git):
-      self._LogError(f'git fetch missing directory: {self._work_git}')
+      self._LogWarning(f'git fetch missing directory: {self._work_git}')
       return False
     if not git_require((2, 28, 0)):
-      print('superproject requires a git version 2.28 or later', file=sys.stderr)
+      self._LogWarning('superproject requires a git version 2.28 or later')
       return False
-    cmd = ['fetch', url, '--depth', '1', '--force', '--no-tags', '--filter', 'blob:none']
+    cmd = ['fetch', self._remote_url, '--depth', '1', '--force', '--no-tags',
+           '--filter', 'blob:none']
     if self._branch:
       cmd += [self._branch + ':' + self._branch]
     p = GitCommand(None,
@@ -171,8 +182,8 @@ class Superproject(object):
                    capture_stderr=True)
     retval = p.Wait()
     if retval:
-      self._LogError(f'repo: error: git fetch call failed, command: git {cmd}, '
+      self._LogWarning(f'git fetch call failed, command: git {cmd}, '
-                     f'return code: {retval}, stderr: {p.stderr}')
+                       f'return code: {retval}, stderr: {p.stderr}')
       return False
     return True
 
@@ -185,7 +196,7 @@ class Superproject(object):
       data: data returned from 'git ls-tree ...' instead of None.
     """
     if not os.path.exists(self._work_git):
-      self._LogError(f'git ls-tree missing directory: {self._work_git}')
+      self._LogWarning(f'git ls-tree missing directory: {self._work_git}')
       return None
     data = None
     branch = 'HEAD' if not self._branch else self._branch
@@ -200,8 +211,8 @@ class Superproject(object):
     if retval == 0:
       data = p.stdout
     else:
-      self._LogError(f'repo: error: git ls-tree call failed, command: git {cmd}, '
+      self._LogWarning(f'git ls-tree call failed, command: git {cmd}, '
-                     f'return code: {retval}, stderr: {p.stderr}')
+                       f'return code: {retval}, stderr: {p.stderr}')
     return data
 
   def Sync(self):
@@ -210,24 +221,22 @@ class Superproject(object):
     Returns:
       SyncResult
     """
-    print('NOTICE: --use-superproject is in beta; report any issues to the '
-          'address described in `repo version`', file=sys.stderr)
-
     if not self._manifest.superproject:
-      self._LogError(f'repo error: superproject tag is not defined in manifest: '
+      self._LogWarning(f'superproject tag is not defined in manifest: '
-                     f'{self._manifest.manifestFile}')
+                       f'{self._manifest.manifestFile}')
       return SyncResult(False, False)
 
+    print('NOTICE: --use-superproject is in beta; report any issues to the '
+          'address described in `repo version`', file=sys.stderr)
     should_exit = True
-    url = self._manifest.superproject['remote'].url
+    if not self._remote_url:
-    if not url:
+      self._LogWarning(f'superproject URL is not defined in manifest: '
-      self._LogError(f'repo error: superproject URL is not defined in manifest: '
+                       f'{self._manifest.manifestFile}')
-                     f'{self._manifest.manifestFile}')
       return SyncResult(False, should_exit)
 
     if not self._Init():
       return SyncResult(False, should_exit)
-    if not self._Fetch(url):
+    if not self._Fetch():
       return SyncResult(False, should_exit)
     if not self._quiet:
       print('%s: Initial setup for superproject completed.' % self._work_git)
@@ -245,8 +254,8 @@ class Superproject(object):
 
     data = self._LsTree()
     if not data:
-      print('warning: git ls-tree failed to return data for superproject',
+      self._LogWarning(f'git ls-tree failed to return data for manifest: '
-            file=sys.stderr)
+                       f'{self._manifest.manifestFile}')
       return CommitIdsResult(None, True)
 
     # Parse lines like the following to select lines starting with '160000' and
@@ -265,14 +274,14 @@ class Superproject(object):
     self._project_commit_ids = commit_ids
     return CommitIdsResult(commit_ids, False)
 
-  def _WriteManfiestFile(self):
+  def _WriteManifestFile(self):
     """Writes manifest to a file.
 
     Returns:
       manifest_path: Path name of the file into which manifest is written instead of None.
     """
     if not os.path.exists(self._superproject_path):
-      self._LogError(f'error: missing superproject directory: {self._superproject_path}')
+      self._LogWarning(f'missing superproject directory: {self._superproject_path}')
       return None
     manifest_str = self._manifest.ToXml(groups=self._manifest.GetGroupsStr()).toxml()
     manifest_path = self._manifest_path
@@ -280,7 +289,7 @@ class Superproject(object):
       with open(manifest_path, 'w', encoding='utf-8') as fp:
         fp.write(manifest_str)
     except IOError as e:
-      self._LogError(f'error: cannot write manifest to : {manifest_path} {e}')
+      self._LogError(f'cannot write manifest to : {manifest_path} {e}')
       return None
     return manifest_path
 
@@ -316,7 +325,6 @@ class Superproject(object):
     commit_ids_result = self._GetAllProjectsCommitIds()
     commit_ids = commit_ids_result.commit_ids
     if not commit_ids:
-      print('warning: Cannot get project commit ids from manifest', file=sys.stderr)
       return UpdateProjectsResult(None, commit_ids_result.fatal)
 
     projects_missing_commit_ids = []
@@ -331,15 +339,15 @@ class Superproject(object):
     # If superproject doesn't have a commit id for a project, then report an
     # error event and continue as if do not use superproject is specified.
     if projects_missing_commit_ids:
-      self._LogError(f'error: please file a bug using {self._manifest.contactinfo.bugurl} '
+      self._LogWarning(f'please file a bug using {self._manifest.contactinfo.bugurl} '
-                     f'to report missing commit_ids for: {projects_missing_commit_ids}')
+                       f'to report missing commit_ids for: {projects_missing_commit_ids}')
       return UpdateProjectsResult(None, False)
 
     for project in projects:
       if not self._SkipUpdatingProjectRevisionId(project):
         project.SetRevisionId(commit_ids.get(project.relpath))
 
-    manifest_path = self._WriteManfiestFile()
+    manifest_path = self._WriteManifestFile()
     return UpdateProjectsResult(manifest_path, False)
 
 
@@ -347,67 +355,51 @@ class Superproject(object):
 def _UseSuperprojectFromConfiguration():
   """Returns the user choice of whether to use superproject."""
   user_cfg = RepoConfig.ForUser()
-  system_cfg = RepoConfig.ForSystem()
   time_now = int(time.time())
 
   user_value = user_cfg.GetBoolean('repo.superprojectChoice')
   if user_value is not None:
     user_expiration = user_cfg.GetInt('repo.superprojectChoiceExpire')
-    if user_expiration is not None and (user_expiration <= 0 or user_expiration >= time_now):
+    if user_expiration is None or user_expiration <= 0 or user_expiration >= time_now:
       # TODO(b/190688390) - Remove prompt when we are comfortable with the new
       # default value.
-      print(('You are currently enrolled in Git submodules experiment '
+      if user_value:
-             '(go/android-submodules-quickstart).  Use --no-use-superproject '
+        print(('You are currently enrolled in Git submodules experiment '
-             'to override.\n'), file=sys.stderr)
+               '(go/android-submodules-quickstart).  Use --no-use-superproject '
-    return user_value
+               'to override.\n'), file=sys.stderr)
+      else:
+        print(('You are not currently enrolled in Git submodules experiment '
+               '(go/android-submodules-quickstart).  Use --use-superproject '
+               'to override.\n'), file=sys.stderr)
+      return user_value
 
   # We don't have an unexpired choice, ask for one.
+  system_cfg = RepoConfig.ForSystem()
   system_value = system_cfg.GetBoolean('repo.superprojectChoice')
   if system_value:
     # The system configuration is proposing that we should enable the
-    # use of superproject. Present this to user for confirmation if we
+    # use of superproject. Treat the user as enrolled for two weeks.
-    # are on a TTY, or, when we are not on a TTY, accept the system
-    # default for this time only.
     #
     # TODO(b/190688390) - Remove prompt when we are comfortable with the new
     # default value.
-    prompt = ('Repo can now use Git submodules (go/android-submodules-quickstart) '
+    userchoice = True
-              'instead of manifests to represent the state of the Android '
+    time_choiceexpire = time_now + (86400 * 14)
-              'superproject, which results in faster syncs and better atomicity.\n\n')
+    user_cfg.SetString('repo.superprojectChoiceExpire', str(time_choiceexpire))
-    if sys.stdout.isatty():
+    user_cfg.SetBoolean('repo.superprojectChoice', userchoice)
-      prompt += 'Would you like to opt in for two weeks (y/N)? '
+    print('You are automatically enrolled in Git submodules experiment '
-      response = input(prompt).lower()
+          '(go/android-submodules-quickstart) for another two weeks.\n',
-      time_choiceexpire = time_now + (86400 * 14)
+          file=sys.stderr)
-      if response in ('y', 'yes'):
+    return True
-        userchoice = True
-      elif response in ('a', 'always'):
-        userchoice = True
-        time_choiceexpire = 0
-      elif response == 'never':
-        userchoice = False
-        time_choiceexpire = 0
-      elif response in ('n', 'no'):
-        userchoice = False
-      else:
-        # Unrecognized user response, assume the intention was no, but
-        # only for 2 hours instead of 2 weeks to balance between not
-        # being overly pushy while still retain the opportunity to
-        # enroll.
-        userchoice = False
-        time_choiceexpire = time_now + 7200
-
-      user_cfg.SetString('repo.superprojectChoiceExpire', str(time_choiceexpire))
-      user_cfg.SetBoolean('repo.superprojectChoice', userchoice)
-
-      return userchoice
-    else:
-      print('Accepting once since we are not on a TTY', file=sys.stderr)
-      return True
 
   # For all other cases, we would not use superproject by default.
   return False
 
 
+def PrintMessages(opt, manifest):
+  """Returns a boolean if error/warning messages are to be printed."""
+  return opt.use_superproject is not None or manifest.superproject
+
+
 def UseSuperproject(opt, manifest):
   """Returns a boolean if use-superproject option is enabled."""
 
@@ -418,4 +410,6 @@ def UseSuperproject(opt, manifest):
     if client_value is not None:
       return client_value
     else:
+      if not manifest.superproject:
+        return False
       return _UseSuperprojectFromConfiguration()

git_trace2_event_log.py

@@ -144,6 +144,19 @@ class EventLog(object):
     command_event['subcommands'] = subcommands
     self._log.append(command_event)
 
+  def LogConfigEvents(self, config, event_dict_name):
+    """Append a |event_dict_name| event for each config key in |config|.
+
+    Args:
+      config: Configuration dictionary.
+      event_dict_name: Name of the event dictionary for items to be logged under.
+    """
+    for param, value in config.items():
+      event = self._CreateEventDict(event_dict_name)
+      event['param'] = param
+      event['value'] = value
+      self._log.append(event)
+
   def DefParamRepoEvents(self, config):
     """Append a 'def_param' event for each repo.* config key to the current log.
 
@@ -152,12 +165,27 @@ class EventLog(object):
     """
     # Only output the repo.* config parameters.
     repo_config = {k: v for k, v in config.items() if k.startswith('repo.')}
+    self.LogConfigEvents(repo_config, 'def_param')
 
-    for param, value in repo_config.items():
+  def GetDataEventName(self, value):
-      def_param_event = self._CreateEventDict('def_param')
+    """Returns 'data-json' if the value is an array else returns 'data'."""
-      def_param_event['param'] = param
+    return 'data-json' if value[0] == '[' and value[-1] == ']' else 'data'
-      def_param_event['value'] = value
+
-      self._log.append(def_param_event)
+  def LogDataConfigEvents(self, config, prefix):
+    """Append a 'data' event for each config key/value in |config| to the current log.
+
+    For each keyX and valueX of the config, "key" field of the event is '|prefix|/keyX'
+    and the "value" of the "key" field is valueX.
+
+    Args:
+      config: Configuration dictionary.
+      prefix: Prefix for each key that is logged.
+    """
+    for key, value in config.items():
+      event = self._CreateEventDict(self.GetDataEventName(value))
+      event['key'] = f'{prefix}/{key}'
+      event['value'] = value
+      self._log.append(event)
 
   def ErrorEvent(self, msg, fmt):
     """Append a 'error' event to the current log."""

main.py

@@ -95,6 +95,8 @@ global_options = optparse.OptionParser(
     add_help_option=False)
 global_options.add_option('-h', '--help', action='store_true',
                           help='show this help message and exit')
+global_options.add_option('--help-all', action='store_true',
+                          help='show this help message with all subcommands and exit')
 global_options.add_option('-p', '--paginate',
                           dest='pager', action='store_true',
                           help='display command output in the pager')
@@ -116,6 +118,10 @@ global_options.add_option('--time',
 global_options.add_option('--version',
                           dest='show_version', action='store_true',
                           help='display this version of repo')
+global_options.add_option('--show-toplevel',
+                          action='store_true',
+                          help='display the path of the top-level directory of '
+                               'the repo client checkout')
 global_options.add_option('--event-log',
                           dest='event_log', action='store',
                           help='filename of event log to append timeline to')
@@ -128,34 +134,40 @@ class _Repo(object):
     self.repodir = repodir
     self.commands = all_commands
 
+  def _PrintHelp(self, short: bool = False, all_commands: bool = False):
+    """Show --help screen."""
+    global_options.print_help()
+    print()
+    if short:
+      commands = ' '.join(sorted(self.commands))
+      wrapped_commands = textwrap.wrap(commands, width=77)
+      print('Available commands:\n  %s' % ('\n  '.join(wrapped_commands),))
+      print('\nRun `repo help <command>` for command-specific details.')
+      print('Bug reports:', Wrapper().BUG_URL)
+    else:
+      cmd = self.commands['help']()
+      if all_commands:
+        cmd.PrintAllCommandsBody()
+      else:
+        cmd.PrintCommonCommandsBody()
+
   def _ParseArgs(self, argv):
     """Parse the main `repo` command line options."""
-    name = None
+    for i, arg in enumerate(argv):
-    glob = []
+      if not arg.startswith('-'):
-
+        name = arg
-    for i in range(len(argv)):
+        glob = argv[:i]
-      if not argv[i].startswith('-'):
-        name = argv[i]
-        if i > 0:
-          glob = argv[:i]
         argv = argv[i + 1:]
         break
-    if not name:
+    else:
+      name = None
       glob = argv
-      name = 'help'
       argv = []
     gopts, _gargs = global_options.parse_args(glob)
 
-    name, alias_args = self._ExpandAlias(name)
+    if name:
-    argv = alias_args + argv
+      name, alias_args = self._ExpandAlias(name)
-
+      argv = alias_args + argv
-    if gopts.help:
-      global_options.print_help()
-      commands = ' '.join(sorted(self.commands))
-      wrapped_commands = textwrap.wrap(commands, width=77)
-      print('\nAvailable commands:\n  %s' % ('\n  '.join(wrapped_commands),))
-      print('\nRun `repo help <command>` for command-specific details.')
-      global_options.exit()
 
     return (name, gopts, argv)
 
@@ -186,12 +198,21 @@ class _Repo(object):
 
     if gopts.trace:
       SetTrace()
-    if gopts.show_version:
+
-      if name == 'help':
+    # Handle options that terminate quickly first.
-        name = 'version'
+    if gopts.help or gopts.help_all:
-      else:
+      self._PrintHelp(short=False, all_commands=gopts.help_all)
-        print('fatal: invalid usage of --version', file=sys.stderr)
+      return 0
-        return 1
+    elif gopts.show_version:
+      # Always allow global --version regardless of subcommand validity.
+      name = 'version'
+    elif gopts.show_toplevel:
+      print(os.path.dirname(self.repodir))
+      return 0
+    elif not name:
+      # No subcommand specified, so show the help/subcommand.
+      self._PrintHelp(short=True)
+      return 1
 
     SetDefaultColoring(gopts.color)
 

man/repo-abandon.1

@@ -20,7 +20,8 @@ It is equivalent to "git branch \fB\-D\fR <branchname>".
 show this help message and exit
 .TP
 \fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
-number of jobs to run in parallel (default: 4)
+number of jobs to run in parallel (default: based on
+number of CPU cores)
 .TP
 \fB\-\-all\fR
 delete all branches in all projects

man/repo-branches.1

@@ -46,7 +46,8 @@ is shown, then the branch appears in all projects.
 show this help message and exit
 .TP
 \fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
-number of jobs to run in parallel (default: 4)
+number of jobs to run in parallel (default: based on
+number of CPU cores)
 .SS Logging options:
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR

man/repo-checkout.1

@@ -15,7 +15,8 @@ Checkout a branch for development
 show this help message and exit
 .TP
 \fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
-number of jobs to run in parallel (default: 4)
+number of jobs to run in parallel (default: based on
+number of CPU cores)
 .SS Logging options:
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR

man/repo-diff.1

@@ -19,7 +19,8 @@ to the Unix 'patch' command.
 show this help message and exit
 .TP
 \fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
-number of jobs to run in parallel (default: 4)
+number of jobs to run in parallel (default: based on
+number of CPU cores)
 .TP
 \fB\-u\fR, \fB\-\-absolute\fR
 paths are relative to the repository root

man/repo-forall.1

@@ -17,7 +17,8 @@ repo forall \fB\-r\fR str1 [str2] ... \fB\-c\fR <command> [<arg>...]
 show this help message and exit
 .TP
 \fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
-number of jobs to run in parallel (default: 4)
+number of jobs to run in parallel (default: based on
+number of CPU cores)
 .TP
 \fB\-r\fR, \fB\-\-regex\fR
 execute the command only on projects matching regex or

man/repo-gitc-init.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo gitc-init" "Repo Manual"
+.TH REPO "1" "November 2021" "repo gitc-init" "Repo Manual"
 .SH NAME
 repo \- repo gitc-init - manual page for repo gitc-init
 .SH SYNOPSIS
@@ -41,6 +41,10 @@ platform group [auto|all|none|linux|darwin|...]
 .TP
 \fB\-\-submodules\fR
 sync any submodules associated with the manifest repo
+.TP
+\fB\-\-standalone\-manifest\fR
+download the manifest as a static file rather then
+create a git checkout of the manifest repo
 .SS Manifest (only) checkout options:
 .TP
 \fB\-\-current\-branch\fR
@@ -92,7 +96,8 @@ filter for use with \fB\-\-partial\-clone\fR [default:
 blob:none]
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects

man/repo-grep.1

@@ -15,7 +15,8 @@ Print lines matching a pattern
 show this help message and exit
 .TP
 \fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
-number of jobs to run in parallel (default: 4)
+number of jobs to run in parallel (default: based on
+number of CPU cores)
 .SS Logging options:
 .TP
 \fB\-\-verbose\fR

man/repo-init.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo init" "Repo Manual"
+.TH REPO "1" "November 2021" "repo init" "Repo Manual"
 .SH NAME
 repo \- repo init - manual page for repo init
 .SH SYNOPSIS
@@ -41,6 +41,10 @@ platform group [auto|all|none|linux|darwin|...]
 .TP
 \fB\-\-submodules\fR
 sync any submodules associated with the manifest repo
+.TP
+\fB\-\-standalone\-manifest\fR
+download the manifest as a static file rather then
+create a git checkout of the manifest repo
 .SS Manifest (only) checkout options:
 .TP
 \fB\-c\fR, \fB\-\-current\-branch\fR
@@ -92,7 +96,8 @@ filter for use with \fB\-\-partial\-clone\fR [default:
 blob:none]
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects
@@ -137,6 +142,12 @@ equivalent to using \fB\-b\fR HEAD.
 The optional \fB\-m\fR argument can be used to specify an alternate manifest to be
 used. If no manifest is specified, the manifest default.xml will be used.
 .PP
+If the \fB\-\-standalone\-manifest\fR argument is set, the manifest will be downloaded
+directly from the specified \fB\-\-manifest\-url\fR as a static file (rather than setting
+up a manifest git checkout). With \fB\-\-standalone\-manifest\fR, the manifest will be
+fully static and will not be re\-downloaded during subsesquent `repo init` and
+`repo sync` calls.
+.PP
 The \fB\-\-reference\fR option can be used to point to a directory that has the content
 of a \fB\-\-mirror\fR sync. This will make the working directory use as much data as
 possible from the local reference directory when fetching from the server. This

man/repo-list.1

@@ -27,15 +27,19 @@ project is in
 \fB\-a\fR, \fB\-\-all\fR
 show projects regardless of checkout state
 .TP
-\fB\-f\fR, \fB\-\-fullpath\fR
-display the full work tree path instead of the
-relative path
-.TP
 \fB\-n\fR, \fB\-\-name\-only\fR
 display only the name of the repository
 .TP
 \fB\-p\fR, \fB\-\-path\-only\fR
 display only the path of the repository
+.TP
+\fB\-f\fR, \fB\-\-fullpath\fR
+display the full work tree path instead of the
+relative path
+.TP
+\fB\-\-relative\-to\fR=\fI\,PATH\/\fR
+display paths relative to this one (default: top of
+repo client checkout)
 .SS Logging options:
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR

man/repo-manifest.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo manifest" "Repo Manual"
+.TH REPO "1" "November 2021" "repo manifest" "Repo Manual"
 .SH NAME
 repo \- repo manifest - manual page for repo manifest
 .SH SYNOPSIS
@@ -36,6 +36,9 @@ output manifest in JSON format (experimental)
 \fB\-\-pretty\fR
 format output for humans to read
 .TP
+\fB\-\-no\-local\-manifests\fR
+ignore local manifests
+.TP
 \fB\-o\fR \-|NAME.xml, \fB\-\-output\-file\fR=\fI\,\-\/\fR|NAME.xml
 file to save the manifest to
 .SS Logging options:
@@ -95,7 +98,7 @@ include*)>
 .IP
 <!ELEMENT notice (#PCDATA)>
 .IP
-<!ELEMENT remote EMPTY>
+<!ELEMENT remote (annotation*)>
 <!ATTLIST remote name         ID    #REQUIRED>
 <!ATTLIST remote alias        CDATA #IMPLIED>
 <!ATTLIST remote fetch        CDATA #REQUIRED>
@@ -158,6 +161,7 @@ CDATA #IMPLIED>
 <!ELEMENT extend\-project EMPTY>
 <!ATTLIST extend\-project name CDATA #REQUIRED>
 <!ATTLIST extend\-project path CDATA #IMPLIED>
+<!ATTLIST extend\-project dest\-path CDATA #IMPLIED>
 <!ATTLIST extend\-project groups CDATA #IMPLIED>
 <!ATTLIST extend\-project revision CDATA #IMPLIED>
 <!ATTLIST extend\-project remote CDATA #IMPLIED>
@@ -171,8 +175,9 @@ CDATA #IMPLIED>
 <!ATTLIST repo\-hooks enabled\-list CDATA #REQUIRED>
 .IP
 <!ELEMENT superproject EMPTY>
-<!ATTLIST superproject name    CDATA #REQUIRED>
+<!ATTLIST superproject name     CDATA #REQUIRED>
-<!ATTLIST superproject remote  IDREF #IMPLIED>
+<!ATTLIST superproject remote   IDREF #IMPLIED>
+<!ATTLIST superproject revision CDATA #IMPLIED>
 .IP
 <!ELEMENT contactinfo EMPTY>
 <!ATTLIST contactinfo bugurl  CDATA #REQUIRED>
@@ -382,6 +387,11 @@ original manifest.
 Attribute `path`: If specified, limit the change to projects checked out at the
 specified path, rather than all projects with the given name.
 .PP
+Attribute `dest\-path`: If specified, a path relative to the top directory of the
+repo client where the Git working directory for this project should be placed.
+This is used to move a project in the checkout by overriding the existing `path`
+setting.
+.PP
 Attribute `groups`: List of additional groups to which this project belongs.
 Same syntax as the corresponding element of `project`.
 .PP
@@ -393,13 +403,13 @@ Same syntax as the corresponding element of `project`.
 .PP
 Element annotation
 .PP
-Zero or more annotation elements may be specified as children of a project
+Zero or more annotation elements may be specified as children of a project or
-element. Each element describes a name\-value pair that will be exported into
+remote element. Each element describes a name\-value pair. For projects, this
-each project's environment during a 'forall' command, prefixed with REPO__. In
+name\-value pair will be exported into each project's environment during a
-addition, there is an optional attribute "keep" which accepts the case
+\&'forall' command, prefixed with `REPO__`. In addition, there is an optional
-insensitive values "true" (default) or "false". This attribute determines
+attribute "keep" which accepts the case insensitive values "true" (default) or
-whether or not the annotation will be kept when exported with the manifest
+"false". This attribute determines whether or not the annotation will be kept
-subcommand.
+when exported with the manifest subcommand.
 .PP
 Element copyfile
 .PP
@@ -474,6 +484,10 @@ project](#element\-project) for more information.
 Attribute `remote`: Name of a previously defined remote element. If not supplied
 the remote given by the default element is used.
 .PP
+Attribute `revision`: Name of the Git branch the manifest wants to track for
+this superproject. If not supplied the revision given by the remote element is
+used if applicable, else the default element is used.
+.PP
 Element contactinfo
 .PP
 *** *Note*: This is currently a WIP. ***

man/repo-prune.1

@@ -15,7 +15,8 @@ Prune (delete) already merged topics
 show this help message and exit
 .TP
 \fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
-number of jobs to run in parallel (default: 4)
+number of jobs to run in parallel (default: based on
+number of CPU cores)
 .SS Logging options:
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR

man/repo-smartsync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo smartsync" "Repo Manual"
+.TH REPO "1" "November 2021" "repo smartsync" "Repo Manual"
 .SH NAME
 repo \- repo smartsync - manual page for repo smartsync
 .SH SYNOPSIS
@@ -15,7 +15,8 @@ Update working tree to the latest known good revision
 show this help message and exit
 .TP
 \fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
-number of jobs to run in parallel (default: 1)
+number of jobs to run in parallel (default: based on
+number of CPU cores)
 .TP
 \fB\-\-jobs\-network\fR=\fI\,JOBS\/\fR
 number of network jobs to run in parallel (defaults to
@@ -79,7 +80,8 @@ password to authenticate with the manifest server
 fetch submodules from server
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects
@@ -88,7 +90,7 @@ disable use of manifest superprojects
 fetch tags
 .TP
 \fB\-\-no\-tags\fR
-don't fetch tags
+don't fetch tags (default)
 .TP
 \fB\-\-optimized\-fetch\fR
 only fetch projects fixed to sha1 if revision does not
@@ -99,6 +101,10 @@ number of times to retry fetches on transient errors
 .TP
 \fB\-\-prune\fR
 delete refs that no longer exist on the remote
+(default)
+.TP
+\fB\-\-no\-prune\fR
+do not delete refs that no longer exist on the remote
 .SS Logging options:
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR

man/repo-start.1

@@ -15,7 +15,8 @@ Start a new branch for development
 show this help message and exit
 .TP
 \fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
-number of jobs to run in parallel (default: 4)
+number of jobs to run in parallel (default: based on
+number of CPU cores)
 .TP
 \fB\-\-all\fR
 begin branch in all projects

man/repo-status.1

@@ -15,7 +15,8 @@ Show the working tree status
 show this help message and exit
 .TP
 \fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
-number of jobs to run in parallel (default: 4)
+number of jobs to run in parallel (default: based on
+number of CPU cores)
 .TP
 \fB\-o\fR, \fB\-\-orphans\fR
 include objects in working directory outside of repo

man/repo-sync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo sync" "Repo Manual"
+.TH REPO "1" "November 2021" "repo sync" "Repo Manual"
 .SH NAME
 repo \- repo sync - manual page for repo sync
 .SH SYNOPSIS
@@ -15,7 +15,8 @@ Update working tree to the latest revision
 show this help message and exit
 .TP
 \fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
-number of jobs to run in parallel (default: 1)
+number of jobs to run in parallel (default: based on
+number of CPU cores)
 .TP
 \fB\-\-jobs\-network\fR=\fI\,JOBS\/\fR
 number of network jobs to run in parallel (defaults to
@@ -79,7 +80,8 @@ password to authenticate with the manifest server
 fetch submodules from server
 .TP
 \fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects
+use the manifest superproject to sync projects;
+implies \fB\-c\fR
 .TP
 \fB\-\-no\-use\-superproject\fR
 disable use of manifest superprojects
@@ -88,7 +90,7 @@ disable use of manifest superprojects
 fetch tags
 .TP
 \fB\-\-no\-tags\fR
-don't fetch tags
+don't fetch tags (default)
 .TP
 \fB\-\-optimized\-fetch\fR
 only fetch projects fixed to sha1 if revision does not
@@ -99,6 +101,10 @@ number of times to retry fetches on transient errors
 .TP
 \fB\-\-prune\fR
 delete refs that no longer exist on the remote
+(default)
+.TP
+\fB\-\-no\-prune\fR
+do not delete refs that no longer exist on the remote
 .TP
 \fB\-s\fR, \fB\-\-smart\-sync\fR
 smart sync using manifest from the latest known good

man/repo-upload.1

@@ -15,7 +15,8 @@ Upload changes for code review
 show this help message and exit
 .TP
 \fB\-j\fR JOBS, \fB\-\-jobs\fR=\fI\,JOBS\/\fR
-number of jobs to run in parallel (default: 4)
+number of jobs to run in parallel (default: based on
+number of CPU cores)
 .TP
 \fB\-t\fR
 send local branch name to Gerrit Code Review

man/repo.1

@@ -1,10 +1,49 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2021" "repo" "Repo Manual"
+.TH REPO "1" "November 2021" "repo" "Repo Manual"
 .SH NAME
 repo \- repository management tool built on top of git
-.SH DESCRIPTION
+.SH SYNOPSIS
-usage: repo COMMAND [ARGS]
+.B repo
-The complete list of recognized repo commands are:
+[\fI\,-p|--paginate|--no-pager\/\fR] \fI\,COMMAND \/\fR[\fI\,ARGS\/\fR]
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-\-help\-all\fR
+show this help message with all subcommands and exit
+.TP
+\fB\-p\fR, \fB\-\-paginate\fR
+display command output in the pager
+.TP
+\fB\-\-no\-pager\fR
+disable the pager
+.TP
+\fB\-\-color\fR=\fI\,COLOR\/\fR
+control color usage: auto, always, never
+.TP
+\fB\-\-trace\fR
+trace git command execution (REPO_TRACE=1)
+.TP
+\fB\-\-trace\-python\fR
+trace python command execution
+.TP
+\fB\-\-time\fR
+time repo command execution
+.TP
+\fB\-\-version\fR
+display this version of repo
+.TP
+\fB\-\-show\-toplevel\fR
+display the path of the top\-level directory of the
+repo client checkout
+.TP
+\fB\-\-event\-log\fR=\fI\,EVENT_LOG\/\fR
+filename of event log to append timeline to
+.TP
+\fB\-\-git\-trace2\-event\-log\fR=\fI\,GIT_TRACE2_EVENT_LOG\/\fR
+directory to write git trace2 event log to
+.SS "The complete list of recognized repo commands is:"
 .TP
 abandon
 Permanently abandon a development branch
@@ -91,3 +130,4 @@ version
 Display the version of repo
 .PP
 See 'repo help <command>' for more information on a specific command.
+Bug reports: https://bugs.chromium.org/p/gerrit/issues/entry?template=Repo+tool+issue

manifest_xml.py

@@ -25,7 +25,7 @@ import gitc_utils
 from git_config import GitConfig, IsId
 from git_refs import R_HEADS, HEAD
 import platform_utils
-from project import RemoteSpec, Project, MetaProject
+from project import Annotation, RemoteSpec, Project, MetaProject
 from error import (ManifestParseError, ManifestInvalidPathError,
                    ManifestInvalidRevisionError)
 from wrapper import Wrapper
@@ -149,16 +149,18 @@ class _XmlRemote(object):
     self.reviewUrl = review
     self.revision = revision
     self.resolvedFetchUrl = self._resolveFetchUrl()
+    self.annotations = []
 
   def __eq__(self, other):
     if not isinstance(other, _XmlRemote):
       return False
-    return self.__dict__ == other.__dict__
+    return (sorted(self.annotations) == sorted(other.annotations) and
+      self.name == other.name and self.fetchUrl == other.fetchUrl and
+      self.pushUrl == other.pushUrl and self.remoteAlias == other.remoteAlias
+      and self.reviewUrl == other.reviewUrl and self.revision == other.revision)
 
   def __ne__(self, other):
-    if not isinstance(other, _XmlRemote):
+    return not self.__eq__(other)
-      return True
-    return self.__dict__ != other.__dict__
 
   def _resolveFetchUrl(self):
     if self.fetchUrl is None:
@@ -191,6 +193,9 @@ class _XmlRemote(object):
                       orig_name=self.name,
                       fetchUrl=self.fetchUrl)
 
+  def AddAnnotation(self, name, value, keep):
+    self.annotations.append(Annotation(name, value, keep))
+
 
 class XmlManifest(object):
   """manages the repo configuration file"""
@@ -265,8 +270,7 @@ class XmlManifest(object):
     self.Override(name)
 
     # Old versions of repo would generate symlinks we need to clean up.
-    if os.path.lexists(self.manifestFile):
+    platform_utils.remove(self.manifestFile, missing_ok=True)
-      platform_utils.remove(self.manifestFile)
     # This file is interpreted as if it existed inside the manifest repo.
     # That allows us to use <include> with the relative file name.
     with open(self.manifestFile, 'w') as fp:
@@ -300,6 +304,13 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     if r.revision is not None:
       e.setAttribute('revision', r.revision)
 
+    for a in r.annotations:
+      if a.keep == 'true':
+        ae = doc.createElement('annotation')
+        ae.setAttribute('name', a.name)
+        ae.setAttribute('value', a.value)
+        e.appendChild(ae)
+
   def _ParseList(self, field):
     """Parse fields that contain flattened lists.
 
@@ -495,6 +506,9 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
       if not d.remote or remote.orig_name != remoteName:
         remoteName = remote.orig_name
         e.setAttribute('remote', remoteName)
+      revision = remote.revision or d.revisionExpr
+      if not revision or revision != self._superproject['revision']:
+        e.setAttribute('revision', self._superproject['revision'])
       root.appendChild(e)
 
     if self._contactinfo.bugurl != Wrapper().BUG_URL:
@@ -840,6 +854,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
       for subproject in project.subprojects:
         recursively_add_projects(subproject)
 
+    repo_hooks_project = None
+    enabled_repo_hooks = None
     for node in itertools.chain(*node_list):
       if node.nodeName == 'project':
         project = self._ParseProject(node)
@@ -852,6 +868,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                                    'project: %s' % name)
 
         path = node.getAttribute('path')
+        dest_path = node.getAttribute('dest-path')
         groups = node.getAttribute('groups')
         if groups:
           groups = self._ParseList(groups)
@@ -860,46 +877,37 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         if remote:
           remote = self._get_remote(node)
 
+        named_projects = self._projects[name]
+        if dest_path and not path and len(named_projects) > 1:
+          raise ManifestParseError('extend-project cannot use dest-path when '
+                                   'matching multiple projects: %s' % name)
         for p in self._projects[name]:
           if path and p.relpath != path:
             continue
           if groups:
             p.groups.extend(groups)
           if revision:
-            p.revisionExpr = revision
+            p.SetRevision(revision)
-            if IsId(revision):
+
-              p.revisionId = revision
-            else:
-              p.revisionId = None
           if remote:
             p.remote = remote.ToRemoteSpec(name)
-      if node.nodeName == 'repo-hooks':
-        # Get the name of the project and the (space-separated) list of enabled.
-        repo_hooks_project = self._reqatt(node, 'in-project')
-        enabled_repo_hooks = self._ParseList(self._reqatt(node, 'enabled-list'))
 
+          if dest_path:
+            del self._paths[p.relpath]
+            relpath, worktree, gitdir, objdir, _ = self.GetProjectPaths(name, dest_path)
+            p.UpdatePaths(relpath, worktree, gitdir, objdir)
+            self._paths[p.relpath] = p
+
+      if node.nodeName == 'repo-hooks':
         # Only one project can be the hooks project
-        if self._repo_hooks_project is not None:
+        if repo_hooks_project is not None:
           raise ManifestParseError(
               'duplicate repo-hooks in %s' %
               (self.manifestFile))
 
-        # Store a reference to the Project.
+        # Get the name of the project and the (space-separated) list of enabled.
-        try:
+        repo_hooks_project = self._reqatt(node, 'in-project')
-          repo_hooks_projects = self._projects[repo_hooks_project]
+        enabled_repo_hooks = self._ParseList(self._reqatt(node, 'enabled-list'))
-        except KeyError:
-          raise ManifestParseError(
-              'project %s not found for repo-hooks' %
-              (repo_hooks_project))
-
-        if len(repo_hooks_projects) != 1:
-          raise ManifestParseError(
-              'internal error parsing repo-hooks in %s' %
-              (self.manifestFile))
-        self._repo_hooks_project = repo_hooks_projects[0]
-
-        # Store the enabled hooks in the Project object.
-        self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
       if node.nodeName == 'superproject':
         name = self._reqatt(node, 'name')
         # There can only be one superproject.
@@ -917,6 +925,13 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
           raise ManifestParseError("no remote for superproject %s within %s" %
                                    (name, self.manifestFile))
         self._superproject['remote'] = remote.ToRemoteSpec(name)
+        revision = node.getAttribute('revision') or remote.revision
+        if not revision:
+          revision = self._default.revisionExpr
+        if not revision:
+          raise ManifestParseError('no revision for superproject %s within %s' %
+                                   (name, self.manifestFile))
+        self._superproject['revision'] = revision
       if node.nodeName == 'contactinfo':
         bugurl = self._reqatt(node, 'bugurl')
         # This element can be repeated, later entries will clobber earlier ones.
@@ -932,12 +947,30 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
           # If the manifest removes the hooks project, treat it as if it deleted
           # the repo-hooks element too.
-          if self._repo_hooks_project and (self._repo_hooks_project.name == name):
+          if repo_hooks_project == name:
-            self._repo_hooks_project = None
+            repo_hooks_project = None
         elif not XmlBool(node, 'optional', False):
           raise ManifestParseError('remove-project element specifies non-existent '
                                    'project: %s' % name)
 
+    # Store repo hooks project information.
+    if repo_hooks_project:
+      # Store a reference to the Project.
+      try:
+        repo_hooks_projects = self._projects[repo_hooks_project]
+      except KeyError:
+        raise ManifestParseError(
+            'project %s not found for repo-hooks' %
+            (repo_hooks_project))
+
+      if len(repo_hooks_projects) != 1:
+        raise ManifestParseError(
+            'internal error parsing repo-hooks in %s' %
+            (self.manifestFile))
+      self._repo_hooks_project = repo_hooks_projects[0]
+      # Store the enabled hooks in the Project object.
+      self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
+
   def _AddMetaProjectMirror(self, m):
     name = None
     m_url = m.GetRemote(m.remote.name).url
@@ -995,7 +1028,14 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     if revision == '':
       revision = None
     manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
-    return _XmlRemote(name, alias, fetch, pushUrl, manifestUrl, review, revision)
+
+    remote = _XmlRemote(name, alias, fetch, pushUrl, manifestUrl, review, revision)
+
+    for n in node.childNodes:
+      if n.nodeName == 'annotation':
+        self._ParseAnnotation(remote, n)
+
+    return remote
 
   def _ParseDefault(self, node):
     """
@@ -1362,7 +1402,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
       self._ValidateFilePaths('linkfile', src, dest)
       project.AddLinkFile(src, dest, self.topdir)
 
-  def _ParseAnnotation(self, project, node):
+  def _ParseAnnotation(self, element, node):
     name = self._reqatt(node, 'name')
     value = self._reqatt(node, 'value')
     try:
@@ -1372,7 +1412,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
     if keep != "true" and keep != "false":
       raise ManifestParseError('optional "keep" attribute must be '
                                '"true" or "false"')
-    project.AddAnnotation(name, value, keep)
+    element.AddAnnotation(name, value, keep)
 
   def _get_remote(self, node):
     name = node.getAttribute('remote')

platform_utils.py

@@ -124,31 +124,30 @@ def rename(src, dst):
       else:
         raise
   else:
-    os.rename(src, dst)
+    shutil.move(src, dst)
 
 
-def remove(path):
+def remove(path, missing_ok=False):
   """Remove (delete) the file path. This is a replacement for os.remove that
   allows deleting read-only files on Windows, with support for long paths and
   for deleting directory symbolic links.
 
   Availability: Unix, Windows."""
-  if isWindows():
+  longpath = _makelongpath(path) if isWindows() else path
-    longpath = _makelongpath(path)
+  try:
-    try:
+    os.remove(longpath)
-      os.remove(longpath)
+  except OSError as e:
-    except OSError as e:
+    if e.errno == errno.EACCES:
-      if e.errno == errno.EACCES:
+      os.chmod(longpath, stat.S_IWRITE)
-        os.chmod(longpath, stat.S_IWRITE)
+      # Directory symbolic links must be deleted with 'rmdir'.
-        # Directory symbolic links must be deleted with 'rmdir'.
+      if islink(longpath) and isdir(longpath):
-        if islink(longpath) and isdir(longpath):
+        os.rmdir(longpath)
-          os.rmdir(longpath)
-        else:
-          os.remove(longpath)
       else:
-        raise
+        os.remove(longpath)
-  else:
+    elif missing_ok and e.errno == errno.ENOENT:
-    os.remove(path)
+      pass
+    else:
+      raise
 
 
 def walk(top, topdown=True, onerror=None, followlinks=False):

project.py

@@ -251,13 +251,29 @@ class DiffColoring(Coloring):
     self.fail = self.printer('fail', fg='red')
 
 
-class _Annotation(object):
+class Annotation(object):
 
   def __init__(self, name, value, keep):
     self.name = name
     self.value = value
     self.keep = keep
 
+  def __eq__(self, other):
+    if not isinstance(other, Annotation):
+      return False
+    return self.__dict__ == other.__dict__
+
+  def __lt__(self, other):
+    # This exists just so that lists of Annotation objects can be sorted, for
+    # use in comparisons.
+    if not isinstance(other, Annotation):
+      raise ValueError('comparison is not between two Annotation objects')
+    if self.name == other.name:
+      if self.value == other.value:
+        return self.keep < other.keep
+      return self.value < other.value
+    return self.name < other.name
+
 
 def _SafeExpandPath(base, subpath, skipfinal=False):
   """Make sure |subpath| is completely safe under |base|.
@@ -503,21 +519,8 @@ class Project(object):
     self.client = self.manifest = manifest
     self.name = name
     self.remote = remote
-    self.gitdir = gitdir.replace('\\', '/')
+    self.UpdatePaths(relpath, worktree, gitdir, objdir)
-    self.objdir = objdir.replace('\\', '/')
+    self.SetRevision(revisionExpr, revisionId=revisionId)
-    if worktree:
-      self.worktree = os.path.normpath(worktree).replace('\\', '/')
-    else:
-      self.worktree = None
-    self.relpath = relpath
-    self.revisionExpr = revisionExpr
-
-    if revisionId is None \
-            and revisionExpr \
-            and IsId(revisionExpr):
-      self.revisionId = revisionExpr
-    else:
-      self.revisionId = revisionId
 
     self.rebase = rebase
     self.groups = groups
@@ -540,16 +543,6 @@ class Project(object):
     self.copyfiles = []
     self.linkfiles = []
     self.annotations = []
-    self.config = GitConfig.ForRepository(gitdir=self.gitdir,
-                                          defaults=self.client.globalConfig)
-
-    if self.worktree:
-      self.work_git = self._GitGetByExec(self, bare=False, gitdir=gitdir)
-    else:
-      self.work_git = None
-    self.bare_git = self._GitGetByExec(self, bare=True, gitdir=gitdir)
-    self.bare_ref = GitRefs(gitdir)
-    self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=objdir)
     self.dest_branch = dest_branch
     self.old_revision = old_revision
 
@@ -557,6 +550,35 @@ class Project(object):
     # project containing repo hooks.
     self.enabled_repo_hooks = []
 
+  def SetRevision(self, revisionExpr, revisionId=None):
+    """Set revisionId based on revision expression and id"""
+    self.revisionExpr = revisionExpr
+    if revisionId is None and revisionExpr and IsId(revisionExpr):
+      self.revisionId = self.revisionExpr
+    else:
+      self.revisionId = revisionId
+
+  def UpdatePaths(self, relpath, worktree, gitdir, objdir):
+    """Update paths used by this project"""
+    self.gitdir = gitdir.replace('\\', '/')
+    self.objdir = objdir.replace('\\', '/')
+    if worktree:
+      self.worktree = os.path.normpath(worktree).replace('\\', '/')
+    else:
+      self.worktree = None
+    self.relpath = relpath
+
+    self.config = GitConfig.ForRepository(gitdir=self.gitdir,
+                                          defaults=self.manifest.globalConfig)
+
+    if self.worktree:
+      self.work_git = self._GitGetByExec(self, bare=False, gitdir=self.gitdir)
+    else:
+      self.work_git = None
+    self.bare_git = self._GitGetByExec(self, bare=True, gitdir=self.gitdir)
+    self.bare_ref = GitRefs(self.gitdir)
+    self.bare_objdir = self._GitGetByExec(self, bare=True, gitdir=self.objdir)
+
   @property
   def Derived(self):
     return self.is_derived
@@ -1166,10 +1188,8 @@ class Project(object):
       self._InitMRef()
     else:
       self._InitMirrorHead()
-      try:
+      platform_utils.remove(os.path.join(self.gitdir, 'FETCH_HEAD'),
-        platform_utils.remove(os.path.join(self.gitdir, 'FETCH_HEAD'))
+                            missing_ok=True)
-      except OSError:
-        pass
     return True
 
   def PostRepoUpgrade(self):
@@ -1448,7 +1468,7 @@ class Project(object):
     self.linkfiles.append(_LinkFile(self.worktree, src, topdir, dest))
 
   def AddAnnotation(self, name, value, keep):
-    self.annotations.append(_Annotation(name, value, keep))
+    self.annotations.append(Annotation(name, value, keep))
 
   def DownloadPatchSet(self, change_id, patch_id):
     """Download a single patch set of a single change to FETCH_HEAD.
@@ -2024,8 +2044,11 @@ class Project(object):
 
     if current_branch_only:
       if self.revisionExpr.startswith(R_TAGS):
-        # this is a tag and its sha1 value should never change
+        # This is a tag and its commit id should never change.
         tag_name = self.revisionExpr[len(R_TAGS):]
+      elif self.upstream and self.upstream.startswith(R_TAGS):
+        # This is a tag and its commit id should never change.
+        tag_name = self.upstream[len(R_TAGS):]
 
       if is_sha1 or tag_name is not None:
         if self._CheckForImmutableRevision():
@@ -2291,15 +2314,12 @@ class Project(object):
     cmd.append('+refs/tags/*:refs/tags/*')
 
     ok = GitCommand(self, cmd, bare=True).Wait() == 0
-    if os.path.exists(bundle_dst):
+    platform_utils.remove(bundle_dst, missing_ok=True)
-      platform_utils.remove(bundle_dst)
+    platform_utils.remove(bundle_tmp, missing_ok=True)
-    if os.path.exists(bundle_tmp):
-      platform_utils.remove(bundle_tmp)
     return ok
 
   def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet, verbose):
-    if os.path.exists(dstPath):
+    platform_utils.remove(dstPath, missing_ok=True)
-      platform_utils.remove(dstPath)
 
     cmd = ['curl', '--fail', '--output', tmpPath, '--netrc', '--location']
     if quiet:
@@ -2423,7 +2443,7 @@ class Project(object):
     if quiet:
       cmd.append('-q')
     if GitCommand(self, cmd).Wait() != 0:
-      raise GitError('%s submodule update --init --recursive %s ' % self.name)
+      raise GitError('%s submodule update --init --recursive ' % self.name)
 
   def _Rebase(self, upstream, onto=None):
     cmd = ['rebase']
@@ -2449,6 +2469,8 @@ class Project(object):
         os.makedirs(self.objdir)
         self.bare_objdir.init()
 
+        self._UpdateHooks(quiet=quiet)
+
         if self.use_git_worktrees:
           # Enable per-worktree config file support if possible.  This is more a
           # nice-to-have feature for users rather than a hard requirement.
@@ -2509,8 +2531,6 @@ class Project(object):
             _lwrite(os.path.join(self.gitdir, 'objects/info/alternates'),
                     os.path.join(ref_dir, 'objects') + '\n')
 
-        self._UpdateHooks(quiet=quiet)
-
         m = self.manifest.manifestProject.config
         for key in ['user.name', 'user.email']:
           if m.Has(key, include_defaults=False):
@@ -2526,11 +2546,11 @@ class Project(object):
       raise
 
   def _UpdateHooks(self, quiet=False):
-    if os.path.exists(self.gitdir):
+    if os.path.exists(self.objdir):
       self._InitHooks(quiet=quiet)
 
   def _InitHooks(self, quiet=False):
-    hooks = platform_utils.realpath(self._gitdir_path('hooks'))
+    hooks = platform_utils.realpath(os.path.join(self.objdir, 'hooks'))
     if not os.path.exists(hooks):
       os.makedirs(hooks)
     for stock_hook in _ProjectHooks():
@@ -2723,10 +2743,7 @@ class Project(object):
         # If the source file doesn't exist, ensure the destination
         # file doesn't either.
         if name in symlink_files and not os.path.lexists(src):
-          try:
+          platform_utils.remove(dst, missing_ok=True)
-            platform_utils.remove(dst)
-          except OSError:
-            pass
 
       except OSError as e:
         if e.errno == errno.EPERM:
@@ -2764,50 +2781,95 @@ class Project(object):
     self._InitMRef()
 
   def _InitWorkTree(self, force_sync=False, submodules=False):
-    realdotgit = os.path.join(self.worktree, '.git')
+    """Setup the worktree .git path.
-    tmpdotgit = realdotgit + '.tmp'
+
-    init_dotgit = not os.path.exists(realdotgit)
+    This is the user-visible path like src/foo/.git/.
-    if init_dotgit:
+
-      if self.use_git_worktrees:
+    With non-git-worktrees, this will be a symlink to the .repo/projects/ path.
+    With git-worktrees, this will be a .git file using "gitdir: ..." syntax.
+
+    Older checkouts had .git/ directories.  If we see that, migrate it.
+
+    This also handles changes in the manifest.  Maybe this project was backed
+    by "foo/bar" on the server, but now it's "new/foo/bar".  We have to update
+    the path we point to under .repo/projects/ to match.
+    """
+    dotgit = os.path.join(self.worktree, '.git')
+
+    # If using an old layout style (a directory), migrate it.
+    if not platform_utils.islink(dotgit) and platform_utils.isdir(dotgit):
+      self._MigrateOldWorkTreeGitDir(dotgit)
+
+    init_dotgit = not os.path.exists(dotgit)
+    if self.use_git_worktrees:
+      if init_dotgit:
         self._InitGitWorktree()
         self._CopyAndLinkFiles()
-        return
-
-      dotgit = tmpdotgit
-      platform_utils.rmtree(tmpdotgit, ignore_errors=True)
-      os.makedirs(tmpdotgit)
-      self._ReferenceGitDir(self.gitdir, tmpdotgit, share_refs=True,
-                            copy_all=False)
     else:
-      dotgit = realdotgit
+      if not init_dotgit:
+        # See if the project has changed.
+        if platform_utils.realpath(self.gitdir) != platform_utils.realpath(dotgit):
+          platform_utils.remove(dotgit)
 
-    try:
+      if init_dotgit or not os.path.exists(dotgit):
-      self._CheckDirReference(self.gitdir, dotgit, share_refs=True)
+        os.makedirs(self.worktree, exist_ok=True)
-    except GitError as e:
+        platform_utils.symlink(os.path.relpath(self.gitdir, self.worktree), dotgit)
-      if force_sync and not init_dotgit:
-        try:
-          platform_utils.rmtree(dotgit)
-          return self._InitWorkTree(force_sync=False, submodules=submodules)
-        except Exception:
-          raise e
-      raise e
 
-    if init_dotgit:
+      if init_dotgit:
-      _lwrite(os.path.join(tmpdotgit, HEAD), '%s\n' % self.GetRevisionId())
+        _lwrite(os.path.join(dotgit, HEAD), '%s\n' % self.GetRevisionId())
 
-      # Now that the .git dir is fully set up, move it to its final home.
+        # Finish checking out the worktree.
-      platform_utils.rename(tmpdotgit, realdotgit)
+        cmd = ['read-tree', '--reset', '-u', '-v', HEAD]
+        if GitCommand(self, cmd).Wait() != 0:
+          raise GitError('Cannot initialize work tree for ' + self.name)
 
-      # Finish checking out the worktree.
+        if submodules:
-      cmd = ['read-tree', '--reset', '-u']
+          self._SyncSubmodules(quiet=True)
-      cmd.append('-v')
+        self._CopyAndLinkFiles()
-      cmd.append(HEAD)
-      if GitCommand(self, cmd).Wait() != 0:
-        raise GitError('Cannot initialize work tree for ' + self.name)
 
-      if submodules:
+  @classmethod
-        self._SyncSubmodules(quiet=True)
+  def _MigrateOldWorkTreeGitDir(cls, dotgit):
-      self._CopyAndLinkFiles()
+    """Migrate the old worktree .git/ dir style to a symlink.
+
+    This logic specifically only uses state from |dotgit| to figure out where to
+    move content and not |self|.  This way if the backing project also changed
+    places, we only do the .git/ dir to .git symlink migration here.  The path
+    updates will happen independently.
+    """
+    # Figure out where in .repo/projects/ it's pointing to.
+    if not os.path.islink(os.path.join(dotgit, 'refs')):
+      raise GitError(f'{dotgit}: unsupported checkout state')
+    gitdir = os.path.dirname(os.path.realpath(os.path.join(dotgit, 'refs')))
+
+    # Remove known symlink paths that exist in .repo/projects/.
+    KNOWN_LINKS = {
+        'config', 'description', 'hooks', 'info', 'logs', 'objects',
+        'packed-refs', 'refs', 'rr-cache', 'shallow', 'svn',
+    }
+    # Paths that we know will be in both, but are safe to clobber in .repo/projects/.
+    SAFE_TO_CLOBBER = {
+        'COMMIT_EDITMSG', 'FETCH_HEAD', 'HEAD', 'index', 'ORIG_HEAD',
+    }
+
+    # Now walk the paths and sync the .git/ to .repo/projects/.
+    for name in platform_utils.listdir(dotgit):
+      dotgit_path = os.path.join(dotgit, name)
+      if name in KNOWN_LINKS:
+        if platform_utils.islink(dotgit_path):
+          platform_utils.remove(dotgit_path)
+        else:
+          raise GitError(f'{dotgit_path}: should be a symlink')
+      else:
+        gitdir_path = os.path.join(gitdir, name)
+        if name in SAFE_TO_CLOBBER or not os.path.exists(gitdir_path):
+          platform_utils.remove(gitdir_path, missing_ok=True)
+          platform_utils.rename(dotgit_path, gitdir_path)
+        else:
+          raise GitError(f'{dotgit_path}: unknown file; please file a bug')
+
+    # Now that the dir should be empty, clear it out, and symlink it over.
+    platform_utils.rmdir(dotgit)
+    platform_utils.symlink(os.path.relpath(gitdir, os.path.dirname(dotgit)), dotgit)
 
   def _get_symlink_error_message(self):
     if platform_utils.isWindows():
@@ -2817,9 +2879,6 @@ class Project(object):
               'for other options.')
     return 'filesystem must support symlinks'
 
-  def _gitdir_path(self, path):
-    return platform_utils.realpath(os.path.join(self.gitdir, path))
-
   def _revlist(self, *args, **kw):
     a = []
     a.extend(args)

release/sign-launcher.py

@@ -20,6 +20,7 @@ This is intended to be run only by the official Repo release managers.
 
 import argparse
 import os
+import re
 import subprocess
 import sys
 
@@ -49,18 +50,37 @@ def check(opts):
   util.run(opts, ['gpg', '--verify', f'{opts.launcher}.asc'])
 
 
-def postmsg(opts):
+def get_version(opts):
+  """Get the version from |launcher|."""
+  # Make sure we don't search $PATH when signing the "repo" file in the cwd.
+  launcher = os.path.join('.', opts.launcher)
+  cmd = [launcher, '--version']
+  ret = util.run(opts, cmd, encoding='utf-8', stdout=subprocess.PIPE)
+  m = re.search(r'repo launcher version ([0-9.]+)', ret.stdout)
+  if not m:
+    sys.exit(f'{opts.launcher}: unable to detect repo version')
+  return m.group(1)
+
+
+def postmsg(opts, version):
   """Helpful info to show at the end for release manager."""
   print(f"""
 Repo launcher bucket:
   gs://git-repo-downloads/
 
-To upload this launcher directly:
+You should first upload it with a specific version:
-  gsutil cp -a public-read {opts.launcher} {opts.launcher}.asc gs://git-repo-downloads/
+  gsutil cp -a public-read {opts.launcher} gs://git-repo-downloads/repo-{version}
+  gsutil cp -a public-read {opts.launcher}.asc gs://git-repo-downloads/repo-{version}.asc
 
-NB: You probably want to upload it with a specific version first, e.g.:
+Then to make it the public default:
-  gsutil cp -a public-read {opts.launcher} gs://git-repo-downloads/repo-3.0
+  gsutil cp -a public-read gs://git-repo-downloads/repo-{version} gs://git-repo-downloads/repo
-  gsutil cp -a public-read {opts.launcher}.asc gs://git-repo-downloads/repo-3.0.asc
+  gsutil cp -a public-read gs://git-repo-downloads/repo-{version}.asc gs://git-repo-downloads/repo.asc
+
+NB: If a rollback is necessary, the GS bucket archives old versions, and may be
+    accessed by specifying their unique id number.
+  gsutil ls -la gs://git-repo-downloads/repo gs://git-repo-downloads/repo.asc
+  gsutil cp -a public-read gs://git-repo-downloads/repo#<unique id> gs://git-repo-downloads/repo
+  gsutil cp -a public-read gs://git-repo-downloads/repo.asc#<unique id> gs://git-repo-downloads/repo.asc
 """)
 
 
@@ -103,9 +123,10 @@ def main(argv):
     opts.keys = [util.KEYID_DSA, util.KEYID_RSA, util.KEYID_ECC]
     util.import_release_key(opts)
 
+  version = get_version(opts)
   sign(opts)
   check(opts)
-  postmsg(opts)
+  postmsg(opts, version)
 
   return 0
 

release/update-manpages

@@ -47,6 +47,11 @@ def main(argv):
   if not shutil.which('help2man'):
     sys.exit('Please install help2man to continue.')
 
+  # Let repo know we're generating man pages so it can avoid some dynamic
+  # behavior (like probing active number of CPUs).  We use a weird name &
+  # value to make it less likely for users to set this var themselves.
+  os.environ['_REPO_GENERATE_MANPAGES_'] = ' indeed! '
+
   # "repo branch" is an alias for "repo branches".
   del subcmds.all_commands['branch']
   (MANDIR / 'repo-branch.1').write_text('.so man1/repo-branches.1')
@@ -54,12 +59,12 @@ def main(argv):
   version = RepoSourceVersion()
   cmdlist = [['help2man', '-N', '-n', f'repo {cmd} - manual page for repo {cmd}',
     '-S', f'repo {cmd}', '-m', 'Repo Manual', f'--version-string={version}',
-    '-o', MANDIR.joinpath(f'repo-{cmd}.1'), TOPDIR.joinpath('repo'),
+    '-o', MANDIR.joinpath(f'repo-{cmd}.1.tmp'), TOPDIR.joinpath('repo'),
     '-h', f'help {cmd}'] for cmd in subcmds.all_commands]
   cmdlist.append(['help2man', '-N', '-n', 'repository management tool built on top of git',
     '-S', 'repo', '-m', 'Repo Manual', f'--version-string={version}',
-    '-o', MANDIR.joinpath('repo.1'), TOPDIR.joinpath('repo'),
+    '-o', MANDIR.joinpath('repo.1.tmp'), TOPDIR.joinpath('repo'),
-    '-h', 'help --all'])
+    '-h', '--help-all'])
 
   with tempfile.TemporaryDirectory() as tempdir:
     repo_dir = Path(tempdir) / '.repo'
@@ -75,11 +80,23 @@ def main(argv):
       (r'^\.IP\n(.*:)\n', '.SS \g<1>\n'),
       (r'^\.PP\nDescription', '.SH DETAILS'),
   )
-  for path in MANDIR.glob('*.1'):
+  for tmp_path in MANDIR.glob('*.1.tmp'):
-    data = path.read_text()
+    path = tmp_path.parent / tmp_path.stem
+    old_data = path.read_text() if path.exists() else ''
+
+    data = tmp_path.read_text()
+    tmp_path.unlink()
+
     for pattern, replacement in regex:
       data = re.sub(pattern, replacement, data, flags=re.M)
-    path.write_text(data)
+
+    # If the only thing that changed was the date, don't refresh.  This avoids
+    # a lot of noise when only one file actually updates.
+    old_data = re.sub(r'^(\.TH REPO "1" ")([^"]+)', r'\1', old_data, flags=re.M)
+    new_data = re.sub(r'^(\.TH REPO "1" ")([^"]+)', r'\1', data, flags=re.M)
+    if old_data != new_data:
+      path.write_text(data)
+
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))

repo

@@ -149,7 +149,7 @@ if not REPO_REV:
 BUG_URL = 'https://bugs.chromium.org/p/gerrit/issues/entry?template=Repo+tool+issue'
 
 # increment this whenever we make important changes to this script
-VERSION = (2, 15)
+VERSION = (2, 17)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (2, 3)
@@ -312,6 +312,10 @@ def InitParser(parser, gitc_init=False):
                    metavar='PLATFORM')
   group.add_option('--submodules', action='store_true',
                    help='sync any submodules associated with the manifest repo')
+  group.add_option('--standalone-manifest', action='store_true',
+                   help='download the manifest as a static file '
+                        'rather then create a git checkout of '
+                        'the manifest repo')
 
   # Options that only affect manifest project, and not any of the projects
   # specified in the manifest itself.
@@ -368,7 +372,7 @@ def InitParser(parser, gitc_init=False):
                    help='filter for use with --partial-clone '
                         '[default: %default]')
   group.add_option('--use-superproject', action='store_true', default=None,
-                   help='use the manifest superproject to sync projects')
+                   help='use the manifest superproject to sync projects; implies -c')
   group.add_option('--no-use-superproject', action='store_false',
                    dest='use_superproject',
                    help='disable use of manifest superprojects')

ssh.py

@@ -52,6 +52,9 @@ def version():
   """return ssh version as a tuple"""
   try:
     return _parse_ssh_version()
+  except FileNotFoundError:
+    print('fatal: ssh not installed', file=sys.stderr)
+    sys.exit(1)
   except subprocess.CalledProcessError:
     print('fatal: unable to detect ssh version', file=sys.stderr)
     sys.exit(1)

subcmds/help.py

@@ -50,14 +50,21 @@ Displays detailed usage information about a command.
 
   def _PrintAllCommands(self):
     print('usage: repo COMMAND [ARGS]')
-    print('The complete list of recognized repo commands are:')
+    self.PrintAllCommandsBody()
+
+  def PrintAllCommandsBody(self):
+    print('The complete list of recognized repo commands is:')
     commandNames = list(sorted(all_commands))
     self._PrintCommands(commandNames)
     print("See 'repo help <command>' for more information on a "
           'specific command.')
+    print('Bug reports:', Wrapper().BUG_URL)
 
   def _PrintCommonCommands(self):
     print('usage: repo COMMAND [ARGS]')
+    self.PrintCommonCommandsBody()
+
+  def PrintCommonCommandsBody(self):
     print('The most commonly used repo commands are:')
 
     def gitc_supported(cmd):

subcmds/init.py

@@ -24,6 +24,7 @@ from error import ManifestParseError
 from project import SyncBuffer
 from git_config import GitConfig
 from git_command import git_require, MIN_GIT_VERSION_SOFT, MIN_GIT_VERSION_HARD
+import fetch
 import git_superproject
 import platform_utils
 from wrapper import Wrapper
@@ -53,6 +54,12 @@ The optional -m argument can be used to specify an alternate manifest
 to be used. If no manifest is specified, the manifest default.xml
 will be used.
 
+If the --standalone-manifest argument is set, the manifest will be downloaded
+directly from the specified --manifest-url as a static file (rather than
+setting up a manifest git checkout). With --standalone-manifest, the manifest
+will be fully static and will not be re-downloaded during subsesquent
+`repo init` and `repo sync` calls.
+
 The --reference option can be used to point to a directory that
 has the content of a --mirror sync. This will make the working
 directory use as much data as possible from the local reference
@@ -112,6 +119,27 @@ to update the working directory files.
     m = self.manifest.manifestProject
     is_new = not m.Exists
 
+    # If repo has already been initialized, we take -u with the absence of
+    # --standalone-manifest to mean "transition to a standard repo set up",
+    # which necessitates starting fresh.
+    # If --standalone-manifest is set, we always tear everything down and start
+    # anew.
+    if not is_new:
+      was_standalone_manifest = m.config.GetString('manifest.standalone')
+      if was_standalone_manifest and not opt.manifest_url:
+        print('fatal: repo was initialized with a standlone manifest, '
+              'cannot be re-initialized without --manifest-url/-u')
+        sys.exit(1)
+
+      if opt.standalone_manifest or (was_standalone_manifest and
+                                     opt.manifest_url):
+        m.config.ClearCache()
+        if m.gitdir and os.path.exists(m.gitdir):
+          platform_utils.rmtree(m.gitdir)
+        if m.worktree and os.path.exists(m.worktree):
+          platform_utils.rmtree(m.worktree)
+
+    is_new = not m.Exists
     if is_new:
       if not opt.manifest_url:
         print('fatal: manifest url is required.', file=sys.stderr)
@@ -136,6 +164,21 @@ to update the working directory files.
 
       m._InitGitDir(mirror_git=mirrored_manifest_git)
 
+    # If standalone_manifest is set, mark the project as "standalone" -- we'll
+    # still do much of the manifests.git set up, but will avoid actual syncs to
+    # a remote.
+    standalone_manifest = False
+    if opt.standalone_manifest:
+      standalone_manifest = True
+      m.config.SetString('manifest.standalone', opt.manifest_url)
+    elif not opt.manifest_url and not opt.manifest_branch:
+      # If -u is set and --standalone-manifest is not, then we're not in
+      # standalone mode. Otherwise, use config to infer what we were in the last
+      # init.
+      standalone_manifest = bool(m.config.GetString('manifest.standalone'))
+    if not standalone_manifest:
+      m.config.SetString('manifest.standalone', None)
+
     self._ConfigureDepth(opt)
 
     # Set the remote URL before the remote branch as we might need it below.
@@ -145,22 +188,23 @@ to update the working directory files.
       r.ResetFetch()
       r.Save()
 
-    if opt.manifest_branch:
+    if not standalone_manifest:
-      if opt.manifest_branch == 'HEAD':
+      if opt.manifest_branch:
-        opt.manifest_branch = m.ResolveRemoteHead()
+        if opt.manifest_branch == 'HEAD':
-        if opt.manifest_branch is None:
+          opt.manifest_branch = m.ResolveRemoteHead()
-          print('fatal: unable to resolve HEAD', file=sys.stderr)
+          if opt.manifest_branch is None:
-          sys.exit(1)
+            print('fatal: unable to resolve HEAD', file=sys.stderr)
-      m.revisionExpr = opt.manifest_branch
+            sys.exit(1)
-    else:
+        m.revisionExpr = opt.manifest_branch
-      if is_new:
-        default_branch = m.ResolveRemoteHead()
-        if default_branch is None:
-          # If the remote doesn't have HEAD configured, default to master.
-          default_branch = 'refs/heads/master'
-        m.revisionExpr = default_branch
       else:
-        m.PreSync()
+        if is_new:
+          default_branch = m.ResolveRemoteHead()
+          if default_branch is None:
+            # If the remote doesn't have HEAD configured, default to master.
+            default_branch = 'refs/heads/master'
+          m.revisionExpr = default_branch
+        else:
+          m.PreSync()
 
     groups = re.split(r'[,\s]+', opt.groups)
     all_platforms = ['linux', 'darwin', 'windows']
@@ -250,6 +294,16 @@ to update the working directory files.
     if opt.use_superproject is not None:
       m.config.SetBoolean('repo.superproject', opt.use_superproject)
 
+    if standalone_manifest:
+      if is_new:
+        manifest_name = 'default.xml'
+        manifest_data = fetch.fetch_file(opt.manifest_url, verbose=opt.verbose)
+        dest = os.path.join(m.worktree, manifest_name)
+        os.makedirs(os.path.dirname(dest), exist_ok=True)
+        with open(dest, 'wb') as f:
+          f.write(manifest_data)
+      return
+
     if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet, verbose=opt.verbose,
                               clone_bundle=opt.clone_bundle,
                               current_branch_only=opt.current_branch_only,
@@ -423,8 +477,18 @@ to update the working directory files.
 
     # Check this here, else manifest will be tagged "not new" and init won't be
     # possible anymore without removing the .repo/manifests directory.
-    if opt.archive and opt.mirror:
+    if opt.mirror:
-      self.OptionParser.error('--mirror and --archive cannot be used together.')
+      if opt.archive:
+        self.OptionParser.error('--mirror and --archive cannot be used '
+                                'together.')
+      if opt.use_superproject is not None:
+        self.OptionParser.error('--mirror and --use-superproject cannot be '
+                                'used together.')
+
+    if opt.standalone_manifest and (opt.manifest_branch or
+                                    opt.manifest_name != 'default.xml'):
+      self.OptionParser.error('--manifest-branch and --manifest-name cannot'
+                              ' be used with --standalone-manifest.')
 
     if args:
       if opt.manifest_url:

subcmds/list.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+
 from command import Command, MirrorSafeCommand
 
 
@@ -43,20 +45,26 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
     p.add_option('-a', '--all',
                  action='store_true',
                  help='show projects regardless of checkout state')
-    p.add_option('-f', '--fullpath',
-                 dest='fullpath', action='store_true',
-                 help='display the full work tree path instead of the relative path')
     p.add_option('-n', '--name-only',
                  dest='name_only', action='store_true',
                  help='display only the name of the repository')
     p.add_option('-p', '--path-only',
                  dest='path_only', action='store_true',
                  help='display only the path of the repository')
+    p.add_option('-f', '--fullpath',
+                 dest='fullpath', action='store_true',
+                 help='display the full work tree path instead of the relative path')
+    p.add_option('--relative-to', metavar='PATH',
+                 help='display paths relative to this one (default: top of repo client checkout)')
 
   def ValidateOptions(self, opt, args):
     if opt.fullpath and opt.name_only:
       self.OptionParser.error('cannot combine -f and -n')
 
+    # Resolve any symlinks so the output is stable.
+    if opt.relative_to:
+      opt.relative_to = os.path.realpath(opt.relative_to)
+
   def Execute(self, opt, args):
     """List all projects and the associated directories.
 
@@ -76,6 +84,8 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
     def _getpath(x):
       if opt.fullpath:
         return x.worktree
+      if opt.relative_to:
+        return os.path.relpath(x.worktree, opt.relative_to)
       return x.relpath
 
     lines = []

subcmds/sync.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import errno
 import functools
 import http.cookiejar as cookielib
 import io
@@ -235,24 +234,25 @@ later is required to fix a server side protocol bug.
                  dest='fetch_submodules', action='store_true',
                  help='fetch submodules from server')
     p.add_option('--use-superproject', action='store_true',
-                 help='use the manifest superproject to sync projects')
+                 help='use the manifest superproject to sync projects; implies -c')
     p.add_option('--no-use-superproject', action='store_false',
                  dest='use_superproject',
                  help='disable use of manifest superprojects')
-    p.add_option('--tags',
+    p.add_option('--tags', action='store_true',
-                 action='store_false',
                  help='fetch tags')
     p.add_option('--no-tags',
                  dest='tags', action='store_false',
-                 help="don't fetch tags")
+                 help="don't fetch tags (default)")
     p.add_option('--optimized-fetch',
                  dest='optimized_fetch', action='store_true',
                  help='only fetch projects fixed to sha1 if revision does not exist locally')
     p.add_option('--retry-fetches',
                  default=0, action='store', type='int',
                  help='number of times to retry fetches on transient errors')
-    p.add_option('--prune', dest='prune', action='store_true',
+    p.add_option('--prune', action='store_true',
-                 help='delete refs that no longer exist on the remote')
+                 help='delete refs that no longer exist on the remote (default)')
+    p.add_option('--no-prune', dest='prune', action='store_false',
+                 help='do not delete refs that no longer exist on the remote')
     if show_smart:
       p.add_option('-s', '--smart-sync',
                    dest='smart_sync', action='store_true',
@@ -282,7 +282,7 @@ later is required to fix a server side protocol bug.
     """Returns True if current-branch or use-superproject options are enabled."""
     return opt.current_branch_only or git_superproject.UseSuperproject(opt, self.manifest)
 
-  def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests):
+  def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests, superproject_logging_data):
     """Update revisionId of every project with the SHA from superproject.
 
     This function updates each project's revisionId with SHA from superproject.
@@ -293,26 +293,37 @@ later is required to fix a server side protocol bug.
       args: Arguments to pass to GetProjects. See the GetProjects
           docstring for details.
       load_local_manifests: Whether to load local manifests.
+      superproject_logging_data: A dictionary of superproject data that is to be logged.
 
     Returns:
       Returns path to the overriding manifest file instead of None.
     """
+    print_messages = git_superproject.PrintMessages(opt, self.manifest)
     superproject = git_superproject.Superproject(self.manifest,
                                                  self.repodir,
                                                  self.git_event_log,
-                                                 quiet=opt.quiet)
+                                                 quiet=opt.quiet,
+                                                 print_messages=print_messages)
+    if opt.local_only:
+      manifest_path = superproject.manifest_path
+      if manifest_path:
+        self._ReloadManifest(manifest_path, load_local_manifests)
+      return manifest_path
+
     all_projects = self.GetProjects(args,
                                     missing_ok=True,
                                     submodules_ok=opt.fetch_submodules)
     update_result = superproject.UpdateProjectsRevisionId(all_projects)
     manifest_path = update_result.manifest_path
+    superproject_logging_data['updatedrevisionid'] = bool(manifest_path)
     if manifest_path:
       self._ReloadManifest(manifest_path, load_local_manifests)
     else:
-      print('warning: Update of revisionId from superproject has failed, '
+      if print_messages:
-            'repo sync will not use superproject to fetch the source. ',
+        print('warning: Update of revisionId from superproject has failed, '
-            'Please resync with the --no-use-superproject option to avoid this repo warning.',
+              'repo sync will not use superproject to fetch the source. ',
-            file=sys.stderr)
+              'Please resync with the --no-use-superproject option to avoid this repo warning.',
+              file=sys.stderr)
       if update_result.fatal and opt.use_superproject is not None:
         sys.exit(1)
     return manifest_path
@@ -437,8 +448,8 @@ later is required to fix a server side protocol bug.
       else:
         pm.update(inc=0, msg='warming up')
         chunksize = 4
-      with multiprocessing.Pool(
+      with multiprocessing.Pool(jobs, initializer=self._FetchInitChild,
-          jobs, initializer=self._FetchInitChild, initargs=(ssh_proxy,)) as pool:
+                                initargs=(ssh_proxy,)) as pool:
         results = pool.imap_unordered(
             functools.partial(self._FetchProjectList, opt),
             projects_list,
@@ -594,7 +605,7 @@ later is required to fix a server side protocol bug.
     pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet)
     pm.update(inc=0, msg='prescan')
 
-    gc_gitdirs = {}
+    tidy_dirs = {}
     for project in projects:
       # Make sure pruning never kicks in with shared projects.
       if (not project.use_git_worktrees and
@@ -611,17 +622,29 @@ later is required to fix a server side protocol bug.
                 % (project.relpath,),
                 file=sys.stderr)
           project.config.SetString('gc.pruneExpire', 'never')
-      gc_gitdirs[project.gitdir] = project.bare_git
+      project.config.SetString('gc.autoDetach', 'false')
-
+      # Only call git gc once per objdir, but call pack-refs for the remainder.
-    pm.update(inc=len(projects) - len(gc_gitdirs), msg='warming up')
+      if project.objdir not in tidy_dirs:
+        tidy_dirs[project.objdir] = (
+            True,  # Run a full gc.
+            project.bare_git,
+        )
+      elif project.gitdir not in tidy_dirs:
+        tidy_dirs[project.gitdir] = (
+            False,  # Do not run a full gc; just run pack-refs.
+            project.bare_git,
+        )
 
     cpu_count = os.cpu_count()
     jobs = min(self.jobs, cpu_count)
 
     if jobs < 2:
-      for bare_git in gc_gitdirs.values():
+      for (run_gc, bare_git) in tidy_dirs.values():
         pm.update(msg=bare_git._project.name)
-        bare_git.gc('--auto')
+        if run_gc:
+          bare_git.gc('--auto')
+        else:
+          bare_git.pack_refs()
       pm.end()
       return
 
@@ -630,11 +653,14 @@ later is required to fix a server side protocol bug.
     threads = set()
     sem = _threading.Semaphore(jobs)
 
-    def GC(bare_git):
+    def tidy_up(run_gc, bare_git):
       pm.start(bare_git._project.name)
       try:
         try:
-          bare_git.gc('--auto', config=config)
+          if run_gc:
+            bare_git.gc('--auto', config=config)
+          else:
+            bare_git.pack_refs(config=config)
         except GitError:
           err_event.set()
         except Exception:
@@ -644,11 +670,11 @@ later is required to fix a server side protocol bug.
         pm.finish(bare_git._project.name)
         sem.release()
 
-    for bare_git in gc_gitdirs.values():
+    for (run_gc, bare_git) in tidy_dirs.values():
       if err_event.is_set() and opt.fail_fast:
         break
       sem.acquire()
-      t = _threading.Thread(target=GC, args=(bare_git,))
+      t = _threading.Thread(target=tidy_up, args=(run_gc, bare_git,))
       t.daemon = True
       threads.add(t)
       t.start()
@@ -741,7 +767,7 @@ later is required to fix a server side protocol bug.
       with open(copylinkfile_path, 'rb') as fp:
         try:
           old_copylinkfile_paths = json.load(fp)
-        except:
+        except Exception:
           print('error: %s is not a json formatted file.' %
                 copylinkfile_path, file=sys.stderr)
           platform_utils.remove(copylinkfile_path)
@@ -756,13 +782,9 @@ later is required to fix a server side protocol bug.
           set(new_copyfile_paths))
 
       for need_remove_file in need_remove_files:
-        try:
+        # Try to remove the updated copyfile or linkfile.
-          platform_utils.remove(need_remove_file)
+        # So, if the file is not exist, nothing need to do.
-        except OSError as e:
+        platform_utils.remove(need_remove_file, missing_ok=True)
-          if e.errno == errno.ENOENT:
-            # Try to remove the updated copyfile or linkfile.
-            # So, if the file is not exist, nothing need to do.
-            pass
 
     # Create copy-link-files.json, save dest path of "copyfile" and "linkfile".
     with open(copylinkfile_path, 'w', encoding='utf-8') as fp:
@@ -907,6 +929,9 @@ later is required to fix a server side protocol bug.
       if None in [opt.manifest_server_username, opt.manifest_server_password]:
         self.OptionParser.error('both -u and -p must be given')
 
+    if opt.prune is None:
+      opt.prune = True
+
   def Execute(self, opt, args):
     if opt.jobs:
       self.jobs = opt.jobs
@@ -947,19 +972,32 @@ later is required to fix a server side protocol bug.
               file=sys.stderr)
 
     mp = self.manifest.manifestProject
-    mp.PreSync()
+    is_standalone_manifest = mp.config.GetString('manifest.standalone')
+    if not is_standalone_manifest:
+      mp.PreSync()
 
     if opt.repo_upgraded:
       _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
 
     if not opt.mp_update:
       print('Skipping update of local manifest project.')
-    else:
+    elif not is_standalone_manifest:
       self._UpdateManifestProject(opt, mp, manifest_name)
 
     load_local_manifests = not self.manifest.HasLocalManifests
-    if git_superproject.UseSuperproject(opt, self.manifest):
+    use_superproject = git_superproject.UseSuperproject(opt, self.manifest)
-      manifest_name = self._UpdateProjectsRevisionId(opt, args, load_local_manifests) or opt.manifest_name
+    if self.manifest.IsMirror or self.manifest.IsArchive:
+      # Don't use superproject, because we have no working tree.
+      use_superproject = False
+      print('Defaulting to no-use-superproject because there is no working tree.')
+    superproject_logging_data = {
+        'superproject': use_superproject,
+        'haslocalmanifests': bool(self.manifest.HasLocalManifests),
+        'hassuperprojecttag': bool(self.manifest.superproject),
+    }
+    if use_superproject:
+      manifest_name = self._UpdateProjectsRevisionId(
+          opt, args, load_local_manifests, superproject_logging_data) or opt.manifest_name
 
     if self.gitc_manifest:
       gitc_manifest_projects = self.GetProjects(args,
@@ -1073,11 +1111,29 @@ later is required to fix a server side protocol bug.
             file=sys.stderr)
       sys.exit(1)
 
+    # Log the previous sync analysis state from the config.
+    self.git_event_log.LogDataConfigEvents(mp.config.GetSyncAnalysisStateData(),
+                                           'previous_sync_state')
+
+    # Update and log with the new sync analysis state.
+    mp.config.UpdateSyncAnalysisState(opt, superproject_logging_data)
+    self.git_event_log.LogDataConfigEvents(mp.config.GetSyncAnalysisStateData(),
+                                           'current_sync_state')
+
     if not opt.quiet:
       print('repo sync has finished successfully.')
 
 
 def _PostRepoUpgrade(manifest, quiet=False):
+  # Link the docs for the internal .repo/ layout for people
+  link = os.path.join(manifest.repodir, 'internal-fs-layout.md')
+  if not platform_utils.islink(link):
+    target = os.path.join('repo', 'docs', 'internal-fs-layout.md')
+    try:
+      platform_utils.symlink(target, link)
+    except Exception:
+      pass
+
   wrapper = Wrapper()
   if wrapper.NeedSetupGnuPG():
     wrapper.SetupGnuPG(quiet)
@@ -1144,10 +1200,7 @@ class _FetchTimes(object):
         with open(self._path) as f:
           self._times = json.load(f)
       except (IOError, ValueError):
-        try:
+        platform_utils.remove(self._path, missing_ok=True)
-          platform_utils.remove(self._path)
-        except OSError:
-          pass
         self._times = {}
 
   def Save(self):
@@ -1165,10 +1218,7 @@ class _FetchTimes(object):
       with open(self._path, 'w') as f:
         json.dump(self._times, f, indent=2)
     except (IOError, TypeError):
-      try:
+      platform_utils.remove(self._path, missing_ok=True)
-        platform_utils.remove(self._path)
-      except OSError:
-        pass
 
 # This is a replacement for xmlrpc.client.Transport using urllib2
 # and supporting persistent-http[s]. It cannot change hosts from

tests/test_git_config.py

@@ -168,6 +168,25 @@ class GitConfigReadWriteTests(unittest.TestCase):
     config = self.get_config()
     self.assertIsNone(config.GetBoolean('foo.bar'))
 
+  def test_GetSyncAnalysisStateData(self):
+    """Test config entries with a sync state analysis data."""
+    superproject_logging_data = {}
+    superproject_logging_data['test'] = False
+    options = type('options', (object,), {})()
+    options.verbose = 'true'
+    options.mp_update = 'false'
+    TESTS = (
+        ('superproject.test', 'false'),
+        ('options.verbose', 'true'),
+        ('options.mpupdate', 'false'),
+        ('main.version', '1'),
+    )
+    self.config.UpdateSyncAnalysisState(options, superproject_logging_data)
+    sync_data = self.config.GetSyncAnalysisStateData()
+    for key, value in TESTS:
+      self.assertEqual(sync_data[f'{git_config.SYNC_STATE_PREFIX}{key}'], value)
+    self.assertTrue(sync_data[f'{git_config.SYNC_STATE_PREFIX}main.synctime'])
+
 
 if __name__ == '__main__':
   unittest.main()

tests/test_git_superproject.py

@@ -157,7 +157,7 @@ class SuperprojectTestCase(unittest.TestCase):
 """)
     self._superproject = git_superproject.Superproject(manifest, self.repodir,
                                                        self.git_event_log)
-    with mock.patch.object(self._superproject, '_GetBranch', return_value='junk'):
+    with mock.patch.object(self._superproject, '_branch', 'junk'):
       sync_result = self._superproject.Sync()
       self.assertFalse(sync_result.success)
       self.assertTrue(sync_result.fatal)
@@ -203,7 +203,7 @@ class SuperprojectTestCase(unittest.TestCase):
     project.SetRevisionId('ABCDEF')
     # Create temporary directory so that it can write the file.
     os.mkdir(self._superproject._superproject_path)
-    manifest_path = self._superproject._WriteManfiestFile()
+    manifest_path = self._superproject._WriteManifestFile()
     self.assertIsNotNone(manifest_path)
     with open(manifest_path, 'r') as fp:
       manifest_xml_data = fp.read()

tests/test_git_trace2_event_log.py

@@ -42,7 +42,7 @@ class EventLogTestCase(unittest.TestCase):
     self._event_log_module = git_trace2_event_log.EventLog(env=env)
     self._log_data = None
 
-  def verifyCommonKeys(self, log_entry, expected_event_name, full_sid=True):
+  def verifyCommonKeys(self, log_entry, expected_event_name=None, full_sid=True):
     """Helper function to verify common event log keys."""
     self.assertIn('event', log_entry)
     self.assertIn('sid', log_entry)
@@ -50,7 +50,8 @@ class EventLogTestCase(unittest.TestCase):
     self.assertIn('time', log_entry)
 
     # Do basic data format validation.
-    self.assertEqual(expected_event_name, log_entry['event'])
+    if expected_event_name:
+      self.assertEqual(expected_event_name, log_entry['event'])
     if full_sid:
       self.assertRegex(log_entry['sid'], self.FULL_SID_REGEX)
     else:
@@ -65,6 +66,13 @@ class EventLogTestCase(unittest.TestCase):
         log_data.append(json.loads(line))
     return log_data
 
+  def remove_prefix(self, s, prefix):
+    """Return a copy string after removing |prefix| from |s|, if present or the original string."""
+    if s.startswith(prefix):
+        return s[len(prefix):]
+    else:
+        return s
+
   def test_initial_state_with_parent_sid(self):
     """Test initial state when 'GIT_TRACE2_PARENT_SID' is set by parent."""
     self.assertRegex(self._event_log_module.full_sid, self.FULL_SID_REGEX)
@@ -234,6 +242,42 @@ class EventLogTestCase(unittest.TestCase):
     self.assertEqual(len(self._log_data), 1)
     self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
 
+  def test_data_event_config(self):
+    """Test 'data' event data outputs all config keys.
+
+    Expected event log:
+    <version event>
+    <data event>
+    <data event>
+    """
+    config = {
+        'git.foo': 'bar',
+        'repo.partialclone': 'false',
+        'repo.syncstate.superproject.hassuperprojecttag': 'true',
+        'repo.syncstate.superproject.sys.argv': ['--', 'sync', 'protobuf'],
+    }
+    prefix_value = 'prefix'
+    self._event_log_module.LogDataConfigEvents(config, prefix_value)
+
+    with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
+      log_path = self._event_log_module.Write(path=tempdir)
+      self._log_data = self.readLog(log_path)
+
+    self.assertEqual(len(self._log_data), 5)
+    data_events = self._log_data[1:]
+    self.verifyCommonKeys(self._log_data[0], expected_event_name='version')
+
+    for event in data_events:
+      self.verifyCommonKeys(event)
+      # Check for 'data' event specific fields.
+      self.assertIn('key', event)
+      self.assertIn('value', event)
+      key = event['key']
+      key = self.remove_prefix(key, f'{prefix_value}/')
+      value = event['value']
+      self.assertEqual(self._event_log_module.GetDataEventName(value), event['event'])
+      self.assertTrue(key in config and value == config[key])
+
   def test_error_event(self):
     """Test and validate 'error' event data is valid.
 

tests/test_manifest_xml.py

@@ -261,6 +261,19 @@ class XmlManifestTests(ManifestParseTestCase):
   <project name="repohooks" path="src/repohooks"/>
   <repo-hooks in-project="repohooks" enabled-list="a, b"/>
 </manifest>
+""")
+    self.assertEqual(manifest.repo_hooks_project.name, 'repohooks')
+    self.assertEqual(manifest.repo_hooks_project.enabled_repo_hooks, ['a', 'b'])
+
+  def test_repo_hooks_unordered(self):
+    """Check repo-hooks settings work even if the project def comes second."""
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="test-remote" fetch="http://localhost" />
+  <default remote="test-remote" revision="refs/heads/main" />
+  <repo-hooks in-project="repohooks" enabled-list="a, b"/>
+  <project name="repohooks" path="src/repohooks"/>
+</manifest>
 """)
     self.assertEqual(manifest.repo_hooks_project.name, 'repohooks')
     self.assertEqual(manifest.repo_hooks_project.enabled_repo_hooks, ['a', 'b'])
@@ -286,6 +299,25 @@ class XmlManifestTests(ManifestParseTestCase):
         '<superproject name="superproject"/>'
         '</manifest>')
 
+  def test_remote_annotations(self):
+    """Check remote settings."""
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="test-remote" fetch="http://localhost">
+    <annotation name="foo" value="bar"/>
+  </remote>
+</manifest>
+""")
+    self.assertEqual(manifest.remotes['test-remote'].annotations[0].name, 'foo')
+    self.assertEqual(manifest.remotes['test-remote'].annotations[0].value, 'bar')
+    self.assertEqual(
+        sort_attributes(manifest.ToXml().toxml()),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch="http://localhost" name="test-remote">'
+        '<annotation name="foo" value="bar"/>'
+        '</remote>'
+        '</manifest>')
+
 
 class IncludeElementTests(ManifestParseTestCase):
   """Tests for <include>."""
@@ -540,6 +572,7 @@ class SuperProjectElementTests(ManifestParseTestCase):
     self.assertEqual(manifest.superproject['name'], 'superproject')
     self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
     self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject['revision'], 'refs/heads/main')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -548,6 +581,72 @@ class SuperProjectElementTests(ManifestParseTestCase):
         '<superproject name="superproject"/>'
         '</manifest>')
 
+  def test_superproject_revision(self):
+    """Check superproject settings with a different revision attribute"""
+    self.maxDiff = None
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="test-remote" fetch="http://localhost" />
+  <default remote="test-remote" revision="refs/heads/main" />
+  <superproject name="superproject" revision="refs/heads/stable" />
+</manifest>
+""")
+    self.assertEqual(manifest.superproject['name'], 'superproject')
+    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
+    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject['revision'], 'refs/heads/stable')
+    self.assertEqual(
+        sort_attributes(manifest.ToXml().toxml()),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch="http://localhost" name="test-remote"/>'
+        '<default remote="test-remote" revision="refs/heads/main"/>'
+        '<superproject name="superproject" revision="refs/heads/stable"/>'
+        '</manifest>')
+
+  def test_superproject_revision_default_negative(self):
+    """Check superproject settings with a same revision attribute"""
+    self.maxDiff = None
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="test-remote" fetch="http://localhost" />
+  <default remote="test-remote" revision="refs/heads/stable" />
+  <superproject name="superproject" revision="refs/heads/stable" />
+</manifest>
+""")
+    self.assertEqual(manifest.superproject['name'], 'superproject')
+    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
+    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject['revision'], 'refs/heads/stable')
+    self.assertEqual(
+        sort_attributes(manifest.ToXml().toxml()),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch="http://localhost" name="test-remote"/>'
+        '<default remote="test-remote" revision="refs/heads/stable"/>'
+        '<superproject name="superproject"/>'
+        '</manifest>')
+
+  def test_superproject_revision_remote(self):
+    """Check superproject settings with a same revision attribute"""
+    self.maxDiff = None
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="test-remote" fetch="http://localhost" revision="refs/heads/main" />
+  <default remote="test-remote" />
+  <superproject name="superproject" revision="refs/heads/stable" />
+</manifest>
+""")
+    self.assertEqual(manifest.superproject['name'], 'superproject')
+    self.assertEqual(manifest.superproject['remote'].name, 'test-remote')
+    self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/superproject')
+    self.assertEqual(manifest.superproject['revision'], 'refs/heads/stable')
+    self.assertEqual(
+        sort_attributes(manifest.ToXml().toxml()),
+        '<?xml version="1.0" ?><manifest>'
+        '<remote fetch="http://localhost" name="test-remote" revision="refs/heads/main"/>'
+        '<default remote="test-remote"/>'
+        '<superproject name="superproject" revision="refs/heads/stable"/>'
+        '</manifest>')
+
   def test_remote(self):
     """Check superproject settings with a remote."""
     manifest = self.getXmlManifest("""
@@ -561,6 +660,7 @@ class SuperProjectElementTests(ManifestParseTestCase):
     self.assertEqual(manifest.superproject['name'], 'platform/superproject')
     self.assertEqual(manifest.superproject['remote'].name, 'superproject-remote')
     self.assertEqual(manifest.superproject['remote'].url, 'http://localhost/platform/superproject')
+    self.assertEqual(manifest.superproject['revision'], 'refs/heads/main')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -581,6 +681,7 @@ class SuperProjectElementTests(ManifestParseTestCase):
 """)
     self.assertEqual(manifest.superproject['name'], 'superproject')
     self.assertEqual(manifest.superproject['remote'].name, 'default-remote')
+    self.assertEqual(manifest.superproject['revision'], 'refs/heads/main')
     self.assertEqual(
         sort_attributes(manifest.ToXml().toxml()),
         '<?xml version="1.0" ?><manifest>'
@@ -632,9 +733,17 @@ class RemoteElementTests(ManifestParseTestCase):
   def test_remote(self):
     """Check remote settings."""
     a = manifest_xml._XmlRemote(name='foo')
-    b = manifest_xml._XmlRemote(name='bar')
+    a.AddAnnotation('key1', 'value1', 'true')
+    b = manifest_xml._XmlRemote(name='foo')
+    b.AddAnnotation('key2', 'value1', 'true')
+    c = manifest_xml._XmlRemote(name='foo')
+    c.AddAnnotation('key1', 'value2', 'true')
+    d = manifest_xml._XmlRemote(name='foo')
+    d.AddAnnotation('key1', 'value1', 'false')
     self.assertEqual(a, a)
     self.assertNotEqual(a, b)
+    self.assertNotEqual(a, c)
+    self.assertNotEqual(a, d)
     self.assertNotEqual(a, manifest_xml._Default())
     self.assertNotEqual(a, 123)
     self.assertNotEqual(a, None)
@@ -688,3 +797,49 @@ class RemoveProjectElementTests(ManifestParseTestCase):
 </manifest>
 """)
     self.assertEqual(manifest.projects, [])
+
+
+class ExtendProjectElementTests(ManifestParseTestCase):
+  """Tests for <extend-project>."""
+
+  def test_extend_project_dest_path_single_match(self):
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="myproject" />
+  <extend-project name="myproject" dest-path="bar" />
+</manifest>
+""")
+    self.assertEqual(len(manifest.projects), 1)
+    self.assertEqual(manifest.projects[0].relpath, 'bar')
+
+  def test_extend_project_dest_path_multi_match(self):
+    with self.assertRaises(manifest_xml.ManifestParseError):
+      manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="myproject" path="x" />
+  <project name="myproject" path="y" />
+  <extend-project name="myproject" dest-path="bar" />
+</manifest>
+""")
+      manifest.projects
+
+  def test_extend_project_dest_path_multi_match_path_specified(self):
+    manifest = self.getXmlManifest("""
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="myproject" path="x" />
+  <project name="myproject" path="y" />
+  <extend-project name="myproject" path="x" dest-path="bar" />
+</manifest>
+""")
+    self.assertEqual(len(manifest.projects), 2)
+    if manifest.projects[0].relpath == 'y':
+      self.assertEqual(manifest.projects[1].relpath, 'bar')
+    else:
+      self.assertEqual(manifest.projects[0].relpath, 'bar')
+      self.assertEqual(manifest.projects[1].relpath, 'y')

tests/test_platform_utils.py

@@ -0,0 +1,50 @@
+# Copyright 2021 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for the platform_utils.py module."""
+
+import os
+import tempfile
+import unittest
+
+import platform_utils
+
+
+class RemoveTests(unittest.TestCase):
+  """Check remove() helper."""
+
+  def testMissingOk(self):
+    """Check missing_ok handling."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+      path = os.path.join(tmpdir, 'test')
+
+      # Should not fail.
+      platform_utils.remove(path, missing_ok=True)
+
+      # Should fail.
+      self.assertRaises(OSError, platform_utils.remove, path)
+      self.assertRaises(OSError, platform_utils.remove, path, missing_ok=False)
+
+      # Should not fail if it exists.
+      open(path, 'w').close()
+      platform_utils.remove(path, missing_ok=True)
+      self.assertFalse(os.path.exists(path))
+
+      open(path, 'w').close()
+      platform_utils.remove(path)
+      self.assertFalse(os.path.exists(path))
+
+      open(path, 'w').close()
+      platform_utils.remove(path, missing_ok=False)
+      self.assertFalse(os.path.exists(path))

tests/test_project.py

@@ -16,6 +16,7 @@
 
 import contextlib
 import os
+from pathlib import Path
 import shutil
 import subprocess
 import tempfile
@@ -335,3 +336,52 @@ class LinkFile(CopyLinkTestCase):
     platform_utils.symlink(self.tempdir, dest)
     lf._Link()
     self.assertEqual(os.path.join('git-project', 'foo.txt'), os.readlink(dest))
+
+
+class MigrateWorkTreeTests(unittest.TestCase):
+  """Check _MigrateOldWorkTreeGitDir handling."""
+
+  _SYMLINKS = {
+      'config', 'description', 'hooks', 'info', 'logs', 'objects',
+      'packed-refs', 'refs', 'rr-cache', 'shallow', 'svn',
+  }
+  _FILES = {
+      'COMMIT_EDITMSG', 'FETCH_HEAD', 'HEAD', 'index', 'ORIG_HEAD',
+  }
+
+  @classmethod
+  @contextlib.contextmanager
+  def _simple_layout(cls):
+    """Create a simple repo client checkout to test against."""
+    with tempfile.TemporaryDirectory() as tempdir:
+      tempdir = Path(tempdir)
+
+      gitdir = tempdir / '.repo/projects/src/test.git'
+      gitdir.mkdir(parents=True)
+      cmd = ['git', 'init', '--bare', str(gitdir)]
+      subprocess.check_call(cmd)
+
+      dotgit = tempdir / 'src/test/.git'
+      dotgit.mkdir(parents=True)
+      for name in cls._SYMLINKS:
+        (dotgit / name).symlink_to(f'../../../.repo/projects/src/test.git/{name}')
+      for name in cls._FILES:
+        (dotgit / name).write_text(name)
+
+      subprocess.run(['tree', '-a', str(dotgit)])
+      yield tempdir
+
+  def test_standard(self):
+    """Migrate a standard checkout that we expect."""
+    with self._simple_layout() as tempdir:
+      dotgit = tempdir / 'src/test/.git'
+      project.Project._MigrateOldWorkTreeGitDir(str(dotgit))
+
+      # Make sure the dir was transformed into a symlink.
+      self.assertTrue(dotgit.is_symlink())
+      self.assertEqual(str(dotgit.readlink()), '../../.repo/projects/src/test.git')
+
+      # Make sure files were moved over.
+      gitdir = tempdir / '.repo/projects/src/test.git'
+      for name in self._FILES:
+        self.assertEqual(name, (gitdir / name).read_text())