gc: Introduce new command to remove old projects

When projects are removed from manifest, they are only removed from
worktree and not from .repo/projects and .repo/project-objects. Keeping
data under .repo can be desired if user expects deleted projects to be
restored (e.g. checking out a release branch).

Android has ongoing effort to remove many stale projects and this change
allows users to easily free-up their disk space.

Bug: b/344018971
Bug: 40013312
Change-Id: Id23c7524a88082ee6db908f9fd69dcd5d0c4f681
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/445921
Reviewed-by: Mike Frysinger <vapier@google.com>
Commit-Queue: Josip Sokcevic <sokcevic@chromium.org>
Reviewed-by: Gavin Mak <gavinmak@google.com>
Tested-by: Josip Sokcevic <sokcevic@chromium.org>

color.py

@@ -194,7 +194,7 @@ class Coloring:
         if not opt:
             return _Color(fg, bg, attr)
 
-        v = self._config.GetString("%s.%s" % (self._section, opt))
+        v = self._config.GetString(f"{self._section}.{opt}")
         if v is None:
             return _Color(fg, bg, attr)
 
@@ -210,6 +210,7 @@ class Coloring:
                 if have_fg:
                     bg = a
                 else:
+                    have_fg = True
                     fg = a
             elif is_attr(a):
                 attr = a

command.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import contextlib
 import multiprocessing
 import optparse
 import os
@@ -70,6 +71,14 @@ class Command:
     # migrated subcommands can set it to False.
     MULTI_MANIFEST_SUPPORT = True
 
+    # Shared data across parallel execution workers.
+    _parallel_context = None
+
+    @classmethod
+    def get_parallel_context(cls):
+        assert cls._parallel_context is not None
+        return cls._parallel_context
+
     def __init__(
         self,
         repodir=None,
@@ -242,9 +251,39 @@ class Command:
         """Perform the action, after option parsing is complete."""
         raise NotImplementedError
 
-    @staticmethod
+    @classmethod
+    @contextlib.contextmanager
+    def ParallelContext(cls):
+        """Obtains the context, which is shared to ExecuteInParallel workers.
+
+        Callers can store data in the context dict before invocation of
+        ExecuteInParallel. The dict will then be shared to child workers of
+        ExecuteInParallel.
+        """
+        assert cls._parallel_context is None
+        cls._parallel_context = {}
+        try:
+            yield
+        finally:
+            cls._parallel_context = None
+
+    @classmethod
+    def _InitParallelWorker(cls, context, initializer):
+        cls._parallel_context = context
+        if initializer:
+            initializer()
+
+    @classmethod
     def ExecuteInParallel(
-        jobs, func, inputs, callback, output=None, ordered=False
+        cls,
+        jobs,
+        func,
+        inputs,
+        callback,
+        output=None,
+        ordered=False,
+        chunksize=WORKER_BATCH_SIZE,
+        initializer=None,
     ):
         """Helper for managing parallel execution boiler plate.
 
@@ -269,6 +308,9 @@ class Command:
             output: An output manager. May be progress.Progess or
                 color.Coloring.
             ordered: Whether the jobs should be processed in order.
+            chunksize: The number of jobs processed in batch by parallel
+                workers.
+            initializer: Worker initializer.
 
         Returns:
             The |callback| function's results are returned.
@@ -278,12 +320,16 @@ class Command:
             if len(inputs) == 1 or jobs == 1:
                 return callback(None, output, (func(x) for x in inputs))
             else:
-                with multiprocessing.Pool(jobs) as pool:
+                with multiprocessing.Pool(
+                    jobs,
+                    initializer=cls._InitParallelWorker,
+                    initargs=(cls._parallel_context, initializer),
+                ) as pool:
                     submit = pool.imap if ordered else pool.imap_unordered
                     return callback(
                         pool,
                         output,
-                        submit(func, inputs, chunksize=WORKER_BATCH_SIZE),
+                        submit(func, inputs, chunksize=chunksize),
                     )
         finally:
             if isinstance(output, progress.Progress):
@@ -501,7 +547,3 @@ class MirrorSafeCommand:
     """Command permits itself to run within a mirror, and does not require a
     working directory.
     """
-
-
-class GitcClientCommand:
-    """Command that requires the local client to be a GITC client."""

constraints.txt

@@ -0,0 +1 @@
+black<24

docs/manifest-format.md

@@ -107,11 +107,13 @@ following DTD:
   <!ATTLIST extend-project remote CDATA #IMPLIED>
   <!ATTLIST extend-project dest-branch CDATA #IMPLIED>
   <!ATTLIST extend-project upstream CDATA #IMPLIED>
+  <!ATTLIST extend-project base-rev CDATA #IMPLIED>
 
   <!ELEMENT remove-project EMPTY>
   <!ATTLIST remove-project name     CDATA #IMPLIED>
   <!ATTLIST remove-project path     CDATA #IMPLIED>
   <!ATTLIST remove-project optional CDATA #IMPLIED>
+  <!ATTLIST remove-project base-rev CDATA #IMPLIED>
 
   <!ELEMENT repo-hooks EMPTY>
   <!ATTLIST repo-hooks in-project CDATA #REQUIRED>
@@ -433,6 +435,14 @@ project.  Same syntax as the corresponding element of `project`.
 Attribute `upstream`: If specified, overrides the upstream of the original
 project.  Same syntax as the corresponding element of `project`.
 
+Attribute `base-rev`: If specified, adds a check against the revision
+to be extended. Manifest parse will fail and give a list of mismatch extends
+if the revisions being extended have changed since base-rev was set.
+Intended for use with layered manifests using hash revisions to prevent
+patch branches hiding newer upstream revisions. Also compares named refs
+like branches or tags but is misleading if branches are used as base-rev.
+Same syntax as the corresponding element of `project`.
+
 ### Element annotation
 
 Zero or more annotation elements may be specified as children of a
@@ -496,6 +506,14 @@ name. Logic otherwise behaves like both are specified.
 Attribute `optional`: Set to true to ignore remove-project elements with no
 matching `project` element.
 
+Attribute `base-rev`: If specified, adds a check against the revision
+to be removed. Manifest parse will fail and give a list of mismatch removes
+if the revisions being removed have changed since base-rev was set.
+Intended for use with layered manifests using hash revisions to prevent
+patch branches hiding newer upstream revisions. Also compares named refs
+like branches or tags but is misleading if branches are used as base-rev.
+Same syntax as the corresponding element of `project`.
+
 ### Element repo-hooks
 
 NB: See the [practical documentation](./repo-hooks.md) for using repo hooks.

docs/python-support.md

@@ -33,9 +33,8 @@ you have newer versions installed, your choices are:
 
 *   Modify the [repo launcher]'s shebang to suite your environment.
 *   Download an older version of the [repo launcher] and don't upgrade it.
-    Be aware that there is no guarantee old repo launchers are WILL work with
-    current versions of repo.  Bug reports using old launchers will not be
-    accepted.
+    Be aware that we do not guarantee old repo launchers will work with current
+    versions of repo.  Bug reports using old launchers will not be accepted.
 
 ## When to drop support
 

docs/release-process.md

@@ -96,6 +96,9 @@ If that tag is valid, then repo will warn and use that commit instead.
 
 If that tag cannot be verified, it gives up and forces the user to resolve.
 
+If env variable `REPO_SKIP_SELF_UPDATE` is defined, this will
+bypass the self update algorithm.
+
 ### Force an update
 
 The `repo selfupdate` command can be used to force an immediate update.
@@ -202,7 +205,7 @@ still support them.
 Things in italics are things we used to care about but probably don't anymore.
 
 |   Date   |     EOL      | [Git][rel-g] | [Python][rel-p] | [SSH][rel-o] | [Ubuntu][rel-u] / [Debian][rel-d] | Git | Python | SSH |
-|:--------:|:------------:|:------------:|:---------------:|:------------:|-----------------------------------|-----|--------|-----|
+|:--------:|:------------:|:------------:|:---------------:|:------------:|-----------------------------------|:---:|:------:|:---:|
 | Apr 2008 |              |              |                 | 5.0          |
 | Jun 2008 |              |              |                 | 5.1          |
 | Oct 2008 | *Oct 2013*   |              | 2.6.0           |              | *10.04 Lucid* - 10.10 Maverick / *Squeeze* |
@@ -241,7 +244,7 @@ Things in italics are things we used to care about but probably don't anymore.
 | Feb 2014 | *Dec 2014*   | **1.9.0**    |                 |              | *14.04 Trusty* |
 | Mar 2014 | *Mar 2019*   |              | *3.4.0*         |              | *14.04 Trusty* - 15.10 Wily / *Jessie* |
 | Mar 2014 |              |              |                 | 6.6          | *14.04 Trusty* - 14.10 Utopic |
-| Apr 2014 | *Apr 2022*   |              |                 |              | *14.04 Trusty*       | 1.9.1    | 2.7.5 3.4.0  | 6.6 |
+| Apr 2014 | *Apr 2024*   |              |                 |              | *14.04 Trusty*       | 1.9.1    | 2.7.5 3.4.0  | 6.6 |
 | May 2014 | *Dec 2014*   | 2.0.0        |
 | Aug 2014 | *Dec 2014*   | *2.1.0*      |                 |              | 14.10 Utopic - 15.04 Vivid / *Jessie* |
 | Oct 2014 |              |              |                 | 6.7          | 15.04 Vivid |
@@ -262,7 +265,7 @@ Things in italics are things we used to care about but probably don't anymore.
 | Jan 2016 | *Jul 2017*   | *2.7.0*      |                 |              | *16.04 Xenial* |
 | Feb 2016 |              |              |                 | 7.2          | *16.04 Xenial* |
 | Mar 2016 | *Jul 2017*   | 2.8.0        |
-| Apr 2016 | *Apr 2024*   |              |                 |              | *16.04 Xenial*       | 2.7.4    | 2.7.11 3.5.1 | 7.2 |
+| Apr 2016 | *Apr 2026*   |              |                 |              | *16.04 Xenial*       | 2.7.4    | 2.7.11 3.5.1 | 7.2 |
 | Jun 2016 | *Jul 2017*   | 2.9.0        |                 |              | 16.10 Yakkety |
 | Jul 2016 |              |              |                 | 7.3          | 16.10 Yakkety |
 | Sep 2016 | *Sep 2017*   | 2.10.0       |
@@ -312,14 +315,33 @@ Things in italics are things we used to care about but probably don't anymore.
 | Oct 2020 |              |              |                 |              | 20.10 Groovy         | 2.27.0   | 2.7.18 3.8.6 | 8.3 |
 | Oct 2020 | **Oct 2025** |              | 3.9.0           |              | 21.04 Hirsute / **Bullseye** |
 | Dec 2020 | *Mar 2021*   | 2.30.0       |                 |              | 21.04 Hirsute / **Bullseye** |
-| Mar 2021 |              | 2.31.0       |
-| Mar 2021 |              |              |                 | 8.5          |
+| Mar 2021 |              | 2.31.0       |                 | 8.5          |
 | Apr 2021 |              |              |                 | 8.6          |
 | Apr 2021 | *Jan 2022*   |              |                 |              | 21.04 Hirsute        | 2.30.2   | 2.7.18 3.9.4 | 8.4 |
 | Jun 2021 |              | 2.32.0       |
-| Aug 2021 |              | 2.33.0       |
-| Aug 2021 |              |              |                 | 8.7          |
+| Aug 2021 |              | 2.33.0       |                 | 8.7          |
 | Aug 2021 | **Aug 2026** |              |                 |              | **Debian 11 Bullseye** | 2.30.2 | 2.7.18 3.9.2 | 8.4 |
+| Sep 2021 |              |              |                 | 8.8          |
+| Oct 2021 |              | 2.34.0       | 3.10.0          |              | **22.04 Jammy** |
+| Jan 2022 |              | 2.35.0       |
+| Feb 2022 |              |              |                 | 8.9          | **22.04 Jammy** |
+| Apr 2022 |              | 2.36.0       |                 | 9.0          |
+| Apr 2022 | **Apr 2032** |              |                 |              | **22.04 Jammy**      | 2.34.1 | 2.7.18 3.10.6 | 8.9 |
+| Jun 2022 |              | 2.37.0       |
+| Oct 2022 |              | 2.38.0       |                 | 9.1          |
+| Oct 2022 |              |              | 3.11.0          |              | **Bookworm** |
+| Dec 2022 |              | 2.39.0       |                 |              | **Bookworm** |
+| Feb 2023 |              |              |                 | 9.2          | **Bookworm** |
+| Mar 2023 |              | 2.40.0       |                 | 9.3          |
+| Jun 2023 |              | 2.41.0       |
+| Jun 2023 | **Jun 2028** |              |                 |              | **Debian 12 Bookworm** | 2.39.2 | 3.11.2 | 9.2 |
+| Aug 2023 |              | 2.42.0       |                 | 9.4          |
+| Oct 2023 |              |              | 3.12.0          | 9.5          |
+| Nov 2022 |              | 2.43.0       |
+| Dec 2023 |              |              |                 | 9.6          |
+| Feb 2024 |              | 2.44.0       |
+| Mar 2024 |              |              |                 | 9.7          |
+| Oct 2024 |              |              | 3.13.0          |
 | **Date** |   **EOL**    | **[Git][rel-g]** | **[Python][rel-p]** | **[SSH][rel-o]** | **[Ubuntu][rel-u] / [Debian][rel-d]** | **Git** | **Python** | **SSH** |
 
 
@@ -328,7 +350,7 @@ Things in italics are things we used to care about but probably don't anymore.
 [rel-g]: https://en.wikipedia.org/wiki/Git#Releases
 [rel-o]: https://www.openssh.com/releasenotes.html
 [rel-p]: https://en.wikipedia.org/wiki/History_of_Python#Table_of_versions
-[rel-u]: https://en.wikipedia.org/wiki/Ubuntu_version_history#Table_of_versions
+[rel-u]: https://wiki.ubuntu.com/Releases
 [example announcement]: https://groups.google.com/d/topic/repo-discuss/UGBNismWo1M/discussion
 [repo-discuss@googlegroups.com]: https://groups.google.com/forum/#!forum/repo-discuss
 [go/repo-release]: https://goto.google.com/repo-release

editor.py

@@ -104,9 +104,7 @@ least one of these before using this command.""",  # noqa: E501
             try:
                 rc = subprocess.Popen(args, shell=shell).wait()
             except OSError as e:
-                raise EditorError(
-                    "editor failed, %s: %s %s" % (str(e), editor, path)
-                )
+                raise EditorError(f"editor failed, {str(e)}: {editor} {path}")
             if rc != 0:
                 raise EditorError(
                     "editor failed with exit status %d: %s %s"

error.py

@@ -107,8 +107,8 @@ class GitError(RepoError):
         return self.message
 
 
-class GitcUnsupportedError(RepoExitError):
-    """Gitc no longer supported."""
+class GitAuthError(RepoExitError):
+    """Cannot talk to remote due to auth issue."""
 
 
 class UploadError(RepoError):

event_log.py

@@ -168,8 +168,10 @@ class EventLog:
                 f.write("\n")
 
 
-# An integer id that is unique across this invocation of the program.
-_EVENT_ID = multiprocessing.Value("i", 1)
+# An integer id that is unique across this invocation of the program, to be set
+# by the first Add event. We can't set it here since it results in leaked
+# resources (see: https://issues.gerritcodereview.com/353656374).
+_EVENT_ID = None
 
 
 def _NextEventId():
@@ -178,6 +180,12 @@ def _NextEventId():
     Returns:
         A unique, to this invocation of the program, integer id.
     """
+    global _EVENT_ID
+    if _EVENT_ID is None:
+        # There is a small chance of race condition - two parallel processes
+        # setting up _EVENT_ID. However, we expect TASK_COMMAND to happen before
+        # mp kicks in.
+        _EVENT_ID = multiprocessing.Value("i", 1)
     with _EVENT_ID.get_lock():
         val = _EVENT_ID.value
         _EVENT_ID.value += 1

git_command.py

@@ -33,17 +33,6 @@ from wrapper import Wrapper
 
 
 GIT = "git"
-# NB: These do not need to be kept in sync with the repo launcher script.
-# These may be much newer as it allows the repo launcher to roll between
-# different repo releases while source versions might require a newer git.
-#
-# The soft version is when we start warning users that the version is old and
-# we'll be dropping support for it.  We'll refuse to work with versions older
-# than the hard version.
-#
-# git-1.7 is in (EOL) Ubuntu Precise.  git-1.9 is in Ubuntu Trusty.
-MIN_GIT_VERSION_SOFT = (1, 9, 1)
-MIN_GIT_VERSION_HARD = (1, 7, 2)
 GIT_DIR = "GIT_DIR"
 
 LAST_GITDIR = None
@@ -135,6 +124,8 @@ def GetEventTargetPath():
     if retval == 0:
         # Strip trailing carriage-return in path.
         path = p.stdout.rstrip("\n")
+        if path == "":
+            return None
     elif retval != 1:
         # `git config --get` is documented to produce an exit status of `1`
         # if the requested variable is not present in the configuration.
@@ -196,12 +187,10 @@ class UserAgent:
     def git(self):
         """The UA when running git."""
         if self._git_ua is None:
-            self._git_ua = "git/%s (%s) git-repo/%s" % (
-                git.version_tuple().full,
-                self.os,
-                RepoSourceVersion(),
+            self._git_ua = (
+                f"git/{git.version_tuple().full} ({self.os}) "
+                f"git-repo/{RepoSourceVersion()}"
             )
-
         return self._git_ua
 
 
@@ -216,7 +205,7 @@ def git_require(min_version, fail=False, msg=""):
         need = ".".join(map(str, min_version))
         if msg:
             msg = " for " + msg
-        error_msg = "fatal: git %s or later required%s" % (need, msg)
+        error_msg = f"fatal: git {need} or later required{msg}"
         logger.error(error_msg)
         raise GitRequireError(error_msg)
     return False
@@ -243,7 +232,7 @@ def _build_env(
         env["GIT_SSH"] = ssh_proxy.proxy
         env["GIT_SSH_VARIANT"] = "ssh"
     if "http_proxy" in env and "darwin" == sys.platform:
-        s = "'http.proxy=%s'" % (env["http_proxy"],)
+        s = f"'http.proxy={env['http_proxy']}'"
         p = env.get("GIT_CONFIG_PARAMETERS")
         if p is not None:
             s = p + " " + s
@@ -324,12 +313,15 @@ class GitCommand:
             cwd = None
         command_name = cmdv[0]
         command.append(command_name)
-        # Need to use the --progress flag for fetch/clone so output will be
-        # displayed as by default git only does progress output if stderr is a
-        # TTY.
-        if sys.stderr.isatty() and command_name in ("fetch", "clone"):
-            if "--progress" not in cmdv and "--quiet" not in cmdv:
-                command.append("--progress")
+
+        if command_name in ("fetch", "clone"):
+            env["GIT_TERMINAL_PROMPT"] = "0"
+            # Need to use the --progress flag for fetch/clone so output will be
+            # displayed as by default git only does progress output if stderr is
+            # a TTY.
+            if sys.stderr.isatty():
+                if "--progress" not in cmdv and "--quiet" not in cmdv:
+                    command.append("--progress")
         command.extend(cmdv[1:])
 
         event_log = (
@@ -468,7 +460,7 @@ class GitCommand:
                 )
             except Exception as e:
                 raise GitPopenCommandError(
-                    message="%s: %s" % (command[1], e),
+                    message=f"{command[1]}: {e}",
                     project=self.project.name if self.project else None,
                     command_args=self.cmdv,
                 )

git_config.py

@@ -418,7 +418,7 @@ class GitConfig:
         if p.Wait() == 0:
             return p.stdout
         else:
-            raise GitError("git config %s: %s" % (str(args), p.stderr))
+            raise GitError(f"git config {str(args)}: {p.stderr}")
 
 
 class RepoConfig(GitConfig):
@@ -651,13 +651,11 @@ class Remote:
                             userEmail, host, port
                         )
                 except urllib.error.HTTPError as e:
-                    raise UploadError("%s: %s" % (self.review, str(e)))
+                    raise UploadError(f"{self.review}: {str(e)}")
                 except urllib.error.URLError as e:
-                    raise UploadError("%s: %s" % (self.review, str(e)))
+                    raise UploadError(f"{self.review}: {str(e)}")
                 except http.client.HTTPException as e:
-                    raise UploadError(
-                        "%s: %s" % (self.review, e.__class__.__name__)
-                    )
+                    raise UploadError(f"{self.review}: {e.__class__.__name__}")
 
                 REVIEW_CACHE[u] = self._review_url
         return self._review_url + self.projectname
@@ -666,7 +664,7 @@ class Remote:
         username = self._config.GetString("review.%s.username" % self.review)
         if username is None:
             username = userEmail.split("@")[0]
-        return "ssh://%s@%s:%s/" % (username, host, port)
+        return f"ssh://{username}@{host}:{port}/"
 
     def ToLocal(self, rev):
         """Convert a remote revision string to something we have locally."""
@@ -715,11 +713,11 @@ class Remote:
         self._Set("fetch", list(map(str, self.fetch)))
 
     def _Set(self, key, value):
-        key = "remote.%s.%s" % (self.name, key)
+        key = f"remote.{self.name}.{key}"
         return self._config.SetString(key, value)
 
     def _Get(self, key, all_keys=False):
-        key = "remote.%s.%s" % (self.name, key)
+        key = f"remote.{self.name}.{key}"
         return self._config.GetString(key, all_keys=all_keys)
 
 
@@ -762,11 +760,11 @@ class Branch:
                     fd.write("\tmerge = %s\n" % self.merge)
 
     def _Set(self, key, value):
-        key = "branch.%s.%s" % (self.name, key)
+        key = f"branch.{self.name}.{key}"
         return self._config.SetString(key, value)
 
     def _Get(self, key, all_keys=False):
-        key = "branch.%s.%s" % (self.name, key)
+        key = f"branch.{self.name}.{key}"
         return self._config.GetString(key, all_keys=all_keys)
 
 

git_superproject.py

@@ -69,9 +69,9 @@ class UpdateProjectsResult(NamedTuple):
 class Superproject:
     """Get commit ids from superproject.
 
-    Initializes a local copy of a superproject for the manifest. This allows
-    lookup of commit ids for all projects. It contains _project_commit_ids which
-    is a dictionary with project/commit id entries.
+    Initializes a bare local copy of a superproject for the manifest. This
+    allows lookup of commit ids for all projects. It contains
+    _project_commit_ids which is a dictionary with project/commit id entries.
     """
 
     def __init__(
@@ -235,7 +235,8 @@ class Superproject:
         p = GitCommand(
             None,
             cmd,
-            cwd=self._work_git,
+            gitdir=self._work_git,
+            bare=True,
             capture_stdout=True,
             capture_stderr=True,
         )
@@ -271,7 +272,8 @@ class Superproject:
         p = GitCommand(
             None,
             cmd,
-            cwd=self._work_git,
+            gitdir=self._work_git,
+            bare=True,
             capture_stdout=True,
             capture_stderr=True,
         )
@@ -305,8 +307,6 @@ class Superproject:
             )
             return SyncResult(False, False)
 
-        _PrintBetaNotice()
-
         should_exit = True
         if not self._remote_url:
             self._LogWarning(
@@ -450,16 +450,6 @@ class Superproject:
         return UpdateProjectsResult(manifest_path, False)
 
 
-@functools.lru_cache(maxsize=10)
-def _PrintBetaNotice():
-    """Print the notice of beta status."""
-    print(
-        "NOTICE: --use-superproject is in beta; report any issues to the "
-        "address described in `repo version`",
-        file=sys.stderr,
-    )
-
-
 @functools.lru_cache(maxsize=None)
 def _UseSuperprojectFromConfiguration():
     """Returns the user choice of whether to use superproject."""

git_trace2_event_log_base.py

@@ -38,6 +38,8 @@ import tempfile
 import threading
 
 
+# Timeout when sending events via socket (applies to connect, send)
+SOCK_TIMEOUT = 0.5  # in seconds
 # BaseEventLog __init__ Counter that is consistent within the same process
 p_init_count = 0
 
@@ -76,9 +78,8 @@ class BaseEventLog:
         # Save both our sid component and the complete sid.
         # We use our sid component (self._sid) as the unique filename prefix and
         # the full sid (self._full_sid) in the log itself.
-        self._sid = "repo-%s-P%08x" % (
-            self.start.strftime("%Y%m%dT%H%M%SZ"),
-            os.getpid(),
+        self._sid = (
+            f"repo-{self.start.strftime('%Y%m%dT%H%M%SZ')}-P{os.getpid():08x}"
         )
 
         if add_init_count:
@@ -129,10 +130,10 @@ class BaseEventLog:
             "time": datetime.datetime.now(datetime.timezone.utc).isoformat(),
         }
 
-    def StartEvent(self):
+    def StartEvent(self, argv):
         """Append a 'start' event to the current log."""
         start_event = self._CreateEventDict("start")
-        start_event["argv"] = sys.argv
+        start_event["argv"] = argv
         self._log.append(start_event)
 
     def ExitEvent(self, result):
@@ -158,9 +159,11 @@ class BaseEventLog:
             name: Name of the primary command (ex: repo, git)
             subcommands: List of the sub-commands (ex: version, init, sync)
         """
-        command_event = self._CreateEventDict("command")
+        command_event = self._CreateEventDict("cmd_name")
+        name = f"{name}-"
+        name += "-".join(subcommands)
         command_event["name"] = name
-        command_event["subcommands"] = subcommands
+        command_event["hierarchy"] = name
         self._log.append(command_event)
 
     def LogConfigEvents(self, config, event_dict_name):
@@ -297,6 +300,7 @@ class BaseEventLog:
                     with socket.socket(
                         socket.AF_UNIX, socket.SOCK_STREAM
                     ) as sock:
+                        sock.settimeout(SOCK_TIMEOUT)
                         sock.connect(path)
                         self._WriteLog(sock.sendall)
                     return f"af_unix:stream:{path}"

hooks.py

@@ -180,7 +180,7 @@ class RepoHook:
                 abort_if_user_denies was passed to the consturctor.
         """
         hooks_config = self._hooks_project.config
-        git_approval_key = "repo.hooks.%s.%s" % (self._hook_type, subkey)
+        git_approval_key = f"repo.hooks.{self._hook_type}.{subkey}"
 
         # Get the last value that the user approved for this hook; may be None.
         old_val = hooks_config.GetString(git_approval_key)
@@ -193,7 +193,7 @@ class RepoHook:
             else:
                 # Give the user a reason why we're prompting, since they last
                 # told us to "never ask again".
-                prompt = "WARNING: %s\n\n" % (changed_prompt,)
+                prompt = f"WARNING: {changed_prompt}\n\n"
         else:
             prompt = ""
 
@@ -241,9 +241,8 @@ class RepoHook:
         return self._CheckForHookApprovalHelper(
             "approvedmanifest",
             self._manifest_url,
-            "Run hook scripts from %s" % (self._manifest_url,),
-            "Manifest URL has changed since %s was allowed."
-            % (self._hook_type,),
+            f"Run hook scripts from {self._manifest_url}",
+            f"Manifest URL has changed since {self._hook_type} was allowed.",
         )
 
     def _CheckForHookApprovalHash(self):
@@ -262,7 +261,7 @@ class RepoHook:
             "approvedhash",
             self._GetHash(),
             prompt % (self._GetMustVerb(), self._script_fullpath),
-            "Scripts have changed since %s was allowed." % (self._hook_type,),
+            f"Scripts have changed since {self._hook_type} was allowed.",
         )
 
     @staticmethod

hooks/commit-msg

@@ -1,5 +1,8 @@
 #!/bin/sh
-# From Gerrit Code Review 3.6.1 c67916dbdc07555c44e32a68f92ffc484b9b34f0
+# DO NOT EDIT THIS FILE
+# All updates should be sent upstream: https://gerrit.googlesource.com/gerrit/
+# This is synced from commit: 62f5bbea67f6dafa6e22a601a0c298214c510caf
+# DO NOT EDIT THIS FILE
 #
 # Part of Gerrit Code Review (https://www.gerritcodereview.com/)
 #
@@ -31,14 +34,20 @@ if test ! -f "$1" ; then
 fi
 
 # Do not create a change id if requested
-if test "false" = "$(git config --bool --get gerrit.createChangeId)" ; then
-  exit 0
-fi
+case "$(git config --get gerrit.createChangeId)" in
+  false)
+    exit 0
+    ;;
+  always)
+    ;;
+  *)
+    # Do not create a change id for squash/fixup commits.
+    if head -n1 "$1" | LC_ALL=C grep -q '^[a-z][a-z]*! '; then
+      exit 0
+    fi
+    ;;
+esac
 
-# Do not create a change id for squash commits.
-if head -n1 "$1" | grep -q '^squash! '; then
-  exit 0
-fi
 
 if git rev-parse --verify HEAD >/dev/null 2>&1; then
   refhash="$(git rev-parse HEAD)"
@@ -51,7 +60,7 @@ dest="$1.tmp.${random}"
 
 trap 'rm -f "$dest" "$dest-2"' EXIT
 
-if ! git stripspace --strip-comments < "$1" > "${dest}" ; then
+if ! cat "$1" | sed -e '/>8/q' | git stripspace --strip-comments > "${dest}" ; then
    echo "cannot strip comments from $1"
    exit 1
 fi
@@ -65,7 +74,7 @@ reviewurl="$(git config --get gerrit.reviewUrl)"
 if test -n "${reviewurl}" ; then
   token="Link"
   value="${reviewurl%/}/id/I$random"
-  pattern=".*/id/I[0-9a-f]\{40\}$"
+  pattern=".*/id/I[0-9a-f]\{40\}"
 else
   token="Change-Id"
   value="I$random"
@@ -92,7 +101,7 @@ fi
 # Avoid the --where option which only appeared in Git 2.15
 if ! git -c trailer.where=before interpret-trailers \
          --trailer "Signed-off-by: $token: $value" < "$dest-2" |
-     sed -re "s/^Signed-off-by: ($token: )/\1/" \
+     sed -e "s/^Signed-off-by: \($token: \)/\1/" \
          -e "/^Signed-off-by: SENTINEL/d" > "$dest" ; then
   echo "cannot insert $token line in $1"
   exit 1

hooks/pre-auto-gc

@@ -1,33 +1,25 @@
 #!/bin/sh
+# DO NOT EDIT THIS FILE
+# All updates should be sent upstream: https://github.com/git/git
+# This is synced from commit: 00e10ef10e161a913893b8cb33aa080d4ca5baa6
+# DO NOT EDIT THIS FILE
 #
 # An example hook script to verify if you are on battery, in case you
-# are running Windows, Linux or OS X. Called by git-gc --auto with no
-# arguments. The hook should exit with non-zero status after issuing an
-# appropriate message if it wants to stop the auto repacking.
-
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
+# are running Linux or OS X. Called by git-gc --auto with no arguments.
+# The hook should exit with non-zero status after issuing an appropriate
+# message if it wants to stop the auto repacking.
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# This hook is stored in the contrib/hooks directory. Your distribution
+# may have put this somewhere else. If you want to use this hook, you
+# should make this script executable then link to it in the repository
+# you would like to use it in.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-
-if uname -s | grep -q "_NT-"
-then
-	if test -x $SYSTEMROOT/System32/Wbem/wmic
-	then
-		STATUS=$(wmic path win32_battery get batterystatus /format:list | tr -d '\r\n')
-		[ "$STATUS" = "BatteryStatus=2" ] && exit 0 || exit 1
-	fi
-	exit 0
-fi
+# For example, if the hook is stored in
+# /usr/share/git-core/contrib/hooks/pre-auto-gc-battery:
+#
+# cd /path/to/your/repository.git
+# ln -sf /usr/share/git-core/contrib/hooks/pre-auto-gc-battery \
+#	hooks/pre-auto-gc
 
 if test -x /sbin/on_ac_power && (/sbin/on_ac_power;test $? -ne 1)
 then
@@ -48,11 +40,6 @@ elif test -x /usr/bin/pmset && /usr/bin/pmset -g batt |
 	grep -q "drawing from 'AC Power'"
 then
 	exit 0
-elif test -d /sys/bus/acpi/drivers/battery && test 0 = \
-  "$(find /sys/bus/acpi/drivers/battery/ -type l | wc -l)";
-then
-	# No battery exists.
-	exit 0
 fi
 
 echo "Auto packing deferred; not on AC"

main.py

@@ -45,9 +45,9 @@ from command import InteractiveCommand
 from command import MirrorSafeCommand
 from editor import Editor
 from error import DownloadError
-from error import GitcUnsupportedError
 from error import InvalidProjectGroupsError
 from error import ManifestInvalidRevisionError
+from error import ManifestParseError
 from error import NoManifestException
 from error import NoSuchProjectError
 from error import RepoChangedException
@@ -198,9 +198,8 @@ class _Repo:
         if short:
             commands = " ".join(sorted(self.commands))
             wrapped_commands = textwrap.wrap(commands, width=77)
-            print(
-                "Available commands:\n  %s" % ("\n  ".join(wrapped_commands),)
-            )
+            help_commands = "".join(f"\n  {x}" for x in wrapped_commands)
+            print(f"Available commands:{help_commands}")
             print("\nRun `repo help <command>` for command-specific details.")
             print("Bug reports:", Wrapper().BUG_URL)
         else:
@@ -236,7 +235,7 @@ class _Repo:
         if name in self.commands:
             return name, []
 
-        key = "alias.%s" % (name,)
+        key = f"alias.{name}"
         alias = RepoConfig.ForRepository(self.repodir).GetString(key)
         if alias is None:
             alias = RepoConfig.ForUser().GetString(key)
@@ -270,10 +269,14 @@ class _Repo:
             self._PrintHelp(short=True)
             return 1
 
-        run = lambda: self._RunLong(name, gopts, argv) or 0
+        git_trace2_event_log = EventLog()
+        run = (
+            lambda: self._RunLong(name, gopts, argv, git_trace2_event_log) or 0
+        )
         with Trace(
-            "starting new command: %s",
+            "starting new command: %s [sid=%s]",
             ", ".join([name] + argv),
+            git_trace2_event_log.full_sid,
             first_trace=True,
         ):
             if gopts.trace_python:
@@ -290,12 +293,11 @@ class _Repo:
                 result = run()
         return result
 
-    def _RunLong(self, name, gopts, argv):
+    def _RunLong(self, name, gopts, argv, git_trace2_event_log):
         """Execute the (longer running) requested subcommand."""
         result = 0
         SetDefaultColoring(gopts.color)
 
-        git_trace2_event_log = EventLog()
         outer_client = RepoClient(self.repodir)
         repo_client = outer_client
         if gopts.submanifest_path:
@@ -305,10 +307,6 @@ class _Repo:
                 outer_client=outer_client,
             )
 
-        if Wrapper().gitc_parse_clientdir(os.getcwd()):
-            logger.error("GITC is not supported.")
-            raise GitcUnsupportedError()
-
         try:
             cmd = self.commands[name](
                 repodir=self.repodir,
@@ -354,7 +352,7 @@ class _Repo:
         start = time.time()
         cmd_event = cmd.event_log.Add(name, event_log.TASK_COMMAND, start)
         cmd.event_log.SetParent(cmd_event)
-        git_trace2_event_log.StartEvent()
+        git_trace2_event_log.StartEvent(["repo", name] + argv)
         git_trace2_event_log.CommandEvent(name="repo", subcommands=[name])
 
         def execute_command_helper():
@@ -422,7 +420,7 @@ class _Repo:
                         error_info = json.dumps(
                             {
                                 "ErrorType": type(error).__name__,
-                                "Project": project,
+                                "Project": str(project),
                                 "Message": str(error),
                             }
                         )
@@ -440,6 +438,7 @@ class _Repo:
         except (
             DownloadError,
             ManifestInvalidRevisionError,
+            ManifestParseError,
             NoManifestException,
         ) as e:
             logger.error("error: in `%s`: %s", " ".join([name] + argv), e)

man/repo-gc.1

@@ -1,21 +1,24 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "July 2022" "repo gitc-delete" "Repo Manual"
+.TH REPO "1" "December 2024" "repo gc" "Repo Manual"
 .SH NAME
-repo \- repo gitc-delete - manual page for repo gitc-delete
+repo \- repo gc - manual page for repo gc
 .SH SYNOPSIS
 .B repo
-\fI\,gitc-delete\/\fR
+\fI\,gc\/\fR
 .SH DESCRIPTION
 Summary
 .PP
-Delete a GITC Client.
+Cleaning up internal repo state.
 .SH OPTIONS
 .TP
 \fB\-h\fR, \fB\-\-help\fR
 show this help message and exit
 .TP
-\fB\-f\fR, \fB\-\-force\fR
-force the deletion (no prompt)
+\fB\-n\fR, \fB\-\-dry\-run\fR
+do everything except actually delete
+.TP
+\fB\-y\fR, \fB\-\-yes\fR
+answer yes to all safe prompts
 .SS Logging options:
 .TP
 \fB\-v\fR, \fB\-\-verbose\fR
@@ -37,8 +40,4 @@ only operate on this (sub)manifest
 \fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
 operate on this manifest and its submanifests
 .PP
-Run `repo help gitc\-delete` to view the detailed manual.
-.SH DETAILS
-.PP
-This subcommand deletes the current GITC client, deleting the GITC manifest and
-all locally downloaded sources.
+Run `repo help gc` to view the detailed manual.

man/repo-gitc-init.1

@@ -1,175 +0,0 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "October 2022" "repo gitc-init" "Repo Manual"
-.SH NAME
-repo \- repo gitc-init - manual page for repo gitc-init
-.SH SYNOPSIS
-.B repo
-\fI\,gitc-init \/\fR[\fI\,options\/\fR] [\fI\,client name\/\fR]
-.SH DESCRIPTION
-Summary
-.PP
-Initialize a GITC Client.
-.SH OPTIONS
-.TP
-\fB\-h\fR, \fB\-\-help\fR
-show this help message and exit
-.SS Logging options:
-.TP
-\fB\-v\fR, \fB\-\-verbose\fR
-show all output
-.TP
-\fB\-q\fR, \fB\-\-quiet\fR
-only show errors
-.SS Manifest options:
-.TP
-\fB\-u\fR URL, \fB\-\-manifest\-url\fR=\fI\,URL\/\fR
-manifest repository location
-.TP
-\fB\-b\fR REVISION, \fB\-\-manifest\-branch\fR=\fI\,REVISION\/\fR
-manifest branch or revision (use HEAD for default)
-.TP
-\fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
-initial manifest file
-.TP
-\fB\-g\fR GROUP, \fB\-\-groups\fR=\fI\,GROUP\/\fR
-restrict manifest projects to ones with specified
-group(s) [default|all|G1,G2,G3|G4,\-G5,\-G6]
-.TP
-\fB\-p\fR PLATFORM, \fB\-\-platform\fR=\fI\,PLATFORM\/\fR
-restrict manifest projects to ones with a specified
-platform group [auto|all|none|linux|darwin|...]
-.TP
-\fB\-\-submodules\fR
-sync any submodules associated with the manifest repo
-.TP
-\fB\-\-standalone\-manifest\fR
-download the manifest as a static file rather then
-create a git checkout of the manifest repo
-.TP
-\fB\-\-manifest\-depth\fR=\fI\,DEPTH\/\fR
-create a shallow clone of the manifest repo with given
-depth (0 for full clone); see git clone (default: 0)
-.SS Manifest (only) checkout options:
-.TP
-\fB\-\-current\-branch\fR
-fetch only current manifest branch from server
-(default)
-.TP
-\fB\-\-no\-current\-branch\fR
-fetch all manifest branches from server
-.TP
-\fB\-\-tags\fR
-fetch tags in the manifest
-.TP
-\fB\-\-no\-tags\fR
-don't fetch tags in the manifest
-.SS Checkout modes:
-.TP
-\fB\-\-mirror\fR
-create a replica of the remote repositories rather
-than a client working directory
-.TP
-\fB\-\-archive\fR
-checkout an archive instead of a git repository for
-each project. See git archive.
-.TP
-\fB\-\-worktree\fR
-use git\-worktree to manage projects
-.SS Project checkout optimizations:
-.TP
-\fB\-\-reference\fR=\fI\,DIR\/\fR
-location of mirror directory
-.TP
-\fB\-\-dissociate\fR
-dissociate from reference mirrors after clone
-.TP
-\fB\-\-depth\fR=\fI\,DEPTH\/\fR
-create a shallow clone with given depth; see git clone
-.TP
-\fB\-\-partial\-clone\fR
-perform partial clone (https://gitscm.com/docs/gitrepositorylayout#_code_partialclone_code)
-.TP
-\fB\-\-no\-partial\-clone\fR
-disable use of partial clone (https://gitscm.com/docs/gitrepositorylayout#_code_partialclone_code)
-.TP
-\fB\-\-partial\-clone\-exclude\fR=\fI\,PARTIAL_CLONE_EXCLUDE\/\fR
-exclude the specified projects (a comma\-delimited
-project names) from partial clone (https://gitscm.com/docs/gitrepositorylayout#_code_partialclone_code)
-.TP
-\fB\-\-clone\-filter\fR=\fI\,CLONE_FILTER\/\fR
-filter for use with \fB\-\-partial\-clone\fR [default:
-blob:none]
-.TP
-\fB\-\-use\-superproject\fR
-use the manifest superproject to sync projects;
-implies \fB\-c\fR
-.TP
-\fB\-\-no\-use\-superproject\fR
-disable use of manifest superprojects
-.TP
-\fB\-\-clone\-bundle\fR
-enable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS (default if
-not \fB\-\-partial\-clone\fR)
-.TP
-\fB\-\-no\-clone\-bundle\fR
-disable use of \fI\,/clone.bundle\/\fP on HTTP/HTTPS (default if
-\fB\-\-partial\-clone\fR)
-.TP
-\fB\-\-git\-lfs\fR
-enable Git LFS support
-.TP
-\fB\-\-no\-git\-lfs\fR
-disable Git LFS support
-.SS repo Version options:
-.TP
-\fB\-\-repo\-url\fR=\fI\,URL\/\fR
-repo repository location ($REPO_URL)
-.TP
-\fB\-\-repo\-rev\fR=\fI\,REV\/\fR
-repo branch or revision ($REPO_REV)
-.TP
-\fB\-\-no\-repo\-verify\fR
-do not verify repo source code
-.SS Other options:
-.TP
-\fB\-\-config\-name\fR
-Always prompt for name/e\-mail
-.SS GITC options:
-.TP
-\fB\-f\fR MANIFEST_FILE, \fB\-\-manifest\-file\fR=\fI\,MANIFEST_FILE\/\fR
-Optional manifest file to use for this GITC client.
-.TP
-\fB\-c\fR GITC_CLIENT, \fB\-\-gitc\-client\fR=\fI\,GITC_CLIENT\/\fR
-Name of the gitc_client instance to create or modify.
-.SS Multi\-manifest:
-.TP
-\fB\-\-outer\-manifest\fR
-operate starting at the outermost manifest
-.TP
-\fB\-\-no\-outer\-manifest\fR
-do not operate on outer manifests
-.TP
-\fB\-\-this\-manifest\-only\fR
-only operate on this (sub)manifest
-.TP
-\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
-operate on this manifest and its submanifests
-.PP
-Run `repo help gitc\-init` to view the detailed manual.
-.SH DETAILS
-.PP
-The 'repo gitc\-init' command is ran to initialize a new GITC client for use with
-the GITC file system.
-.PP
-This command will setup the client directory, initialize repo, just like repo
-init does, and then downloads the manifest collection and installs it in the
-\&.repo/directory of the GITC client.
-.PP
-Once this is done, a GITC manifest is generated by pulling the HEAD SHA for each
-project and generates the properly formatted XML file and installs it as
-\&.manifest in the GITC client directory.
-.PP
-The \fB\-c\fR argument is required to specify the GITC client name.
-.PP
-The optional \fB\-f\fR argument can be used to specify the manifest file to use for
-this GITC client.

man/repo-init.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "October 2022" "repo init" "Repo Manual"
+.TH REPO "1" "September 2024" "repo init" "Repo Manual"
 .SH NAME
 repo \- repo init - manual page for repo init
 .SH SYNOPSIS
@@ -28,6 +28,11 @@ manifest repository location
 \fB\-b\fR REVISION, \fB\-\-manifest\-branch\fR=\fI\,REVISION\/\fR
 manifest branch or revision (use HEAD for default)
 .TP
+\fB\-\-manifest\-upstream\-branch\fR=\fI\,BRANCH\/\fR
+when a commit is provided to \fB\-\-manifest\-branch\fR, this
+is the name of the git ref in which the commit can be
+found
+.TP
 \fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
 initial manifest file
 .TP
@@ -163,6 +168,10 @@ The optional \fB\-b\fR argument can be used to select the manifest branch to che
 and use. If no branch is specified, the remote's default branch is used. This is
 equivalent to using \fB\-b\fR HEAD.
 .PP
+The optional \fB\-\-manifest\-upstream\-branch\fR argument can be used when a commit is
+provided to \fB\-\-manifest\-branch\fR (or \fB\-b\fR), to specify the name of the git ref in
+which the commit can be found.
+.PP
 The optional \fB\-m\fR argument can be used to specify an alternate manifest to be
 used. If no manifest is specified, the manifest default.xml will be used.
 .PP

man/repo-manifest.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "October 2022" "repo manifest" "Repo Manual"
+.TH REPO "1" "December 2024" "repo manifest" "Repo Manual"
 .SH NAME
 repo \- repo manifest - manual page for repo manifest
 .SH SYNOPSIS
@@ -192,10 +192,13 @@ CDATA #IMPLIED>
 <!ATTLIST extend\-project remote CDATA #IMPLIED>
 <!ATTLIST extend\-project dest\-branch CDATA #IMPLIED>
 <!ATTLIST extend\-project upstream CDATA #IMPLIED>
+<!ATTLIST extend\-project base\-rev CDATA #IMPLIED>
 .IP
 <!ELEMENT remove\-project EMPTY>
-<!ATTLIST remove\-project name  CDATA #REQUIRED>
-<!ATTLIST remove\-project optional  CDATA #IMPLIED>
+<!ATTLIST remove\-project name     CDATA #IMPLIED>
+<!ATTLIST remove\-project path     CDATA #IMPLIED>
+<!ATTLIST remove\-project optional CDATA #IMPLIED>
+<!ATTLIST remove\-project base\-rev CDATA #IMPLIED>
 .IP
 <!ELEMENT repo\-hooks EMPTY>
 <!ATTLIST repo\-hooks in\-project CDATA #REQUIRED>
@@ -210,8 +213,9 @@ CDATA #IMPLIED>
 <!ATTLIST contactinfo bugurl  CDATA #REQUIRED>
 .IP
 <!ELEMENT include EMPTY>
-<!ATTLIST include name   CDATA #REQUIRED>
-<!ATTLIST include groups CDATA #IMPLIED>
+<!ATTLIST include name     CDATA #REQUIRED>
+<!ATTLIST include groups   CDATA #IMPLIED>
+<!ATTLIST include revision CDATA #IMPLIED>
 .PP
 ]>
 ```
@@ -493,6 +497,14 @@ project. Same syntax as the corresponding element of `project`.
 Attribute `upstream`: If specified, overrides the upstream of the original
 project. Same syntax as the corresponding element of `project`.
 .PP
+Attribute `base\-rev`: If specified, adds a check against the revision to be
+extended. Manifest parse will fail and give a list of mismatch extends if the
+revisions being extended have changed since base\-rev was set. Intended for use
+with layered manifests using hash revisions to prevent patch branches hiding
+newer upstream revisions. Also compares named refs like branches or tags but is
+misleading if branches are used as base\-rev. Same syntax as the corresponding
+element of `project`.
+.PP
 Element annotation
 .PP
 Zero or more annotation elements may be specified as children of a project or
@@ -533,16 +545,35 @@ the repo client.
 .PP
 Element remove\-project
 .PP
-Deletes the named project from the internal manifest table, possibly allowing a
+Deletes a project from the internal manifest table, possibly allowing a
 subsequent project element in the same manifest file to replace the project with
 a different source.
 .PP
 This element is mostly useful in a local manifest file, where the user can
 remove a project, and possibly replace it with their own definition.
 .PP
+The project `name` or project `path` can be used to specify the remove target
+meaning one of them is required. If only name is specified, all projects with
+that name are removed.
+.PP
+If both name and path are specified, only projects with the same name and path
+are removed, meaning projects with the same name but in other locations are
+kept.
+.PP
+If only path is specified, a matching project is removed regardless of its name.
+Logic otherwise behaves like both are specified.
+.PP
 Attribute `optional`: Set to true to ignore remove\-project elements with no
 matching `project` element.
 .PP
+Attribute `base\-rev`: If specified, adds a check against the revision to be
+removed. Manifest parse will fail and give a list of mismatch removes if the
+revisions being removed have changed since base\-rev was set. Intended for use
+with layered manifests using hash revisions to prevent patch branches hiding
+newer upstream revisions. Also compares named refs like branches or tags but is
+misleading if branches are used as base\-rev. Same syntax as the corresponding
+element of `project`.
+.PP
 Element repo\-hooks
 .PP
 NB: See the [practical documentation](./repo\-hooks.md) for using repo hooks.
@@ -608,7 +639,10 @@ included manifest belong. This appends and recurses, meaning all projects in
 included manifests carry all parent include groups. Same syntax as the
 corresponding element of `project`.
 .PP
-Local Manifests 
+Attribute `revision`: Name of a Git branch (e.g. `main` or `refs/heads/main`)
+default to which all projects in the included manifest belong.
+.PP
+Local Manifests
 .PP
 Additional remotes and projects may be added through local manifest files stored
 in `$TOP_DIR/.repo/local_manifests/*.xml`.

man/repo-smartsync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "November 2022" "repo smartsync" "Repo Manual"
+.TH REPO "1" "September 2024" "repo smartsync" "Repo Manual"
 .SH NAME
 repo \- repo smartsync - manual page for repo smartsync
 .SH SYNOPSIS
@@ -37,11 +37,20 @@ overwrite an existing git directory if it needs to
 point to a different object directory. WARNING: this
 may cause loss of data
 .TP
+\fB\-\-force\-checkout\fR
+force checkout even if it results in throwing away
+uncommitted modifications. WARNING: this may cause
+loss of data
+.TP
 \fB\-\-force\-remove\-dirty\fR
 force remove projects with uncommitted modifications
 if projects no longer exist in the manifest. WARNING:
 this may cause loss of data
 .TP
+\fB\-\-rebase\fR
+rebase local commits regardless of whether they are
+published
+.TP
 \fB\-l\fR, \fB\-\-local\-only\fR
 only update working tree, don't fetch
 .TP

man/repo-sync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "November 2022" "repo sync" "Repo Manual"
+.TH REPO "1" "September 2024" "repo sync" "Repo Manual"
 .SH NAME
 repo \- repo sync - manual page for repo sync
 .SH SYNOPSIS
@@ -37,11 +37,20 @@ overwrite an existing git directory if it needs to
 point to a different object directory. WARNING: this
 may cause loss of data
 .TP
+\fB\-\-force\-checkout\fR
+force checkout even if it results in throwing away
+uncommitted modifications. WARNING: this may cause
+loss of data
+.TP
 \fB\-\-force\-remove\-dirty\fR
 force remove projects with uncommitted modifications
 if projects no longer exist in the manifest. WARNING:
 this may cause loss of data
 .TP
+\fB\-\-rebase\fR
+rebase local commits regardless of whether they are
+published
+.TP
 \fB\-l\fR, \fB\-\-local\-only\fR
 only update working tree, don't fetch
 .TP
@@ -185,6 +194,11 @@ The \fB\-\-force\-sync\fR option can be used to overwrite existing git directori
 they have previously been linked to a different object directory. WARNING: This
 may cause data to be lost since refs may be removed when overwriting.
 .PP
+The \fB\-\-force\-checkout\fR option can be used to force git to switch revs even if the
+index or the working tree differs from HEAD, and if there are untracked files.
+WARNING: This may cause data to be lost since uncommitted changes may be
+removed.
+.PP
 The \fB\-\-force\-remove\-dirty\fR option can be used to remove previously used projects
 with uncommitted changes. WARNING: This may cause data to be lost since
 uncommitted changes may be removed with projects that no longer exist in the

man/repo-upload.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "August 2022" "repo upload" "Repo Manual"
+.TH REPO "1" "June 2024" "repo upload" "Repo Manual"
 .SH NAME
 repo \- repo upload - manual page for repo upload
 .SH SYNOPSIS
@@ -18,8 +18,11 @@ show this help message and exit
 number of jobs to run in parallel (default: based on
 number of CPU cores)
 .TP
-\fB\-t\fR
-send local branch name to Gerrit Code Review
+\fB\-t\fR, \fB\-\-topic\-branch\fR
+set the topic to the local branch name
+.TP
+\fB\-\-topic\fR=\fI\,TOPIC\/\fR
+set topic for the change
 .TP
 \fB\-\-hashtag\fR=\fI\,HASHTAGS\/\fR, \fB\-\-ht\fR=\fI\,HASHTAGS\/\fR
 add hashtags (comma delimited) to the review
@@ -30,6 +33,9 @@ add local branch name as a hashtag
 \fB\-l\fR LABELS, \fB\-\-label\fR=\fI\,LABELS\/\fR
 add a label when uploading
 .TP
+\fB\-\-pd\fR=\fI\,PATCHSET_DESCRIPTION\/\fR, \fB\-\-patchset\-description\fR=\fI\,PATCHSET_DESCRIPTION\/\fR
+description for patchset
+.TP
 \fB\-\-re\fR=\fI\,REVIEWERS\/\fR, \fB\-\-reviewers\fR=\fI\,REVIEWERS\/\fR
 request reviews from these people
 .TP
@@ -198,6 +204,12 @@ review.URL.uploadnotify:
 Control e\-mail notifications when uploading.
 https://gerrit\-review.googlesource.com/Documentation/user\-upload.html#notify
 .PP
+review.URL.uploadwarningthreshold:
+.PP
+Repo will warn you if you are attempting to upload a large number of commits in
+one or more branches. By default, the threshold is five commits. This option
+allows you to override the warning threshold to a different value.
+.PP
 References
 .PP
 Gerrit Code Review: https://www.gerritcodereview.com/

man/repo.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "June 2023" "repo" "Repo Manual"
+.TH REPO "1" "December 2024" "repo" "Repo Manual"
 .SH NAME
 repo \- repository management tool built on top of git
 .SH SYNOPSIS
@@ -79,11 +79,8 @@ Download and checkout a change
 forall
 Run a shell command in each project
 .TP
-gitc\-delete
-Delete a GITC Client.
-.TP
-gitc\-init
-Initialize a GITC Client.
+gc
+Cleaning up internal repo state.
 .TP
 grep
 Print lines matching a pattern

manifest_xml.py

@@ -114,9 +114,37 @@ def XmlInt(node, attr, default=None):
     try:
         return int(value)
     except ValueError:
-        raise ManifestParseError(
-            'manifest: invalid %s="%s" integer' % (attr, value)
-        )
+        raise ManifestParseError(f'manifest: invalid {attr}="{value}" integer')
+
+
+def normalize_url(url: str) -> str:
+    """Mutate input 'url' into normalized form:
+
+    * remove trailing slashes
+    * convert SCP-like syntax to SSH URL
+
+    Args:
+        url: URL to modify
+
+    Returns:
+        The normalized URL.
+    """
+
+    url = url.rstrip("/")
+    parsed_url = urllib.parse.urlparse(url)
+
+    # This matches patterns like "git@github.com:foo".
+    scp_like_url_re = r"^[^/:]+@[^/:]+:[^/]+"
+
+    # If our URL is missing a schema and matches git's
+    # SCP-like syntax we should convert it to a proper
+    # SSH URL instead to make urljoin() happier.
+    #
+    # See: https://git-scm.com/docs/git-clone#URLS
+    if not parsed_url.scheme and re.match(scp_like_url_re, url):
+        return "ssh://" + url.replace(":", "/", 1)
+
+    return url
 
 
 class _Default:
@@ -182,20 +210,22 @@ class _XmlRemote:
     def _resolveFetchUrl(self):
         if self.fetchUrl is None:
             return ""
-        url = self.fetchUrl.rstrip("/")
-        manifestUrl = self.manifestUrl.rstrip("/")
-        # urljoin will gets confused over quite a few things.  The ones we care
-        # about here are:
-        # * no scheme in the base url, like <hostname:port>
-        # We handle no scheme by replacing it with an obscure protocol, gopher
-        # and then replacing it with the original when we are done.
 
-        if manifestUrl.find(":") != manifestUrl.find("/") - 1:
-            url = urllib.parse.urljoin("gopher://" + manifestUrl, url)
-            url = re.sub(r"^gopher://", "", url)
+        fetch_url = normalize_url(self.fetchUrl)
+        manifest_url = normalize_url(self.manifestUrl)
+
+        # urljoin doesn't like URLs with no scheme in the base URL
+        # such as file paths.  We handle this by prefixing it with
+        # an obscure protocol, gopher, and replacing it with the
+        # original after urljoin
+        if manifest_url.find(":") != manifest_url.find("/") - 1:
+            fetch_url = urllib.parse.urljoin(
+                "gopher://" + manifest_url, fetch_url
+            )
+            fetch_url = re.sub(r"^gopher://", "", fetch_url)
         else:
-            url = urllib.parse.urljoin(manifestUrl, url)
-        return url
+            fetch_url = urllib.parse.urljoin(manifest_url, fetch_url)
+        return fetch_url
 
     def ToRemoteSpec(self, projectName):
         fetchUrl = self.resolvedFetchUrl.rstrip("/")
@@ -275,7 +305,7 @@ class _XmlSubmanifest:
             parent.repodir,
             linkFile,
             parent_groups=",".join(groups) or "",
-            submanifest_path=self.relpath,
+            submanifest_path=os.path.join(parent.path_prefix, self.relpath),
             outer_client=outer_client,
             default_groups=default_groups,
         )
@@ -405,11 +435,6 @@ class XmlManifest:
         self.parent_groups = parent_groups
         self.default_groups = default_groups
 
-        if outer_client and self.isGitcClient:
-            raise ManifestParseError(
-                "Multi-manifest is incompatible with `gitc-init`"
-            )
-
         if submanifest_path and not outer_client:
             # If passing a submanifest_path, there must be an outer_client.
             raise ManifestParseError(f"Bad call to {self.__class__.__name__}")
@@ -810,7 +835,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                         ret.setdefault(child.nodeName, []).append(element)
                     else:
                         raise ManifestParseError(
-                            'Unhandled element "%s"' % (child.nodeName,)
+                            f'Unhandled element "{child.nodeName}"'
                         )
 
                     append_children(element, child)
@@ -1258,12 +1283,10 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         try:
             root = xml.dom.minidom.parse(path)
         except (OSError, xml.parsers.expat.ExpatError) as e:
-            raise ManifestParseError(
-                "error parsing manifest %s: %s" % (path, e)
-            )
+            raise ManifestParseError(f"error parsing manifest {path}: {e}")
 
         if not root or not root.childNodes:
-            raise ManifestParseError("no root node in %s" % (path,))
+            raise ManifestParseError(f"no root node in {path}")
 
         for manifest in root.childNodes:
             if (
@@ -1272,7 +1295,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
             ):
                 break
         else:
-            raise ManifestParseError("no <manifest> in %s" % (path,))
+            raise ManifestParseError(f"no <manifest> in {path}")
 
         nodes = []
         for node in manifest.childNodes:
@@ -1282,7 +1305,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                     msg = self._CheckLocalPath(name)
                     if msg:
                         raise ManifestInvalidPathError(
-                            '<include> invalid "name": %s: %s' % (name, msg)
+                            f'<include> invalid "name": {name}: {msg}'
                         )
                 include_groups = ""
                 if parent_groups:
@@ -1314,7 +1337,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                     raise
                 except Exception as e:
                     raise ManifestParseError(
-                        "failed parsing included manifest %s: %s" % (name, e)
+                        f"failed parsing included manifest {name}: {e}"
                     )
             else:
                 if parent_groups and node.nodeName == "project":
@@ -1422,6 +1445,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
         repo_hooks_project = None
         enabled_repo_hooks = None
+        failed_revision_changes = []
         for node in itertools.chain(*node_list):
             if node.nodeName == "project":
                 project = self._ParseProject(node)
@@ -1448,6 +1472,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                     remote = self._get_remote(node)
                 dest_branch = node.getAttribute("dest-branch")
                 upstream = node.getAttribute("upstream")
+                base_revision = node.getAttribute("base-rev")
 
                 named_projects = self._projects[name]
                 if dest_path and not path and len(named_projects) > 1:
@@ -1461,6 +1486,13 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                     if groups:
                         p.groups.extend(groups)
                     if revision:
+                        if base_revision:
+                            if p.revisionExpr != base_revision:
+                                failed_revision_changes.append(
+                                    "extend-project name %s mismatch base "
+                                    "%s vs revision %s"
+                                    % (name, base_revision, p.revisionExpr)
+                                )
                         p.SetRevision(revision)
 
                     if remote_name:
@@ -1535,6 +1567,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
             if node.nodeName == "remove-project":
                 name = node.getAttribute("name")
                 path = node.getAttribute("path")
+                base_revision = node.getAttribute("base-rev")
 
                 # Name or path needed.
                 if not name and not path:
@@ -1548,6 +1581,13 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                 for projname, projects in list(self._projects.items()):
                     for p in projects:
                         if name == projname and not path:
+                            if base_revision:
+                                if p.revisionExpr != base_revision:
+                                    failed_revision_changes.append(
+                                        "remove-project name %s mismatch base "
+                                        "%s vs revision %s"
+                                        % (name, base_revision, p.revisionExpr)
+                                    )
                             del self._paths[p.relpath]
                             if not removed_project:
                                 del self._projects[name]
@@ -1555,6 +1595,17 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                         elif path == p.relpath and (
                             name == projname or not name
                         ):
+                            if base_revision:
+                                if p.revisionExpr != base_revision:
+                                    failed_revision_changes.append(
+                                        "remove-project path %s mismatch base "
+                                        "%s vs revision %s"
+                                        % (
+                                            p.relpath,
+                                            base_revision,
+                                            p.revisionExpr,
+                                        )
+                                    )
                             self._projects[projname].remove(p)
                             del self._paths[p.relpath]
                             removed_project = p.name
@@ -1574,6 +1625,13 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                         "project: %s" % node.toxml()
                     )
 
+        if failed_revision_changes:
+            raise ManifestParseError(
+                "revision base check failed, rebase patches and update "
+                "base revs for: ",
+                failed_revision_changes,
+            )
+
         # Store repo hooks project information.
         if repo_hooks_project:
             # Store a reference to the Project.
@@ -1765,13 +1823,13 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                 msg = self._CheckLocalPath(name)
                 if msg:
                     raise ManifestInvalidPathError(
-                        '<submanifest> invalid "name": %s: %s' % (name, msg)
+                        f'<submanifest> invalid "name": {name}: {msg}'
                     )
         else:
             msg = self._CheckLocalPath(path)
             if msg:
                 raise ManifestInvalidPathError(
-                    '<submanifest> invalid "path": %s: %s' % (path, msg)
+                    f'<submanifest> invalid "path": {path}: {msg}'
                 )
 
         submanifest = _XmlSubmanifest(
@@ -1806,7 +1864,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         msg = self._CheckLocalPath(name, dir_ok=True)
         if msg:
             raise ManifestInvalidPathError(
-                '<project> invalid "name": %s: %s' % (name, msg)
+                f'<project> invalid "name": {name}: {msg}'
             )
         if parent:
             name = self._JoinName(parent.name, name)
@@ -1816,7 +1874,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
             remote = self._default.remote
         if remote is None:
             raise ManifestParseError(
-                "no remote for project %s within %s" % (name, self.manifestFile)
+                f"no remote for project {name} within {self.manifestFile}"
             )
 
         revisionExpr = node.getAttribute("revision") or remote.revision
@@ -1837,7 +1895,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
             msg = self._CheckLocalPath(path, dir_ok=True, cwd_dot_ok=True)
             if msg:
                 raise ManifestInvalidPathError(
-                    '<project> invalid "path": %s: %s' % (path, msg)
+                    f'<project> invalid "path": {path}: {msg}'
                 )
 
         rebase = XmlBool(node, "rebase", True)
@@ -2094,7 +2152,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         if not cwd_dot_ok or parts != ["."]:
             for part in set(parts):
                 if part in {".", "..", ".git"} or part.startswith(".repo"):
-                    return "bad component: %s" % (part,)
+                    return f"bad component: {part}"
 
         if not dir_ok and resep.match(path[-1]):
             return "dirs not allowed"
@@ -2130,7 +2188,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         msg = cls._CheckLocalPath(dest)
         if msg:
             raise ManifestInvalidPathError(
-                '<%s> invalid "dest": %s: %s' % (element, dest, msg)
+                f'<{element}> invalid "dest": {dest}: {msg}'
             )
 
         # |src| is the file we read from or path we point to for symlinks.
@@ -2141,7 +2199,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         )
         if msg:
             raise ManifestInvalidPathError(
-                '<%s> invalid "src": %s: %s' % (element, src, msg)
+                f'<{element}> invalid "src": {src}: {msg}'
             )
 
     def _ParseCopyFile(self, project, node):
@@ -2185,7 +2243,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         v = self._remotes.get(name)
         if not v:
             raise ManifestParseError(
-                "remote %s not defined in %s" % (name, self.manifestFile)
+                f"remote {name} not defined in {self.manifestFile}"
             )
         return v
 
@@ -2262,7 +2320,6 @@ class RepoClient(XmlManifest):
             submanifest_path: The submanifest root relative to the repo root.
             **kwargs: Additional keyword arguments, passed to XmlManifest.
         """
-        self.isGitcClient = False
         submanifest_path = submanifest_path or ""
         if submanifest_path:
             self._CheckLocalPath(submanifest_path)

platform_utils.py

@@ -57,8 +57,8 @@ def _validate_winpath(path):
     if _winpath_is_valid(path):
         return path
     raise ValueError(
-        'Path "{}" must be a relative path or an absolute '
-        "path starting with a drive letter".format(path)
+        f'Path "{path}" must be a relative path or an absolute '
+        "path starting with a drive letter"
     )
 
 
@@ -251,32 +251,3 @@ def readlink(path):
         return platform_utils_win32.readlink(_makelongpath(path))
     else:
         return os.readlink(path)
-
-
-def realpath(path):
-    """Return the canonical path of the specified filename, eliminating
-    any symbolic links encountered in the path.
-
-    Availability: Windows, Unix.
-    """
-    if isWindows():
-        current_path = os.path.abspath(path)
-        path_tail = []
-        for c in range(0, 100):  # Avoid cycles
-            if islink(current_path):
-                target = readlink(current_path)
-                current_path = os.path.join(
-                    os.path.dirname(current_path), target
-                )
-            else:
-                basename = os.path.basename(current_path)
-                if basename == "":
-                    path_tail.append(current_path)
-                    break
-                path_tail.append(basename)
-                current_path = os.path.dirname(current_path)
-        path_tail.reverse()
-        result = os.path.normpath(os.path.join(*path_tail))
-        return result
-    else:
-        return os.path.realpath(path)

platform_utils_win32.py

@@ -186,9 +186,7 @@ def _create_symlink(source, link_name, dwFlags):
             error_desc = FormatError(code).strip()
             if code == ERROR_PRIVILEGE_NOT_HELD:
                 raise OSError(errno.EPERM, error_desc, link_name)
-            _raise_winerror(
-                code, 'Error creating symbolic link "{}"'.format(link_name)
-            )
+            _raise_winerror(code, f'Error creating symbolic link "{link_name}"')
 
 
 def islink(path):
@@ -210,7 +208,7 @@ def readlink(path):
     )
     if reparse_point_handle == INVALID_HANDLE_VALUE:
         _raise_winerror(
-            get_last_error(), 'Error opening symbolic link "{}"'.format(path)
+            get_last_error(), f'Error opening symbolic link "{path}"'
         )
     target_buffer = c_buffer(MAXIMUM_REPARSE_DATA_BUFFER_SIZE)
     n_bytes_returned = DWORD()
@@ -227,7 +225,7 @@ def readlink(path):
     CloseHandle(reparse_point_handle)
     if not io_result:
         _raise_winerror(
-            get_last_error(), 'Error reading symbolic link "{}"'.format(path)
+            get_last_error(), f'Error reading symbolic link "{path}"'
         )
     rdb = REPARSE_DATA_BUFFER.from_buffer(target_buffer)
     if rdb.ReparseTag == IO_REPARSE_TAG_SYMLINK:
@@ -236,11 +234,11 @@ def readlink(path):
         return rdb.MountPointReparseBuffer.PrintName
     # Unsupported reparse point type.
     _raise_winerror(
-        ERROR_NOT_SUPPORTED, 'Error reading symbolic link "{}"'.format(path)
+        ERROR_NOT_SUPPORTED, f'Error reading symbolic link "{path}"'
     )
 
 
 def _raise_winerror(code, error_desc):
     win_error_desc = FormatError(code).strip()
-    error_desc = "{0}: {1}".format(error_desc, win_error_desc)
+    error_desc = f"{error_desc}: {win_error_desc}"
     raise WinError(code, error_desc)

progress.py

@@ -52,11 +52,11 @@ def duration_str(total):
     uses microsecond resolution.  This makes for noisy output.
     """
     hours, mins, secs = convert_to_hms(total)
-    ret = "%.3fs" % (secs,)
+    ret = f"{secs:.3f}s"
     if mins:
-        ret = "%im%s" % (mins, ret)
+        ret = f"{mins}m{ret}"
     if hours:
-        ret = "%ih%s" % (hours, ret)
+        ret = f"{hours}h{ret}"
     return ret
 
 
@@ -100,6 +100,7 @@ class Progress:
         self._show = not delay
         self._units = units
         self._elide = elide and _TTY
+        self._quiet = quiet
 
         # Only show the active jobs section if we run more than one in parallel.
         self._show_jobs = False
@@ -114,13 +115,7 @@ class Progress:
         )
         self._update_thread.daemon = True
 
-        # When quiet, never show any output.  It's a bit hacky, but reusing the
-        # existing logic that delays initial output keeps the rest of the class
-        # clean.  Basically we set the start time to years in the future.
-        if quiet:
-            self._show = False
-            self._start += 2**32
-        elif show_elapsed:
+        if not quiet and show_elapsed:
             self._update_thread.start()
 
     def _update_loop(self):
@@ -160,7 +155,7 @@ class Progress:
             msg = self._last_msg
         self._last_msg = msg
 
-        if not _TTY or IsTraceToStderr():
+        if not _TTY or IsTraceToStderr() or self._quiet:
             return
 
         elapsed_sec = time.time() - self._start
@@ -202,7 +197,7 @@ class Progress:
 
     def end(self):
         self._update_event.set()
-        if not _TTY or IsTraceToStderr() or not self._show:
+        if not _TTY or IsTraceToStderr() or self._quiet:
             return
 
         duration = duration_str(time.time() - self._start)

project.py

@@ -21,6 +21,7 @@ import random
 import re
 import shutil
 import stat
+import string
 import subprocess
 import sys
 import tarfile
@@ -31,6 +32,7 @@ import urllib.parse
 
 from color import Coloring
 from error import DownloadError
+from error import GitAuthError
 from error import GitError
 from error import ManifestInvalidPathError
 from error import ManifestInvalidRevisionError
@@ -144,7 +146,7 @@ def _ProjectHooks():
     """
     global _project_hook_list
     if _project_hook_list is None:
-        d = platform_utils.realpath(os.path.abspath(os.path.dirname(__file__)))
+        d = os.path.realpath(os.path.abspath(os.path.dirname(__file__)))
         d = os.path.join(d, "hooks")
         _project_hook_list = [
             os.path.join(d, x) for x in platform_utils.listdir(d)
@@ -256,7 +258,7 @@ class ReviewableBranch:
         self,
         people,
         dryrun=False,
-        auto_topic=False,
+        topic=None,
         hashtags=(),
         labels=(),
         private=False,
@@ -266,12 +268,13 @@ class ReviewableBranch:
         dest_branch=None,
         validate_certs=True,
         push_options=None,
+        patchset_description=None,
     ):
         self.project.UploadForReview(
             branch=self.name,
             people=people,
             dryrun=dryrun,
-            auto_topic=auto_topic,
+            topic=topic,
             hashtags=hashtags,
             labels=labels,
             private=private,
@@ -281,6 +284,7 @@ class ReviewableBranch:
             dest_branch=dest_branch,
             validate_certs=validate_certs,
             push_options=push_options,
+            patchset_description=patchset_description,
         )
 
     def GetPublishedRefs(self):
@@ -365,19 +369,19 @@ def _SafeExpandPath(base, subpath, skipfinal=False):
     for part in components:
         if part in {".", ".."}:
             raise ManifestInvalidPathError(
-                '%s: "%s" not allowed in paths' % (subpath, part)
+                f'{subpath}: "{part}" not allowed in paths'
             )
 
         path = os.path.join(path, part)
         if platform_utils.islink(path):
             raise ManifestInvalidPathError(
-                "%s: traversing symlinks not allow" % (path,)
+                f"{path}: traversing symlinks not allow"
             )
 
         if os.path.exists(path):
             if not os.path.isfile(path) and not platform_utils.isdir(path):
                 raise ManifestInvalidPathError(
-                    "%s: only regular files & directories allowed" % (path,)
+                    f"{path}: only regular files & directories allowed"
                 )
 
     if skipfinal:
@@ -409,11 +413,11 @@ class _CopyFile:
 
         if platform_utils.isdir(src):
             raise ManifestInvalidPathError(
-                "%s: copying from directory not supported" % (self.src,)
+                f"{self.src}: copying from directory not supported"
             )
         if platform_utils.isdir(dest):
             raise ManifestInvalidPathError(
-                "%s: copying to directory not allowed" % (self.dest,)
+                f"{self.dest}: copying to directory not allowed"
             )
 
         # Copy file if it does not exist or is out of date.
@@ -572,7 +576,6 @@ class Project:
         dest_branch=None,
         optimized_fetch=False,
         retry_fetches=0,
-        old_revision=None,
     ):
         """Init a Project object.
 
@@ -605,7 +608,6 @@ class Project:
                 only fetch from the remote if the sha1 is not present locally.
             retry_fetches: Retry remote fetches n times upon receiving transient
                 error with exponential backoff and jitter.
-            old_revision: saved git commit id for open GITC projects.
         """
         self.client = self.manifest = manifest
         self.name = name
@@ -635,7 +637,6 @@ class Project:
         self.linkfiles = []
         self.annotations = []
         self.dest_branch = dest_branch
-        self.old_revision = old_revision
 
         # This will be filled in if a project is later identified to be the
         # project containing repo hooks.
@@ -724,12 +725,34 @@ class Project:
         return None
 
     def IsRebaseInProgress(self):
+        """Returns true if a rebase or "am" is in progress"""
+        # "rebase-apply" is used for "git rebase".
+        # "rebase-merge" is used for "git am".
         return (
             os.path.exists(self.work_git.GetDotgitPath("rebase-apply"))
             or os.path.exists(self.work_git.GetDotgitPath("rebase-merge"))
             or os.path.exists(os.path.join(self.worktree, ".dotest"))
         )
 
+    def IsCherryPickInProgress(self):
+        """Returns True if a cherry-pick is in progress."""
+        return os.path.exists(self.work_git.GetDotgitPath("CHERRY_PICK_HEAD"))
+
+    def _AbortRebase(self):
+        """Abort ongoing rebase, cherry-pick or patch apply (am).
+
+        If no rebase, cherry-pick or patch apply was in progress, this method
+        ignores the status and continues.
+        """
+
+        def _git(*args):
+            # Ignore return code, in case there was no rebase in progress.
+            GitCommand(self, args, log_as_error=False).Wait()
+
+        _git("cherry-pick", "--abort")
+        _git("rebase", "--abort")
+        _git("am", "--abort")
+
     def IsDirty(self, consider_untracked=True):
         """Is the working directory modified in some way?"""
         self.work_git.update_index(
@@ -957,15 +980,11 @@ class Project:
                 f_status = "-"
 
             if i and i.src_path:
-                line = " %s%s\t%s => %s (%s%%)" % (
-                    i_status,
-                    f_status,
-                    i.src_path,
-                    p,
-                    i.level,
+                line = (
+                    f" {i_status}{f_status}\t{i.src_path} => {p} ({i.level}%)"
                 )
             else:
-                line = " %s%s\t%s" % (i_status, f_status, p)
+                line = f" {i_status}{f_status}\t{p}"
 
             if i and not f:
                 out.added("%s", line)
@@ -1083,7 +1102,7 @@ class Project:
         branch=None,
         people=([], []),
         dryrun=False,
-        auto_topic=False,
+        topic=None,
         hashtags=(),
         labels=(),
         private=False,
@@ -1093,6 +1112,7 @@ class Project:
         dest_branch=None,
         validate_certs=True,
         push_options=None,
+        patchset_description=None,
     ):
         """Uploads the named branch for code review."""
         if branch is None:
@@ -1145,8 +1165,7 @@ class Project:
         # This stops git from pushing all reachable annotated tags when
         # push.followTags is configured. Gerrit does not accept any tags
         # pushed to a CL.
-        if git_require((1, 8, 3)):
-            cmd.append("--no-follow-tags")
+        cmd.append("--no-follow-tags")
 
         for push_option in push_options or []:
             cmd.append("-o")
@@ -1157,10 +1176,10 @@ class Project:
         if dest_branch.startswith(R_HEADS):
             dest_branch = dest_branch[len(R_HEADS) :]
 
-        ref_spec = "%s:refs/for/%s" % (R_HEADS + branch.name, dest_branch)
+        ref_spec = f"{R_HEADS + branch.name}:refs/for/{dest_branch}"
         opts = []
-        if auto_topic:
-            opts += ["topic=" + branch.name]
+        if topic is not None:
+            opts += [f"topic={topic}"]
         opts += ["t=%s" % p for p in hashtags]
         # NB: No need to encode labels as they've been validated above.
         opts += ["l=%s" % p for p in labels]
@@ -1175,6 +1194,10 @@ class Project:
             opts += ["wip"]
         if ready:
             opts += ["ready"]
+        if patchset_description:
+            opts += [
+                f"m={self._encode_patchset_description(patchset_description)}"
+            ]
         if opts:
             ref_spec = ref_spec + "%" + ",".join(opts)
         cmd.append(ref_spec)
@@ -1182,11 +1205,35 @@ class Project:
         GitCommand(self, cmd, bare=True, verify_command=True).Wait()
 
         if not dryrun:
-            msg = "posted to %s for %s" % (branch.remote.review, dest_branch)
+            msg = f"posted to {branch.remote.review} for {dest_branch}"
             self.bare_git.UpdateRef(
                 R_PUB + branch.name, R_HEADS + branch.name, message=msg
             )
 
+    @staticmethod
+    def _encode_patchset_description(original):
+        """Applies percent-encoding for strings sent as patchset description.
+
+        The encoding used is based on but stricter than URL encoding (Section
+        2.1 of RFC 3986). The only non-escaped characters are alphanumerics, and
+        'SPACE' (U+0020) can be represented as 'LOW LINE' (U+005F) or
+        'PLUS SIGN' (U+002B).
+
+        For more information, see the Gerrit docs here:
+        https://gerrit-review.googlesource.com/Documentation/user-upload.html#patch_set_description
+        """
+        SAFE = {ord(x) for x in string.ascii_letters + string.digits}
+
+        def _enc(b):
+            if b in SAFE:
+                return chr(b)
+            elif b == ord(" "):
+                return "_"
+            else:
+                return f"%{b:02x}"
+
+        return "".join(_enc(x) for x in original.encode("utf-8"))
+
     def _ExtractArchive(self, tarpath, path=None):
         """Extract the given tar on its current location
 
@@ -1281,7 +1328,20 @@ class Project:
         if is_new:
             self._InitGitDir(force_sync=force_sync, quiet=quiet)
         else:
-            self._UpdateHooks(quiet=quiet)
+            try:
+                # At this point, it's possible that gitdir points to an old
+                # objdir (e.g. name changed, but objdir exists). Check
+                # references to ensure that's not the case. See
+                # https://issues.gerritcodereview.com/40013418 for more
+                # details.
+                self._CheckDirReference(self.objdir, self.gitdir)
+
+                self._UpdateHooks(quiet=quiet)
+            except GitError as e:
+                if not force_sync:
+                    raise e
+                # Let _InitGitDir fix the issue, force_sync is always True here.
+                self._InitGitDir(force_sync=True, quiet=quiet)
         self._InitRemote()
 
         if self.UseAlternates:
@@ -1421,8 +1481,6 @@ class Project:
         self._InitHooks()
 
     def _CopyAndLinkFiles(self):
-        if self.client.isGitcClient:
-            return
         for copyfile in self.copyfiles:
             copyfile._Copy()
         for linkfile in self.linkfiles:
@@ -1444,7 +1502,7 @@ class Project:
             return self.bare_git.rev_list(self.revisionExpr, "-1")[0]
         except GitError:
             raise ManifestInvalidRevisionError(
-                "revision %s in %s not found" % (self.revisionExpr, self.name)
+                f"revision {self.revisionExpr} in {self.name} not found"
             )
 
     def GetRevisionId(self, all_refs=None):
@@ -1461,7 +1519,7 @@ class Project:
             return self.bare_git.rev_parse("--verify", "%s^0" % rev)
         except GitError:
             raise ManifestInvalidRevisionError(
-                "revision %s in %s not found" % (self.revisionExpr, self.name)
+                f"revision {self.revisionExpr} in {self.name} not found"
             )
 
     def SetRevisionId(self, revisionId):
@@ -1471,7 +1529,14 @@ class Project:
         self.revisionId = revisionId
 
     def Sync_LocalHalf(
-        self, syncbuf, force_sync=False, submodules=False, errors=None
+        self,
+        syncbuf,
+        force_sync=False,
+        force_checkout=False,
+        force_rebase=False,
+        submodules=False,
+        errors=None,
+        verbose=False,
     ):
         """Perform only the local IO portion of the sync process.
 
@@ -1539,7 +1604,15 @@ class Project:
         if branch is None or syncbuf.detach_head:
             # Currently on a detached HEAD.  The user is assumed to
             # not have any local modifications worth worrying about.
-            if self.IsRebaseInProgress():
+            rebase_in_progress = (
+                self.IsRebaseInProgress() or self.IsCherryPickInProgress()
+            )
+            if rebase_in_progress and force_checkout:
+                self._AbortRebase()
+                rebase_in_progress = (
+                    self.IsRebaseInProgress() or self.IsCherryPickInProgress()
+                )
+            if rebase_in_progress:
                 fail(_PriorSyncFailedError(project=self.name))
                 return
 
@@ -1552,11 +1625,11 @@ class Project:
                     return
             else:
                 lost = self._revlist(not_rev(revid), HEAD)
-                if lost:
+                if lost and verbose:
                     syncbuf.info(self, "discarding %d commits", len(lost))
 
             try:
-                self._Checkout(revid, quiet=True)
+                self._Checkout(revid, force_checkout=force_checkout, quiet=True)
                 if submodules:
                     self._SyncSubmodules(quiet=True)
             except GitError as e:
@@ -1606,25 +1679,28 @@ class Project:
         if pub:
             not_merged = self._revlist(not_rev(revid), pub)
             if not_merged:
-                if upstream_gain:
+                if upstream_gain and not force_rebase:
                     # The user has published this branch and some of those
                     # commits are not yet merged upstream.  We do not want
                     # to rewrite the published commits so we punt.
                     fail(
                         LocalSyncFail(
                             "branch %s is published (but not merged) and is "
-                            "now %d commits behind"
+                            "now %d commits behind. Fix this manually or rerun "
+                            "with the --rebase option to force a rebase."
                             % (branch.name, len(upstream_gain)),
                             project=self.name,
                         )
                     )
+                    return
+                syncbuf.later1(self, _doff, not verbose)
                 return
             elif pub == head:
                 # All published commits are merged, and thus we are a
                 # strict subset.  We can fast-forward safely.
-                syncbuf.later1(self, _doff)
+                syncbuf.later1(self, _doff, not verbose)
                 if submodules:
-                    syncbuf.later1(self, _dosubmodules)
+                    syncbuf.later1(self, _dosubmodules, not verbose)
                 return
 
         # Examine the local commits not in the remote.  Find the
@@ -1683,10 +1759,10 @@ class Project:
             def _dorebase():
                 self._Rebase(upstream="%s^1" % last_mine, onto=revid)
 
-            syncbuf.later2(self, _dorebase)
+            syncbuf.later2(self, _dorebase, not verbose)
             if submodules:
-                syncbuf.later2(self, _dosubmodules)
-            syncbuf.later2(self, _docopyandlink)
+                syncbuf.later2(self, _dosubmodules, not verbose)
+            syncbuf.later2(self, _docopyandlink, not verbose)
         elif local_changes:
             try:
                 self._ResetHard(revid)
@@ -1697,9 +1773,9 @@ class Project:
                 fail(e)
                 return
         else:
-            syncbuf.later1(self, _doff)
+            syncbuf.later1(self, _doff, not verbose)
             if submodules:
-                syncbuf.later1(self, _dosubmodules)
+                syncbuf.later1(self, _dosubmodules, not verbose)
 
     def AddCopyFile(self, src, dest, topdir):
         """Mark |src| for copying to |dest| (relative to |topdir|).
@@ -1742,7 +1818,7 @@ class Project:
             self.bare_git.rev_parse("FETCH_HEAD"),
         )
 
-    def DeleteWorktree(self, quiet=False, force=False):
+    def DeleteWorktree(self, verbose=False, force=False):
         """Delete the source checkout and any other housekeeping tasks.
 
         This currently leaves behind the internal .repo/ cache state.  This
@@ -1751,7 +1827,7 @@ class Project:
         at some point.
 
         Args:
-            quiet: Whether to hide normal messages.
+            verbose: Whether to show verbose messages.
             force: Always delete tree even if dirty.
 
         Returns:
@@ -1766,22 +1842,20 @@ class Project:
                 )
             else:
                 msg = (
-                    "error: %s: Cannot remove project: uncommitted"
+                    "error: %s: Cannot remove project: uncommitted "
                     "changes are present.\n" % self.RelPath(local=False)
                 )
                 logger.error(msg)
-                raise DeleteDirtyWorktreeError(msg, project=self)
+                raise DeleteDirtyWorktreeError(msg, project=self.name)
 
-        if not quiet:
-            print(
-                "%s: Deleting obsolete checkout." % (self.RelPath(local=False),)
-            )
+        if verbose:
+            print(f"{self.RelPath(local=False)}: Deleting obsolete checkout.")
 
         # Unlock and delink from the main worktree.  We don't use git's worktree
         # remove because it will recursively delete projects -- we handle that
         # ourselves below.  https://crbug.com/git/48
         if self.use_git_worktrees:
-            needle = platform_utils.realpath(self.gitdir)
+            needle = os.path.realpath(self.gitdir)
             # Find the git worktree commondir under .repo/worktrees/.
             output = self.bare_git.worktree("list", "--porcelain").splitlines()[
                 0
@@ -1795,7 +1869,7 @@ class Project:
                 with open(gitdir) as fp:
                     relpath = fp.read().strip()
                 # Resolve the checkout path and see if it matches this project.
-                fullpath = platform_utils.realpath(
+                fullpath = os.path.realpath(
                     os.path.join(configs, name, relpath)
                 )
                 if fullpath == needle:
@@ -1836,7 +1910,7 @@ class Project:
                     platform_utils.remove(path)
                 except OSError as e:
                     if e.errno != errno.ENOENT:
-                        logger.error("error: %s: Failed to remove: %s", path, e)
+                        logger.warning("%s: Failed to remove: %s", path, e)
                         failed = True
                         errors.append(e)
             dirs[:] = [
@@ -1855,7 +1929,7 @@ class Project:
                     platform_utils.remove(d)
                 except OSError as e:
                     if e.errno != errno.ENOENT:
-                        logger.error("error: %s: Failed to remove: %s", d, e)
+                        logger.warning("%s: Failed to remove: %s", d, e)
                         failed = True
                         errors.append(e)
             elif not platform_utils.listdir(d):
@@ -1863,18 +1937,30 @@ class Project:
                     platform_utils.rmdir(d)
                 except OSError as e:
                     if e.errno != errno.ENOENT:
-                        logger.error("error: %s: Failed to remove: %s", d, e)
+                        logger.warning("%s: Failed to remove: %s", d, e)
                         failed = True
                         errors.append(e)
         if failed:
-            logger.error(
-                "error: %s: Failed to delete obsolete checkout.",
-                self.RelPath(local=False),
+            rename_path = (
+                f"{self.worktree}_repo_to_be_deleted_{int(time.time())}"
             )
-            logger.error(
-                "       Remove manually, then run `repo sync -l`.",
-            )
-            raise DeleteWorktreeError(aggregate_errors=errors)
+            try:
+                platform_utils.rename(self.worktree, rename_path)
+                logger.warning(
+                    "warning: renamed %s to %s. You can delete it, but you "
+                    "might need elevated permissions (e.g. root)",
+                    self.worktree,
+                    rename_path,
+                )
+                # Rename successful! Clear the errors.
+                errors = []
+            except OSError:
+                logger.error(
+                    "%s: Failed to delete obsolete checkout.\n",
+                    "       Remove manually, then run `repo sync -l`.",
+                    self.RelPath(local=False),
+                )
+                raise DeleteWorktreeError(aggregate_errors=errors)
 
         # Try deleting parent dirs if they are empty.
         path = self.worktree
@@ -1968,7 +2054,7 @@ class Project:
             # target branch, but otherwise take no other action.
             _lwrite(
                 self.work_git.GetDotgitPath(subpath=HEAD),
-                "ref: %s%s\n" % (R_HEADS, name),
+                f"ref: {R_HEADS}{name}\n",
             )
             return True
 
@@ -2207,7 +2293,9 @@ class Project:
 
         try:
             rev = self.GetRevisionId()
-        except GitError:
+        except (GitError, ManifestInvalidRevisionError):
+            # The git repo may be outdated (i.e. not fetched yet) and querying
+            # its submodules using the revision may not work; so return here.
             return []
         return get_submodules(self.gitdir, rev)
 
@@ -2277,7 +2365,7 @@ class Project:
             self.config.SetString("core.repositoryFormatVersion", str(version))
 
         # Enable the extension!
-        self.config.SetString("extensions.%s" % (key,), value)
+        self.config.SetString(f"extensions.{key}", value)
 
     def ResolveRemoteHead(self, name=None):
         """Find out what the default branch (HEAD) points to.
@@ -2307,26 +2395,25 @@ class Project:
         try:
             # if revision (sha or tag) is not present then following function
             # throws an error.
+            revs = [f"{self.revisionExpr}^0"]
+            upstream_rev = None
+            if self.upstream:
+                upstream_rev = self.GetRemote().ToLocal(self.upstream)
+                revs.append(upstream_rev)
+
             self.bare_git.rev_list(
                 "-1",
                 "--missing=allow-any",
-                "%s^0" % self.revisionExpr,
+                *revs,
                 "--",
                 log_as_error=False,
             )
+
             if self.upstream:
-                rev = self.GetRemote().ToLocal(self.upstream)
-                self.bare_git.rev_list(
-                    "-1",
-                    "--missing=allow-any",
-                    "%s^0" % rev,
-                    "--",
-                    log_as_error=False,
-                )
                 self.bare_git.merge_base(
                     "--is-ancestor",
                     self.revisionExpr,
-                    rev,
+                    upstream_rev,
                     log_as_error=False,
                 )
             return True
@@ -2447,7 +2534,7 @@ class Project:
                 old_packed_lines = []
 
                 for r in sorted(all_refs):
-                    line = "%s %s\n" % (all_refs[r], r)
+                    line = f"{all_refs[r]} {r}\n"
                     tmp_packed_lines.append(line)
                     if r not in tmp:
                         old_packed_lines.append(line)
@@ -2510,12 +2597,7 @@ class Project:
             branch = None
         else:
             branch = self.revisionExpr
-        if (
-            not self.manifest.IsMirror
-            and is_sha1
-            and depth
-            and git_require((1, 8, 3))
-        ):
+        if not self.manifest.IsMirror and is_sha1 and depth:
             # Shallow checkout of a specific commit, fetch from that commit and
             # not the heads only as the commit might be deeper in the history.
             spec.append(branch)
@@ -2578,6 +2660,20 @@ class Project:
                 # Fallthru to sleep+retry logic at the bottom.
                 pass
 
+            # TODO(b/360889369#comment24): git may gc commits incorrectly.
+            # Until the root cause is fixed, retry fetch with --refetch which
+            # will bring the repository into a good state.
+            elif gitcmd.stdout and (
+                "could not parse commit" in gitcmd.stdout
+                or "unable to parse commit" in gitcmd.stdout
+            ):
+                cmd.insert(1, "--refetch")
+                print(
+                    "could not parse commit error, retrying with refetch",
+                    file=output_redir,
+                )
+                continue
+
             # Try to prune remote branches once in case there are conflicts.
             # For example, if the remote had refs/heads/upstream, but deleted
             # that and now has refs/heads/upstream/foo.
@@ -2603,6 +2699,33 @@ class Project:
                 )
                 # Continue right away so we don't sleep as we shouldn't need to.
                 continue
+            elif (
+                ret == 128
+                and gitcmd.stdout
+                and "fatal: could not read Username" in gitcmd.stdout
+            ):
+                # User needs to be authenticated, and Git wants to prompt for
+                # username and password.
+                print(
+                    "git requires authentication, but repo cannot perform "
+                    "interactive authentication. Check git credentials.",
+                    file=output_redir,
+                )
+                break
+            elif (
+                ret == 128
+                and gitcmd.stdout
+                and "remote helper 'sso' aborted session" in gitcmd.stdout
+            ):
+                # User needs to be authenticated, and Git wants to prompt for
+                # username and password.
+                print(
+                    "git requires authentication, but repo cannot perform "
+                    "interactive authentication.",
+                    file=output_redir,
+                )
+                raise GitAuthError(gitcmd.stdout)
+                break
             elif current_branch_only and is_sha1 and ret == 128:
                 # Exit code 128 means "couldn't find the ref you asked for"; if
                 # we're in sha1 mode, we just tried sync'ing from the upstream
@@ -2617,7 +2740,7 @@ class Project:
             # one.
             if not verbose and gitcmd.stdout:
                 print(
-                    "\n%s:\n%s" % (self.name, gitcmd.stdout),
+                    f"\n{self.name}:\n{gitcmd.stdout}",
                     end="",
                     file=output_redir,
                 )
@@ -2727,6 +2850,8 @@ class Project:
     def _FetchBundle(self, srcUrl, tmpPath, dstPath, quiet, verbose):
         platform_utils.remove(dstPath, missing_ok=True)
 
+        # We do not use curl's --retry option since it generally doesn't
+        # actually retry anything; code 18 for example, it will not retry on.
         cmd = ["curl", "--fail", "--output", tmpPath, "--netrc", "--location"]
         if quiet:
             cmd += ["--silent", "--show-error"]
@@ -2752,7 +2877,7 @@ class Project:
             proc = None
             with Trace("Fetching bundle: %s", " ".join(cmd)):
                 if verbose:
-                    print("%s: Downloading bundle: %s" % (self.name, srcUrl))
+                    print(f"{self.name}: Downloading bundle: {srcUrl}")
                 stdout = None if verbose else subprocess.PIPE
                 stderr = None if verbose else subprocess.STDOUT
                 try:
@@ -2763,11 +2888,18 @@ class Project:
             (output, _) = proc.communicate()
             curlret = proc.returncode
 
-            if curlret == 22:
+            if curlret in (22, 35, 56, 92):
+                # We use --fail so curl exits with unique status.
                 # From curl man page:
-                # 22: HTTP page not retrieved. The requested url was not found
-                # or returned another error with the HTTP error code being 400
-                # or above. This return code only appears if -f, --fail is used.
+                # 22: HTTP page not retrieved.  The requested url was not found
+                #     or returned another error with the HTTP error code being
+                #     400 or above.
+                # 35: SSL connect error.  The SSL handshaking failed.  This can
+                #     be thrown by Google storage sometimes.
+                # 56: Failure in receiving network data.  This shows up with
+                #     HTTP/404 on Google storage.
+                # 92: Stream error in HTTP/2 framing layer.  Basically the same
+                #     as 22 -- Google storage sometimes throws 500's.
                 if verbose:
                     print(
                         "%s: Unable to retrieve clone.bundle; ignoring."
@@ -2801,16 +2933,18 @@ class Project:
         except OSError:
             return False
 
-    def _Checkout(self, rev, quiet=False):
+    def _Checkout(self, rev, force_checkout=False, quiet=False):
         cmd = ["checkout"]
         if quiet:
             cmd.append("-q")
+        if force_checkout:
+            cmd.append("-f")
         cmd.append(rev)
         cmd.append("--")
         if GitCommand(self, cmd).Wait() != 0:
             if self._allrefs:
                 raise GitError(
-                    "%s checkout %s " % (self.name, rev), project=self.name
+                    f"{self.name} checkout {rev} ", project=self.name
                 )
 
     def _CherryPick(self, rev, ffonly=False, record_origin=False):
@@ -2824,7 +2958,7 @@ class Project:
         if GitCommand(self, cmd).Wait() != 0:
             if self._allrefs:
                 raise GitError(
-                    "%s cherry-pick %s " % (self.name, rev), project=self.name
+                    f"{self.name} cherry-pick {rev} ", project=self.name
                 )
 
     def _LsRemote(self, refs):
@@ -2841,9 +2975,7 @@ class Project:
         cmd.append("--")
         if GitCommand(self, cmd).Wait() != 0:
             if self._allrefs:
-                raise GitError(
-                    "%s revert %s " % (self.name, rev), project=self.name
-                )
+                raise GitError(f"{self.name} revert {rev} ", project=self.name)
 
     def _ResetHard(self, rev, quiet=True):
         cmd = ["reset", "--hard"]
@@ -2852,7 +2984,7 @@ class Project:
         cmd.append(rev)
         if GitCommand(self, cmd).Wait() != 0:
             raise GitError(
-                "%s reset --hard %s " % (self.name, rev), project=self.name
+                f"{self.name} reset --hard {rev} ", project=self.name
             )
 
     def _SyncSubmodules(self, quiet=True):
@@ -2871,18 +3003,16 @@ class Project:
             cmd.extend(["--onto", onto])
         cmd.append(upstream)
         if GitCommand(self, cmd).Wait() != 0:
-            raise GitError(
-                "%s rebase %s " % (self.name, upstream), project=self.name
-            )
+            raise GitError(f"{self.name} rebase {upstream} ", project=self.name)
 
-    def _FastForward(self, head, ffonly=False):
+    def _FastForward(self, head, ffonly=False, quiet=True):
         cmd = ["merge", "--no-stat", head]
         if ffonly:
             cmd.append("--ff-only")
+        if quiet:
+            cmd.append("-q")
         if GitCommand(self, cmd).Wait() != 0:
-            raise GitError(
-                "%s merge %s " % (self.name, head), project=self.name
-            )
+            raise GitError(f"{self.name} merge {head} ", project=self.name)
 
     def _InitGitDir(self, mirror_git=None, force_sync=False, quiet=False):
         init_git_dir = not os.path.exists(self.gitdir)
@@ -2920,14 +3050,12 @@ class Project:
                             "Retrying clone after deleting %s", self.gitdir
                         )
                         try:
-                            platform_utils.rmtree(
-                                platform_utils.realpath(self.gitdir)
-                            )
+                            platform_utils.rmtree(os.path.realpath(self.gitdir))
                             if self.worktree and os.path.exists(
-                                platform_utils.realpath(self.worktree)
+                                os.path.realpath(self.worktree)
                             ):
                                 platform_utils.rmtree(
-                                    platform_utils.realpath(self.worktree)
+                                    os.path.realpath(self.worktree)
                                 )
                             return self._InitGitDir(
                                 mirror_git=mirror_git,
@@ -3013,7 +3141,7 @@ class Project:
             self._InitHooks(quiet=quiet)
 
     def _InitHooks(self, quiet=False):
-        hooks = platform_utils.realpath(os.path.join(self.objdir, "hooks"))
+        hooks = os.path.realpath(os.path.join(self.objdir, "hooks"))
         if not os.path.exists(hooks):
             os.makedirs(hooks)
 
@@ -3156,9 +3284,9 @@ class Project:
             dst_path = os.path.join(destdir, name)
             src_path = os.path.join(srcdir, name)
 
-            dst = platform_utils.realpath(dst_path)
+            dst = os.path.realpath(dst_path)
             if os.path.lexists(dst):
-                src = platform_utils.realpath(src_path)
+                src = os.path.realpath(src_path)
                 # Fail if the links are pointing to the wrong place.
                 if src != dst:
                     logger.error(
@@ -3171,8 +3299,9 @@ class Project:
                         "--force-sync not enabled; cannot overwrite a local "
                         "work tree. If you're comfortable with the "
                         "possibility of losing the work tree's git metadata,"
-                        " use `repo sync --force-sync {0}` to "
-                        "proceed.".format(self.RelPath(local=False)),
+                        " use "
+                        f"`repo sync --force-sync {self.RelPath(local=False)}` "
+                        "to proceed.",
                         project=self.name,
                     )
 
@@ -3193,10 +3322,10 @@ class Project:
         if copy_all:
             to_copy = platform_utils.listdir(gitdir)
 
-        dotgit = platform_utils.realpath(dotgit)
+        dotgit = os.path.realpath(dotgit)
         for name in set(to_copy).union(to_symlink):
             try:
-                src = platform_utils.realpath(os.path.join(gitdir, name))
+                src = os.path.realpath(os.path.join(gitdir, name))
                 dst = os.path.join(dotgit, name)
 
                 if os.path.lexists(dst):
@@ -3243,24 +3372,29 @@ class Project:
             setting = fp.read()
             assert setting.startswith("gitdir:")
             git_worktree_path = setting.split(":", 1)[1].strip()
-        # Some platforms (e.g. Windows) won't let us update dotgit in situ
-        # because of file permissions.  Delete it and recreate it from scratch
-        # to avoid.
-        platform_utils.remove(dotgit)
-        # Use relative path from checkout->worktree & maintain Unix line endings
-        # on all OS's to match git behavior.
-        with open(dotgit, "w", newline="\n") as fp:
-            print(
-                "gitdir:",
-                os.path.relpath(git_worktree_path, self.worktree),
-                file=fp,
-            )
-        # Use relative path from worktree->checkout & maintain Unix line endings
-        # on all OS's to match git behavior.
-        with open(
-            os.path.join(git_worktree_path, "gitdir"), "w", newline="\n"
-        ) as fp:
-            print(os.path.relpath(dotgit, git_worktree_path), file=fp)
+
+        # `gitdir` maybe be either relative or absolute depending on the
+        # behavior of the local copy of git, so only convert the path to
+        # relative if it needs to be converted.
+        if os.path.isabs(git_worktree_path):
+            # Some platforms (e.g. Windows) won't let us update dotgit in situ
+            # because of file permissions.  Delete it and recreate it from
+            # scratch to avoid.
+            platform_utils.remove(dotgit)
+            # Use relative path from checkout->worktree & maintain Unix line
+            # endings on all OS's to match git behavior.
+            with open(dotgit, "w", newline="\n") as fp:
+                print(
+                    "gitdir:",
+                    os.path.relpath(git_worktree_path, self.worktree),
+                    file=fp,
+                )
+            # Use relative path from worktree->checkout & maintain Unix line
+            # endings on all OS's to match git behavior.
+            with open(
+                os.path.join(git_worktree_path, "gitdir"), "w", newline="\n"
+            ) as fp:
+                print(os.path.relpath(dotgit, git_worktree_path), file=fp)
 
         self._InitMRef()
 
@@ -3285,7 +3419,7 @@ class Project:
         if not platform_utils.islink(dotgit) and platform_utils.isdir(dotgit):
             self._MigrateOldWorkTreeGitDir(dotgit, project=self.name)
 
-        init_dotgit = not os.path.exists(dotgit)
+        init_dotgit = not os.path.lexists(dotgit)
         if self.use_git_worktrees:
             if init_dotgit:
                 self._InitGitWorktree()
@@ -3293,9 +3427,7 @@ class Project:
         else:
             if not init_dotgit:
                 # See if the project has changed.
-                if platform_utils.realpath(
-                    self.gitdir
-                ) != platform_utils.realpath(dotgit):
+                if os.path.realpath(self.gitdir) != os.path.realpath(dotgit):
                     platform_utils.remove(dotgit)
 
             if init_dotgit or not os.path.exists(dotgit):
@@ -3686,12 +3818,12 @@ class Project:
                 config = kwargs.pop("config", None)
                 for k in kwargs:
                     raise TypeError(
-                        "%s() got an unexpected keyword argument %r" % (name, k)
+                        f"{name}() got an unexpected keyword argument {k!r}"
                     )
                 if config is not None:
                     for k, v in config.items():
                         cmdv.append("-c")
-                        cmdv.append("%s=%s" % (k, v))
+                        cmdv.append(f"{k}={v}")
                 cmdv.append(name)
                 cmdv.extend(args)
                 p = GitCommand(
@@ -3752,17 +3884,20 @@ class _Failure:
 
 
 class _Later:
-    def __init__(self, project, action):
+    def __init__(self, project, action, quiet):
         self.project = project
         self.action = action
+        self.quiet = quiet
 
     def Run(self, syncbuf):
         out = syncbuf.out
-        out.project("project %s/", self.project.RelPath(local=False))
-        out.nl()
+        if not self.quiet:
+            out.project("project %s/", self.project.RelPath(local=False))
+            out.nl()
         try:
             self.action()
-            out.nl()
+            if not self.quiet:
+                out.nl()
             return True
         except GitError:
             out.nl()
@@ -3798,11 +3933,11 @@ class SyncBuffer:
         self._failures.append(_Failure(project, err))
         self._MarkUnclean()
 
-    def later1(self, project, what):
-        self._later_queue1.append(_Later(project, what))
+    def later1(self, project, what, quiet):
+        self._later_queue1.append(_Later(project, what, quiet))
 
-    def later2(self, project, what):
-        self._later_queue2.append(_Later(project, what))
+    def later2(self, project, what, quiet):
+        self._later_queue2.append(_Later(project, what, quiet))
 
     def Finish(self):
         self._PrintMessages()
@@ -3911,13 +4046,13 @@ class RepoProject(MetaProject):
 class ManifestProject(MetaProject):
     """The MetaProject for manifests."""
 
-    def MetaBranchSwitch(self, submodules=False):
+    def MetaBranchSwitch(self, submodules=False, verbose=False):
         """Prepare for manifest branch switch."""
 
         # detach and delete manifest branch, allowing a new
         # branch to take over
         syncbuf = SyncBuffer(self.config, detach_head=True)
-        self.Sync_LocalHalf(syncbuf, submodules=submodules)
+        self.Sync_LocalHalf(syncbuf, submodules=submodules, verbose=verbose)
         syncbuf.Finish()
 
         return (
@@ -4448,10 +4583,10 @@ class ManifestProject(MetaProject):
                 return False
 
             if manifest_branch:
-                self.MetaBranchSwitch(submodules=submodules)
+                self.MetaBranchSwitch(submodules=submodules, verbose=verbose)
 
             syncbuf = SyncBuffer(self.config)
-            self.Sync_LocalHalf(syncbuf, submodules=submodules)
+            self.Sync_LocalHalf(syncbuf, submodules=submodules, verbose=verbose)
             syncbuf.Finish()
 
             if is_new or self.CurrentBranch is None:

release/update-hooks

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helper tool for updating hooks from their various upstreams."""
+
+import argparse
+import base64
+import json
+from pathlib import Path
+import sys
+from typing import List, Optional
+import urllib.request
+
+
+assert sys.version_info >= (3, 8), "Python 3.8+ required"
+
+
+TOPDIR = Path(__file__).resolve().parent.parent
+HOOKS_DIR = TOPDIR / "hooks"
+
+
+def update_hook_commit_msg() -> None:
+    """Update commit-msg hook from Gerrit."""
+    hook = HOOKS_DIR / "commit-msg"
+    print(
+        f"{hook.name}: Updating from https://gerrit.googlesource.com/gerrit/"
+        "+/HEAD/resources/com/google/gerrit/server/tools/root/hooks/commit-msg"
+    )
+
+    # Get the current commit.
+    url = "https://gerrit.googlesource.com/gerrit/+/HEAD?format=JSON"
+    with urllib.request.urlopen(url) as fp:
+        data = fp.read()
+    # Discard the xss protection.
+    data = data.split(b"\n", 1)[1]
+    data = json.loads(data)
+    commit = data["commit"]
+
+    # Fetch the data for that commit.
+    url = (
+        f"https://gerrit.googlesource.com/gerrit/+/{commit}/"
+        "resources/com/google/gerrit/server/tools/root/hooks/commit-msg"
+    )
+    with urllib.request.urlopen(f"{url}?format=TEXT") as fp:
+        data = fp.read()
+
+    # gitiles base64 encodes text data.
+    data = base64.b64decode(data)
+
+    # Inject header into the hook.
+    lines = data.split(b"\n")
+    lines = (
+        lines[:1]
+        + [
+            b"# DO NOT EDIT THIS FILE",
+            (
+                b"# All updates should be sent upstream: "
+                b"https://gerrit.googlesource.com/gerrit/"
+            ),
+            f"# This is synced from commit: {commit}".encode("utf-8"),
+            b"# DO NOT EDIT THIS FILE",
+        ]
+        + lines[1:]
+    )
+    data = b"\n".join(lines)
+
+    # Update the hook.
+    hook.write_bytes(data)
+    hook.chmod(0o755)
+
+
+def update_hook_pre_auto_gc() -> None:
+    """Update pre-auto-gc hook from git."""
+    hook = HOOKS_DIR / "pre-auto-gc"
+    print(
+        f"{hook.name}: Updating from https://github.com/git/git/"
+        "HEAD/contrib/hooks/pre-auto-gc-battery"
+    )
+
+    # Get the current commit.
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    url = "https://api.github.com/repos/git/git/git/refs/heads/master"
+    req = urllib.request.Request(url, headers=headers)
+    with urllib.request.urlopen(req) as fp:
+        data = fp.read()
+    data = json.loads(data)
+
+    # Fetch the data for that commit.
+    commit = data["object"]["sha"]
+    url = (
+        f"https://raw.githubusercontent.com/git/git/{commit}/"
+        "contrib/hooks/pre-auto-gc-battery"
+    )
+    with urllib.request.urlopen(url) as fp:
+        data = fp.read()
+
+    # Inject header into the hook.
+    lines = data.split(b"\n")
+    lines = (
+        lines[:1]
+        + [
+            b"# DO NOT EDIT THIS FILE",
+            (
+                b"# All updates should be sent upstream: "
+                b"https://github.com/git/git/"
+            ),
+            f"# This is synced from commit: {commit}".encode("utf-8"),
+            b"# DO NOT EDIT THIS FILE",
+        ]
+        + lines[1:]
+    )
+    data = b"\n".join(lines)
+
+    # Update the hook.
+    hook.write_bytes(data)
+    hook.chmod(0o755)
+
+
+def main(argv: Optional[List[str]] = None) -> Optional[int]:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.parse_args(argv)
+
+    update_hook_commit_msg()
+    update_hook_pre_auto_gc()
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

repo

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright (C) 2008 The Android Open Source Project
 #
@@ -33,7 +33,7 @@ import sys
 # bit more flexible with older systems.  See that file for more details on the
 # versions we select.
 MIN_PYTHON_VERSION_SOFT = (3, 6)
-MIN_PYTHON_VERSION_HARD = (3, 5)
+MIN_PYTHON_VERSION_HARD = (3, 6)
 
 
 # Keep basic logic in sync with repo_trace.py.
@@ -79,24 +79,13 @@ def check_python_version():
     major = ver.major
     minor = ver.minor
 
-    # Abort on very old Python 2 versions.
-    if (major, minor) < (2, 7):
-        print(
-            "repo: error: Your Python version is too old. "
-            "Please use Python {}.{} or newer instead.".format(
-                *MIN_PYTHON_VERSION_SOFT
-            ),
-            file=sys.stderr,
-        )
-        sys.exit(1)
-
-    # Try to re-exec the version specific Python 3 if needed.
+    # Try to re-exec the version specific Python if needed.
     if (major, minor) < MIN_PYTHON_VERSION_SOFT:
         # Python makes releases ~once a year, so try our min version +10 to help
         # bridge the gap.  This is the fallback anyways so perf isn't critical.
         min_major, min_minor = MIN_PYTHON_VERSION_SOFT
         for inc in range(0, 10):
-            reexec("python{}.{}".format(min_major, min_minor + inc))
+            reexec(f"python{min_major}.{min_minor + inc}")
 
         # Fallback to older versions if possible.
         for inc in range(
@@ -105,47 +94,12 @@ def check_python_version():
             # Don't downgrade, and don't reexec ourselves (which would infinite loop).
             if (min_major, min_minor - inc) <= (major, minor):
                 break
-            reexec("python{}.{}".format(min_major, min_minor - inc))
-
-        # Try the generic Python 3 wrapper, but only if it's new enough.  If it
-        # isn't, we want to just give up below and make the user resolve things.
-        try:
-            proc = subprocess.Popen(
-                [
-                    "python3",
-                    "-c",
-                    "import sys; "
-                    "print(sys.version_info.major, sys.version_info.minor)",
-                ],
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-            )
-            (output, _) = proc.communicate()
-            python3_ver = tuple(int(x) for x in output.decode("utf-8").split())
-        except (OSError, subprocess.CalledProcessError):
-            python3_ver = None
-
-        # If the python3 version looks like it's new enough, give it a try.
-        if (
-            python3_ver
-            and python3_ver >= MIN_PYTHON_VERSION_HARD
-            and python3_ver != (major, minor)
-        ):
-            reexec("python3")
+            reexec(f"python{min_major}.{min_minor - inc}")
 
         # We're still here, so diagnose things for the user.
-        if major < 3:
+        if (major, minor) < MIN_PYTHON_VERSION_HARD:
             print(
-                "repo: error: Python 2 is no longer supported; "
-                "Please upgrade to Python {}.{}+.".format(
-                    *MIN_PYTHON_VERSION_HARD
-                ),
-                file=sys.stderr,
-            )
-            sys.exit(1)
-        elif (major, minor) < MIN_PYTHON_VERSION_HARD:
-            print(
-                "repo: error: Python 3 version is too old; "
+                "repo: error: Python version is too old; "
                 "Please use Python {}.{} or newer.".format(
                     *MIN_PYTHON_VERSION_HARD
                 ),
@@ -170,7 +124,7 @@ if not REPO_REV:
 BUG_URL = "https://issues.gerritcodereview.com/issues/new?component=1370071"
 
 # increment this whenever we make important changes to this script
-VERSION = (2, 39)
+VERSION = (2, 50)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (2, 3)
@@ -256,15 +210,11 @@ GIT = "git"  # our git command
 # NB: The version of git that the repo launcher requires may be much older than
 # the version of git that the main repo source tree requires.  Keeping this at
 # an older version also makes it easier for users to upgrade/rollback as needed.
-#
-# git-1.7 is in (EOL) Ubuntu Precise.
-MIN_GIT_VERSION = (1, 7, 2)  # minimum supported git version
+MIN_GIT_VERSION = (1, 7, 9)  # minimum supported git version
 repodir = ".repo"  # name of repo's private directory
 S_repo = "repo"  # special repo repository
 S_manifests = "manifests"  # special manifest repository
 REPO_MAIN = S_repo + "/main.py"  # main script
-GITC_CONFIG_FILE = "/gitc/.config"
-GITC_FS_ROOT_DIR = "/gitc/manifest-rw/"
 
 
 import collections
@@ -274,19 +224,8 @@ import optparse
 import re
 import shutil
 import stat
-
-
-if sys.version_info[0] == 3:
-    import urllib.error
-    import urllib.request
-else:
-    import imp
-
-    import urllib2
-
-    urllib = imp.new_module("urllib")
-    urllib.request = urllib2
-    urllib.error = urllib2
+import urllib.error
+import urllib.request
 
 
 repo_config_dir = os.getenv("REPO_CONFIG_DIR", os.path.expanduser("~"))
@@ -294,12 +233,9 @@ home_dot_repo = os.path.join(repo_config_dir, ".repoconfig")
 gpg_dir = os.path.join(home_dot_repo, "gnupg")
 
 
-def GetParser(gitc_init=False):
+def GetParser():
     """Setup the CLI parser."""
-    if gitc_init:
-        sys.exit("repo: fatal: GITC not supported.")
-    else:
-        usage = "repo init [options] [-u] url"
+    usage = "repo init [options] [-u] url"
 
     parser = optparse.OptionParser(usage=usage)
     InitParser(parser)
@@ -341,6 +277,12 @@ def InitParser(parser):
         metavar="REVISION",
         help="manifest branch or revision (use HEAD for default)",
     )
+    group.add_option(
+        "--manifest-upstream-branch",
+        help="when a commit is provided to --manifest-branch, this "
+        "is the name of the git ref in which the commit can be found",
+        metavar="BRANCH",
+    )
     group.add_option(
         "-m",
         "--manifest-name",
@@ -566,8 +508,7 @@ def run_command(cmd, **kwargs):
             return output.decode("utf-8")
         except UnicodeError:
             print(
-                "repo: warning: Invalid UTF-8 output:\ncmd: %r\n%r"
-                % (cmd, output),
+                f"repo: warning: Invalid UTF-8 output:\ncmd: {cmd!r}\n{output}",
                 file=sys.stderr,
             )
             return output.decode("utf-8", "backslashreplace")
@@ -590,20 +531,17 @@ def run_command(cmd, **kwargs):
     # If things failed, print useful debugging output.
     if check and ret.returncode:
         print(
-            'repo: error: "%s" failed with exit status %s'
-            % (cmd[0], ret.returncode),
-            file=sys.stderr,
-        )
-        print(
-            "  cwd: %s\n  cmd: %r" % (kwargs.get("cwd", os.getcwd()), cmd),
+            f'repo: error: "{cmd[0]}" failed with exit status {ret.returncode}',
             file=sys.stderr,
         )
+        cwd = kwargs.get("cwd", os.getcwd())
+        print(f"  cwd: {cwd}\n  cmd: {cmd!r}", file=sys.stderr)
 
         def _print_output(name, output):
             if output:
                 print(
-                    "  %s:\n  >> %s"
-                    % (name, "\n  >> ".join(output.splitlines())),
+                    f"  {name}:"
+                    + "".join(f"\n  >> {x}" for x in output.splitlines()),
                     file=sys.stderr,
                 )
 
@@ -614,49 +552,6 @@ def run_command(cmd, **kwargs):
     return ret
 
 
-_gitc_manifest_dir = None
-
-
-def get_gitc_manifest_dir():
-    global _gitc_manifest_dir
-    if _gitc_manifest_dir is None:
-        _gitc_manifest_dir = ""
-        try:
-            with open(GITC_CONFIG_FILE) as gitc_config:
-                for line in gitc_config:
-                    match = re.match("gitc_dir=(?P<gitc_manifest_dir>.*)", line)
-                    if match:
-                        _gitc_manifest_dir = match.group("gitc_manifest_dir")
-        except OSError:
-            pass
-    return _gitc_manifest_dir
-
-
-def gitc_parse_clientdir(gitc_fs_path):
-    """Parse a path in the GITC FS and return its client name.
-
-    Args:
-        gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR.
-
-    Returns:
-        The GITC client name.
-    """
-    if gitc_fs_path == GITC_FS_ROOT_DIR:
-        return None
-    if not gitc_fs_path.startswith(GITC_FS_ROOT_DIR):
-        manifest_dir = get_gitc_manifest_dir()
-        if manifest_dir == "":
-            return None
-        if manifest_dir[-1] != "/":
-            manifest_dir += "/"
-        if gitc_fs_path == manifest_dir:
-            return None
-        if not gitc_fs_path.startswith(manifest_dir):
-            return None
-        return gitc_fs_path.split(manifest_dir)[1].split("/")[0]
-    return gitc_fs_path.split(GITC_FS_ROOT_DIR)[1].split("/")[0]
-
-
 class CloneFailure(Exception):
 
     """Indicate the remote clone of repo itself failed."""
@@ -695,9 +590,9 @@ def check_repo_rev(dst, rev, repo_verify=True, quiet=False):
     return (remote_ref, rev)
 
 
-def _Init(args, gitc_init=False):
+def _Init(args):
     """Installs repo by cloning it over the network."""
-    parser = GetParser(gitc_init=gitc_init)
+    parser = GetParser()
     opt, args = parser.parse_args(args)
     if args:
         if not opt.manifest_url:
@@ -719,7 +614,7 @@ def _Init(args, gitc_init=False):
     except OSError as e:
         if e.errno != errno.EEXIST:
             print(
-                "fatal: cannot make %s directory: %s" % (repodir, e.strerror),
+                f"fatal: cannot make {repodir} directory: {e.strerror}",
                 file=sys.stderr,
             )
             # Don't raise CloneFailure; that would delete the
@@ -817,7 +712,7 @@ def _CheckGitVersion():
     if ver_act < MIN_GIT_VERSION:
         need = ".".join(map(str, MIN_GIT_VERSION))
         print(
-            "fatal: git %s or later required; found %s" % (need, ver_act.full),
+            f"fatal: git {need} or later required; found {ver_act.full}",
             file=sys.stderr,
         )
         raise CloneFailure()
@@ -836,7 +731,8 @@ def SetGitTrace2ParentSid(env=None):
     KEY = "GIT_TRACE2_PARENT_SID"
 
     now = datetime.datetime.now(datetime.timezone.utc)
-    value = "repo-%s-P%08x" % (now.strftime("%Y%m%dT%H%M%SZ"), os.getpid())
+    timestamp = now.strftime("%Y%m%dT%H%M%SZ")
+    value = f"repo-{timestamp}-P{os.getpid():08x}"
 
     # If it's already set, then append ourselves.
     if KEY in env:
@@ -880,8 +776,7 @@ def SetupGnuPG(quiet):
     except OSError as e:
         if e.errno != errno.EEXIST:
             print(
-                "fatal: cannot make %s directory: %s"
-                % (home_dot_repo, e.strerror),
+                f"fatal: cannot make {home_dot_repo} directory: {e.strerror}",
                 file=sys.stderr,
             )
             sys.exit(1)
@@ -891,15 +786,15 @@ def SetupGnuPG(quiet):
     except OSError as e:
         if e.errno != errno.EEXIST:
             print(
-                "fatal: cannot make %s directory: %s" % (gpg_dir, e.strerror),
+                f"fatal: cannot make {gpg_dir} directory: {e.strerror}",
                 file=sys.stderr,
             )
             sys.exit(1)
 
     if not quiet:
         print(
-            "repo: Updating release signing keys to keyset ver %s"
-            % (".".join(str(x) for x in KEYRING_VERSION),)
+            "repo: Updating release signing keys to keyset ver "
+            + ".".join(str(x) for x in KEYRING_VERSION),
         )
     # NB: We use --homedir (and cwd below) because some environments (Windows) do
     # not correctly handle full native paths.  We avoid the issue by changing to
@@ -951,7 +846,7 @@ def _GetRepoConfig(name):
         return None
     else:
         print(
-            "repo: error: git %s failed:\n%s" % (" ".join(cmd), ret.stderr),
+            f"repo: error: git {' '.join(cmd)} failed:\n{ret.stderr}",
             file=sys.stderr,
         )
         raise RunError()
@@ -1064,7 +959,7 @@ def _Clone(url, cwd, clone_bundle, quiet, verbose):
         os.mkdir(cwd)
     except OSError as e:
         print(
-            "fatal: cannot make %s directory: %s" % (cwd, e.strerror),
+            f"fatal: cannot make {cwd} directory: {e.strerror}",
             file=sys.stderr,
         )
         raise CloneFailure()
@@ -1104,7 +999,7 @@ def resolve_repo_rev(cwd, committish):
         ret = run_git(
             "rev-parse",
             "--verify",
-            "%s^{commit}" % (committish,),
+            f"{committish}^{{commit}}",
             cwd=cwd,
             check=False,
         )
@@ -1117,7 +1012,7 @@ def resolve_repo_rev(cwd, committish):
         rev = resolve("refs/remotes/origin/%s" % committish)
         if rev is None:
             print(
-                'repo: error: unknown branch "%s"' % (committish,),
+                f'repo: error: unknown branch "{committish}"',
                 file=sys.stderr,
             )
             raise CloneFailure()
@@ -1130,7 +1025,8 @@ def resolve_repo_rev(cwd, committish):
         rev = resolve(remote_ref)
         if rev is None:
             print(
-                'repo: error: unknown tag "%s"' % (committish,), file=sys.stderr
+                f'repo: error: unknown tag "{committish}"',
+                file=sys.stderr,
             )
             raise CloneFailure()
         return (remote_ref, rev)
@@ -1138,12 +1034,12 @@ def resolve_repo_rev(cwd, committish):
     # See if it's a short branch name.
     rev = resolve("refs/remotes/origin/%s" % committish)
     if rev:
-        return ("refs/heads/%s" % (committish,), rev)
+        return (f"refs/heads/{committish}", rev)
 
     # See if it's a tag.
-    rev = resolve("refs/tags/%s" % committish)
+    rev = resolve(f"refs/tags/{committish}")
     if rev:
-        return ("refs/tags/%s" % (committish,), rev)
+        return (f"refs/tags/{committish}", rev)
 
     # See if it's a commit.
     rev = resolve(committish)
@@ -1152,7 +1048,8 @@ def resolve_repo_rev(cwd, committish):
 
     # Give up!
     print(
-        'repo: error: unable to resolve "%s"' % (committish,), file=sys.stderr
+        f'repo: error: unable to resolve "{committish}"',
+        file=sys.stderr,
     )
     raise CloneFailure()
 
@@ -1168,8 +1065,8 @@ def verify_rev(cwd, remote_ref, rev, quiet):
         if not quiet:
             print(file=sys.stderr)
             print(
-                "warning: '%s' is not signed; falling back to signed release '%s'"
-                % (remote_ref, cur),
+                f"warning: '{remote_ref}' is not signed; "
+                f"falling back to signed release '{cur}'",
                 file=sys.stderr,
             )
             print(file=sys.stderr)
@@ -1219,10 +1116,10 @@ class _Options:
 def _ExpandAlias(name):
     """Look up user registered aliases."""
     # We don't resolve aliases for existing subcommands.  This matches git.
-    if name in {"gitc-init", "help", "init"}:
+    if name in {"help", "init"}:
         return name, []
 
-    alias = _GetRepoConfig("alias.%s" % (name,))
+    alias = _GetRepoConfig(f"alias.{name}")
     if alias is None:
         return name, []
 
@@ -1278,7 +1175,6 @@ class Requirements:
             with open(path, "rb") as f:
                 data = f.read()
         except OSError:
-            # NB: EnvironmentError is used for Python 2 & 3 compatibility.
             # If we couldn't open the file, assume it's an old source tree.
             return None
 
@@ -1298,13 +1194,13 @@ class Requirements:
 
         return cls(json_data)
 
-    def _get_soft_ver(self, pkg):
+    def get_soft_ver(self, pkg):
         """Return the soft version for |pkg| if it exists."""
-        return self.requirements.get(pkg, {}).get("soft", ())
+        return tuple(self.requirements.get(pkg, {}).get("soft", ()))
 
-    def _get_hard_ver(self, pkg):
+    def get_hard_ver(self, pkg):
         """Return the hard version for |pkg| if it exists."""
-        return self.requirements.get(pkg, {}).get("hard", ())
+        return tuple(self.requirements.get(pkg, {}).get("hard", ()))
 
     @staticmethod
     def _format_ver(ver):
@@ -1314,22 +1210,24 @@ class Requirements:
     def assert_ver(self, pkg, curr_ver):
         """Verify |pkg|'s |curr_ver| is new enough."""
         curr_ver = tuple(curr_ver)
-        soft_ver = tuple(self._get_soft_ver(pkg))
-        hard_ver = tuple(self._get_hard_ver(pkg))
+        soft_ver = tuple(self.get_soft_ver(pkg))
+        hard_ver = tuple(self.get_hard_ver(pkg))
         if curr_ver < hard_ver:
             print(
-                'repo: error: Your version of "%s" (%s) is unsupported; '
-                "Please upgrade to at least version %s to continue."
-                % (pkg, self._format_ver(curr_ver), self._format_ver(soft_ver)),
+                f'repo: error: Your version of "{pkg}" '
+                f"({self._format_ver(curr_ver)}) is unsupported; "
+                "Please upgrade to at least version "
+                f"{self._format_ver(soft_ver)} to continue.",
                 file=sys.stderr,
             )
             sys.exit(1)
 
         if curr_ver < soft_ver:
             print(
-                'repo: warning: Your version of "%s" (%s) is no longer supported; '
-                "Please upgrade to at least version %s to avoid breakage."
-                % (pkg, self._format_ver(curr_ver), self._format_ver(soft_ver)),
+                f'repo: error: Your version of "{pkg}" '
+                f"({self._format_ver(curr_ver)}) is no longer supported; "
+                "Please upgrade to at least version "
+                f"{self._format_ver(soft_ver)} to continue.",
                 file=sys.stderr,
             )
 
@@ -1346,10 +1244,6 @@ class Requirements:
 
 
 def _Usage():
-    gitc_usage = ""
-    if get_gitc_manifest_dir():
-        gitc_usage = "  gitc-init Initialize a GITC Client.\n"
-
     print(
         """usage: repo COMMAND [ARGS]
 
@@ -1358,9 +1252,7 @@ repo is not yet installed.  Use "repo init" to install it here.
 The most commonly used repo commands are:
 
   init      Install repo in the current working directory
-"""
-        + gitc_usage
-        + """  help      Display detailed help on a command
+  help      Display detailed help on a command
 
 For access to the full online help, install repo ("repo init").
 """
@@ -1371,8 +1263,8 @@ For access to the full online help, install repo ("repo init").
 
 def _Help(args):
     if args:
-        if args[0] in {"init", "gitc-init"}:
-            parser = GetParser(gitc_init=args[0] == "gitc-init")
+        if args[0] in {"init"}:
+            parser = GetParser()
             parser.print_help()
             sys.exit(0)
         else:
@@ -1389,21 +1281,16 @@ def _Help(args):
 
 def _Version():
     """Show version information."""
+    git_version = ParseGitVersion()
     print("<repo not installed>")
-    print("repo launcher version %s" % (".".join(str(x) for x in VERSION),))
-    print("       (from %s)" % (__file__,))
-    print("git %s" % (ParseGitVersion().full,))
-    print("Python %s" % sys.version)
+    print(f"repo launcher version {'.'.join(str(x) for x in VERSION)}")
+    print(f"       (from {__file__})")
+    print(f"git {git_version.full}" if git_version else "git not installed")
+    print(f"Python {sys.version}")
     uname = platform.uname()
-    if sys.version_info.major < 3:
-        # Python 3 returns a named tuple, but Python 2 is simpler.
-        print(uname)
-    else:
-        print("OS %s %s (%s)" % (uname.system, uname.release, uname.version))
-        print(
-            "CPU %s (%s)"
-            % (uname.machine, uname.processor if uname.processor else "unknown")
-        )
+    print(f"OS {uname.system} {uname.release} ({uname.version})")
+    processor = uname.processor if uname.processor else "unknown"
+    print(f"CPU {uname.machine} ({processor})")
     print("Bug reports:", BUG_URL)
     sys.exit(0)
 
@@ -1431,11 +1318,11 @@ def _RunSelf(wrapper_path):
     my_main = os.path.join(my_dir, "main.py")
     my_git = os.path.join(my_dir, ".git")
 
-    if os.path.isfile(my_main) and os.path.isdir(my_git):
+    if os.path.isfile(my_main):
         for name in ["git_config.py", "project.py", "subcmds"]:
             if not os.path.exists(os.path.join(my_dir, name)):
                 return None, None
-        return my_main, my_git
+        return my_main, my_git if os.path.isdir(my_git) else None
     return None, None
 
 
@@ -1466,23 +1353,11 @@ def main(orig_args):
     # We run this early as we run some git commands ourselves.
     SetGitTrace2ParentSid()
 
-    repo_main, rel_repo_dir = None, None
-    # Don't use the local repo copy, make sure to switch to the gitc client first.
-    if cmd != "gitc-init":
-        repo_main, rel_repo_dir = _FindRepo()
+    repo_main, rel_repo_dir = _FindRepo()
 
     wrapper_path = os.path.abspath(__file__)
     my_main, my_git = _RunSelf(wrapper_path)
 
-    cwd = os.getcwd()
-    if get_gitc_manifest_dir() and cwd.startswith(get_gitc_manifest_dir()):
-        print(
-            "error: repo cannot be used in the GITC local manifest directory."
-            "\nIf you want to work on this GITC client please rerun this "
-            "command from the corresponding client under /gitc/",
-            file=sys.stderr,
-        )
-        sys.exit(1)
     if not repo_main:
         # Only expand aliases here since we'll be parsing the CLI ourselves.
         # If we had repo_main, alias expansion would happen in main.py.
@@ -1497,11 +1372,11 @@ def main(orig_args):
             _Version()
         if not cmd:
             _NotInstalled()
-        if cmd == "init" or cmd == "gitc-init":
+        if cmd == "init":
             if my_git:
                 _SetDefaultsTo(my_git)
             try:
-                _Init(args, gitc_init=(cmd == "gitc-init"))
+                _Init(args)
             except CloneFailure:
                 path = os.path.join(repodir, S_repo)
                 print(
@@ -1527,6 +1402,14 @@ def main(orig_args):
     if reqs:
         reqs.assert_all()
 
+    # Python 3.11 introduces PYTHONSAFEPATH and the -P flag which, if enabled,
+    # does not prepend the script's directory to sys.path by default.
+    # repo relies on this import path, so add directory of REPO_MAIN to
+    # PYTHONPATH so that this continues to work when PYTHONSAFEPATH is enabled.
+    python_paths = os.environ.get("PYTHONPATH", "").split(os.pathsep)
+    new_python_paths = [os.path.join(rel_repo_dir, S_repo)] + python_paths
+    os.environ["PYTHONPATH"] = os.pathsep.join(new_python_paths)
+
     ver_str = ".".join(map(str, VERSION))
     me = [
         sys.executable,

repo_logging.py

@@ -39,8 +39,8 @@ class _LogColoring(Coloring):
 
     def __init__(self, config):
         super().__init__(config, "logs")
-        self.error = self.colorer("error", fg="red")
-        self.warning = self.colorer("warn", fg="yellow")
+        self.error = self.nofmt_colorer("error", fg="red")
+        self.warning = self.nofmt_colorer("warn", fg="yellow")
         self.levelMap = {
             "WARNING": self.warning,
             "ERROR": self.error,
@@ -77,6 +77,7 @@ class RepoLogger(logging.Logger):
 
         if not err.aggregate_errors:
             self.error("Repo command failed: %s", type(err).__name__)
+            self.error("\t%s", str(err))
             return
 
         self.error(

requirements.json

@@ -46,12 +46,14 @@
 
   # Supported git versions.
   #
-  # git-1.7.2 is in Debian Squeeze.
-  # git-1.7.9 is in Ubuntu Precise.
   # git-1.9.1 is in Ubuntu Trusty.
-  # git-1.7.10 is in Debian Wheezy.
+  # git-2.1.4 is in Debian Jessie.
+  # git-2.7.4 is in Ubuntu Xenial.
+  # git-2.11.0 is in Debian Stretch.
+  # git-2.17.0 is in Ubuntu Bionic.
+  # git-2.20.1 is in Debian Buster.
   "git": {
-    "hard": [1, 7, 2],
-    "soft": [1, 9, 1]
+    "hard": [1, 9, 1],
+    "soft": [2, 7, 4]
   }
 }

run_tests

@@ -32,6 +32,7 @@ def run_black():
     extra_programs = [
         "repo",
         "run_tests",
+        "release/update-hooks",
         "release/update-manpages",
     ]
     return subprocess.run(

ssh.py

@@ -24,6 +24,7 @@ import sys
 import tempfile
 import time
 
+from git_command import git
 import platform_utils
 from repo_trace import Trace
 
@@ -57,8 +58,12 @@ def version():
     except FileNotFoundError:
         print("fatal: ssh not installed", file=sys.stderr)
         sys.exit(1)
-    except subprocess.CalledProcessError:
-        print("fatal: unable to detect ssh version", file=sys.stderr)
+    except subprocess.CalledProcessError as e:
+        print(
+            "fatal: unable to detect ssh version"
+            f" (code={e.returncode}, output={e.stdout})",
+            file=sys.stderr,
+        )
         sys.exit(1)
 
 
@@ -165,7 +170,7 @@ class ProxyManager:
         # Check to see whether we already think that the master is running; if
         # we think it's already running, return right away.
         if port is not None:
-            key = "%s:%s" % (host, port)
+            key = f"{host}:{port}"
         else:
             key = host
 
@@ -207,7 +212,33 @@ class ProxyManager:
                 # and print to the log there.
                 pass
 
-        command = command_base[:1] + ["-M", "-N"] + command_base[1:]
+        # Git protocol V2 is a new feature in git 2.18.0, made default in
+        # git 2.26.0
+        # It is faster and more efficient than V1.
+        # To enable it when using SSH, the environment variable GIT_PROTOCOL
+        # must be set in the SSH side channel when establishing the connection
+        # to the git server.
+        # See https://git-scm.com/docs/protocol-v2#_ssh_and_file_transport
+        # Normally git does this by itself. But here, where the SSH connection
+        # is established manually over ControlMaster via the repo-tool, it must
+        # be passed in explicitly instead.
+        # Based on https://git-scm.com/docs/gitprotocol-pack#_extra_parameters,
+        # GIT_PROTOCOL is considered an "Extra Parameter" and must be ignored
+        # by servers that do not understand it. This means that it is safe to
+        # set it even when connecting to older servers.
+        # It should also be safe to set the environment variable for older
+        # local git versions, since it is only part of the ssh side channel.
+        git_protocol_version = _get_git_protocol_version()
+        ssh_git_protocol_args = [
+            "-o",
+            f"SetEnv GIT_PROTOCOL=version={git_protocol_version}",
+        ]
+
+        command = (
+            command_base[:1]
+            + ["-M", "-N", *ssh_git_protocol_args]
+            + command_base[1:]
+        )
         p = None
         try:
             with Trace("Call to ssh: %s", " ".join(command)):
@@ -289,3 +320,32 @@ class ProxyManager:
                 tempfile.mkdtemp("", "ssh-", tmp_dir), "master-" + tokens
             )
         return self._sock_path
+
+
+@functools.lru_cache(maxsize=1)
+def _get_git_protocol_version() -> str:
+    """Return the git protocol version.
+
+    The version is found by first reading the global git config.
+    If no git config for protocol version exists, try to deduce the default
+    protocol version based on the git version.
+
+    See https://git-scm.com/docs/gitprotocol-v2 for details.
+    """
+    try:
+        return subprocess.check_output(
+            ["git", "config", "--get", "--global", "protocol.version"],
+            encoding="utf-8",
+            stderr=subprocess.PIPE,
+        ).strip()
+    except subprocess.CalledProcessError as e:
+        if e.returncode == 1:
+            # Exit code 1 means that the git config key was not found.
+            # Try to imitate the defaults that git would have used.
+            git_version = git.version_tuple()
+            if git_version >= (2, 26, 0):
+                # Since git version 2.26, protocol v2 is the default.
+                return "2"
+            return "1"
+        # Other exit codes indicate error with reading the config.
+        raise

subcmds/__init__.py

@@ -37,9 +37,7 @@ for py in os.listdir(my_dir):
         try:
             cmd = getattr(mod, clsn)
         except AttributeError:
-            raise SyntaxError(
-                "%s/%s does not define class %s" % (__name__, py, clsn)
-            )
+            raise SyntaxError(f"{__name__}/{py} does not define class {clsn}")
 
         name = name.replace("_", "-")
         cmd.NAME = name

subcmds/abandon.py

@@ -70,8 +70,10 @@ It is equivalent to "git branch -D <branchname>".
         else:
             args.insert(0, "'All local branches'")
 
-    def _ExecuteOne(self, all_branches, nb, project):
+    @classmethod
+    def _ExecuteOne(cls, all_branches, nb, project_idx):
         """Abandon one project."""
+        project = cls.get_parallel_context()["projects"][project_idx]
         if all_branches:
             branches = project.GetBranches()
         else:
@@ -89,7 +91,7 @@ It is equivalent to "git branch -D <branchname>".
             if status is not None:
                 ret[name] = status
 
-        return (ret, project, errors)
+        return (ret, project_idx, errors)
 
     def Execute(self, opt, args):
         nb = args[0].split()
@@ -102,7 +104,8 @@ It is equivalent to "git branch -D <branchname>".
         _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
 
         def _ProcessResults(_pool, pm, states):
-            for results, project, errors in states:
+            for results, project_idx, errors in states:
+                project = all_projects[project_idx]
                 for branch, status in results.items():
                     if status:
                         success[branch].append(project)
@@ -111,15 +114,18 @@ It is equivalent to "git branch -D <branchname>".
                 aggregate_errors.extend(errors)
                 pm.update(msg="")
 
-        self.ExecuteInParallel(
-            opt.jobs,
-            functools.partial(self._ExecuteOne, opt.all, nb),
-            all_projects,
-            callback=_ProcessResults,
-            output=Progress(
-                "Abandon %s" % (nb,), len(all_projects), quiet=opt.quiet
-            ),
-        )
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = all_projects
+            self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(self._ExecuteOne, opt.all, nb),
+                range(len(all_projects)),
+                callback=_ProcessResults,
+                output=Progress(
+                    f"Abandon {nb}", len(all_projects), quiet=opt.quiet
+                ),
+                chunksize=1,
+            )
 
         width = max(
             itertools.chain(
@@ -152,4 +158,4 @@ It is equivalent to "git branch -D <branchname>".
                             _RelPath(p) for p in success[br]
                         )
                     )
-                print("%s%s| %s\n" % (br, " " * (width - len(br)), result))
+                print(f"{br}{' ' * (width - len(br))}| {result}\n")

subcmds/branches.py

@@ -98,6 +98,22 @@ is shown, then the branch appears in all projects.
 """
     PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
+    @classmethod
+    def _ExpandProjectToBranches(cls, project_idx):
+        """Expands a project into a list of branch names & associated info.
+
+        Args:
+            project_idx: project.Project index
+
+        Returns:
+            List[Tuple[str, git_config.Branch, int]]
+        """
+        branches = []
+        project = cls.get_parallel_context()["projects"][project_idx]
+        for name, b in project.GetBranches().items():
+            branches.append((name, b, project_idx))
+        return branches
+
     def Execute(self, opt, args):
         projects = self.GetProjects(
             args, all_manifests=not opt.this_manifest_only
@@ -107,17 +123,20 @@ is shown, then the branch appears in all projects.
         project_cnt = len(projects)
 
         def _ProcessResults(_pool, _output, results):
-            for name, b in itertools.chain.from_iterable(results):
+            for name, b, project_idx in itertools.chain.from_iterable(results):
+                b.project = projects[project_idx]
                 if name not in all_branches:
                     all_branches[name] = BranchInfo(name)
                 all_branches[name].add(b)
 
-        self.ExecuteInParallel(
-            opt.jobs,
-            expand_project_to_branches,
-            projects,
-            callback=_ProcessResults,
-        )
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = projects
+            self.ExecuteInParallel(
+                opt.jobs,
+                self._ExpandProjectToBranches,
+                range(len(projects)),
+                callback=_ProcessResults,
+            )
 
         names = sorted(all_branches)
 
@@ -148,7 +167,10 @@ is shown, then the branch appears in all projects.
             else:
                 published = " "
 
-            hdr("%c%c %-*s" % (current, published, width, name))
+            # A branch name can contain a percent sign, so we need to escape it.
+            # Escape after f-string formatting to properly account for leading
+            # spaces.
+            hdr(f"{current}{published} {name:{width}}".replace("%", "%%"))
             out.write(" |")
 
             _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
@@ -174,7 +196,7 @@ is shown, then the branch appears in all projects.
                         if _RelPath(p) not in have:
                             paths.append(_RelPath(p))
 
-                s = " %s %s" % (in_type, ", ".join(paths))
+                s = f" {in_type} {', '.join(paths)}"
                 if not i.IsSplitCurrent and (width + 7 + len(s) < 80):
                     fmt = out.current if i.IsCurrent else fmt
                     fmt(s)
@@ -191,19 +213,3 @@ is shown, then the branch appears in all projects.
             else:
                 out.write(" in all projects")
             out.nl()
-
-
-def expand_project_to_branches(project):
-    """Expands a project into a list of branch names & associated information.
-
-    Args:
-        project: project.Project
-
-    Returns:
-        List[Tuple[str, git_config.Branch]]
-    """
-    branches = []
-    for name, b in project.GetBranches().items():
-        b.project = project
-        branches.append((name, b))
-    return branches

subcmds/checkout.py

@@ -20,7 +20,6 @@ from command import DEFAULT_LOCAL_JOBS
 from error import GitError
 from error import RepoExitError
 from progress import Progress
-from project import Project
 from repo_logging import RepoLogger
 
 
@@ -30,7 +29,7 @@ logger = RepoLogger(__file__)
 class CheckoutBranchResult(NamedTuple):
     # Whether the Project is on the branch (i.e. branch exists and no errors)
     result: bool
-    project: Project
+    project_idx: int
     error: Exception
 
 
@@ -62,15 +61,17 @@ The command is equivalent to:
         if not args:
             self.Usage()
 
-    def _ExecuteOne(self, nb, project):
+    @classmethod
+    def _ExecuteOne(cls, nb, project_idx):
         """Checkout one project."""
         error = None
         result = None
+        project = cls.get_parallel_context()["projects"][project_idx]
         try:
             result = project.CheckoutBranch(nb)
         except GitError as e:
             error = e
-        return CheckoutBranchResult(result, project, error)
+        return CheckoutBranchResult(result, project_idx, error)
 
     def Execute(self, opt, args):
         nb = args[0]
@@ -83,22 +84,25 @@ The command is equivalent to:
 
         def _ProcessResults(_pool, pm, results):
             for result in results:
+                project = all_projects[result.project_idx]
                 if result.error is not None:
                     err.append(result.error)
-                    err_projects.append(result.project)
+                    err_projects.append(project)
                 elif result.result:
-                    success.append(result.project)
+                    success.append(project)
                 pm.update(msg="")
 
-        self.ExecuteInParallel(
-            opt.jobs,
-            functools.partial(self._ExecuteOne, nb),
-            all_projects,
-            callback=_ProcessResults,
-            output=Progress(
-                "Checkout %s" % (nb,), len(all_projects), quiet=opt.quiet
-            ),
-        )
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = all_projects
+            self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(self._ExecuteOne, nb),
+                range(len(all_projects)),
+                callback=_ProcessResults,
+                output=Progress(
+                    f"Checkout {nb}", len(all_projects), quiet=opt.quiet
+                ),
+            )
 
         if err_projects:
             for p in err_projects:

subcmds/diff.py

@@ -40,7 +40,8 @@ to the Unix 'patch' command.
             help="paths are relative to the repository root",
         )
 
-    def _ExecuteOne(self, absolute, local, project):
+    @classmethod
+    def _ExecuteOne(cls, absolute, local, project_idx):
         """Obtains the diff for a specific project.
 
         Args:
@@ -48,12 +49,13 @@ to the Unix 'patch' command.
             local: a boolean, if True, the path is relative to the local
                 (sub)manifest.  If false, the path is relative to the outermost
                 manifest.
-            project: Project to get status of.
+            project_idx: Project index to get status of.
 
         Returns:
             The status of the project.
         """
         buf = io.StringIO()
+        project = cls.get_parallel_context()["projects"][project_idx]
         ret = project.PrintWorkTreeDiff(absolute, output_redir=buf, local=local)
         return (ret, buf.getvalue())
 
@@ -71,12 +73,15 @@ to the Unix 'patch' command.
                     ret = 1
             return ret
 
-        return self.ExecuteInParallel(
-            opt.jobs,
-            functools.partial(
-                self._ExecuteOne, opt.absolute, opt.this_manifest_only
-            ),
-            all_projects,
-            callback=_ProcessResults,
-            ordered=True,
-        )
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = all_projects
+            return self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(
+                    self._ExecuteOne, opt.absolute, opt.this_manifest_only
+                ),
+                range(len(all_projects)),
+                callback=_ProcessResults,
+                ordered=True,
+                chunksize=1,
+            )

subcmds/diffmanifests.py

@@ -87,25 +87,17 @@ synced and their revisions won't be found.
     def _printRawDiff(self, diff, pretty_format=None, local=False):
         _RelPath = lambda p: p.RelPath(local=local)
         for project in diff["added"]:
-            self.printText(
-                "A %s %s" % (_RelPath(project), project.revisionExpr)
-            )
+            self.printText(f"A {_RelPath(project)} {project.revisionExpr}")
             self.out.nl()
 
         for project in diff["removed"]:
-            self.printText(
-                "R %s %s" % (_RelPath(project), project.revisionExpr)
-            )
+            self.printText(f"R {_RelPath(project)} {project.revisionExpr}")
             self.out.nl()
 
         for project, otherProject in diff["changed"]:
             self.printText(
-                "C %s %s %s"
-                % (
-                    _RelPath(project),
-                    project.revisionExpr,
-                    otherProject.revisionExpr,
-                )
+                f"C {_RelPath(project)} {project.revisionExpr} "
+                f"{otherProject.revisionExpr}"
             )
             self.out.nl()
             self._printLogs(
@@ -118,12 +110,8 @@ synced and their revisions won't be found.
 
         for project, otherProject in diff["unreachable"]:
             self.printText(
-                "U %s %s %s"
-                % (
-                    _RelPath(project),
-                    project.revisionExpr,
-                    otherProject.revisionExpr,
-                )
+                f"U {_RelPath(project)} {project.revisionExpr} "
+                f"{otherProject.revisionExpr}"
             )
             self.out.nl()
 

subcmds/forall.py

@@ -15,7 +15,6 @@
 import errno
 import functools
 import io
-import multiprocessing
 import os
 import re
 import signal
@@ -26,7 +25,6 @@ from color import Coloring
 from command import Command
 from command import DEFAULT_LOCAL_JOBS
 from command import MirrorSafeCommand
-from command import WORKER_BATCH_SIZE
 from error import ManifestInvalidRevisionError
 from repo_logging import RepoLogger
 
@@ -241,7 +239,6 @@ without iterating through the remaining projects.
                     cmd.insert(cmd.index(cn) + 1, "--color")
 
         mirror = self.manifest.IsMirror
-        rc = 0
 
         smart_sync_manifest_name = "smart_sync_override.xml"
         smart_sync_manifest_path = os.path.join(
@@ -264,35 +261,44 @@ without iterating through the remaining projects.
 
         os.environ["REPO_COUNT"] = str(len(projects))
 
+        def _ProcessResults(_pool, _output, results):
+            rc = 0
+            first = True
+            for r, output in results:
+                if output:
+                    if first:
+                        first = False
+                    elif opt.project_header:
+                        print()
+                    # To simplify the DoWorkWrapper, take care of automatic
+                    # newlines.
+                    end = "\n"
+                    if output[-1] == "\n":
+                        end = ""
+                    print(output, end=end)
+                rc = rc or r
+                if r != 0 and opt.abort_on_errors:
+                    raise Exception("Aborting due to previous error")
+            return rc
+
         try:
             config = self.manifest.manifestProject.config
-            with multiprocessing.Pool(opt.jobs, InitWorker) as pool:
-                results_it = pool.imap(
+            with self.ParallelContext():
+                self.get_parallel_context()["projects"] = projects
+                rc = self.ExecuteInParallel(
+                    opt.jobs,
                     functools.partial(
-                        DoWorkWrapper, mirror, opt, cmd, shell, config
+                        self.DoWorkWrapper, mirror, opt, cmd, shell, config
                     ),
-                    enumerate(projects),
-                    chunksize=WORKER_BATCH_SIZE,
+                    range(len(projects)),
+                    callback=_ProcessResults,
+                    ordered=True,
+                    initializer=self.InitWorker,
+                    chunksize=1,
                 )
-                first = True
-                for r, output in results_it:
-                    if output:
-                        if first:
-                            first = False
-                        elif opt.project_header:
-                            print()
-                        # To simplify the DoWorkWrapper, take care of automatic
-                        # newlines.
-                        end = "\n"
-                        if output[-1] == "\n":
-                            end = ""
-                        print(output, end=end)
-                    rc = rc or r
-                    if r != 0 and opt.abort_on_errors:
-                        raise Exception("Aborting due to previous error")
         except (KeyboardInterrupt, WorkerKeyboardInterrupt):
             # Catch KeyboardInterrupt raised inside and outside of workers
-            rc = rc or errno.EINTR
+            rc = errno.EINTR
         except Exception as e:
             # Catch any other exceptions raised
             logger.error(
@@ -300,35 +306,35 @@ without iterating through the remaining projects.
                 type(e).__name__,
                 e,
             )
-            rc = rc or getattr(e, "errno", 1)
+            rc = getattr(e, "errno", 1)
         if rc != 0:
             sys.exit(rc)
 
+    @classmethod
+    def InitWorker(cls):
+        signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+    @classmethod
+    def DoWorkWrapper(cls, mirror, opt, cmd, shell, config, project_idx):
+        """A wrapper around the DoWork() method.
+
+        Catch the KeyboardInterrupt exceptions here and re-raise them as a
+        different, ``Exception``-based exception to stop it flooding the console
+        with stacktraces and making the parent hang indefinitely.
+
+        """
+        project = cls.get_parallel_context()["projects"][project_idx]
+        try:
+            return DoWork(project, mirror, opt, cmd, shell, project_idx, config)
+        except KeyboardInterrupt:
+            print("%s: Worker interrupted" % project.name)
+            raise WorkerKeyboardInterrupt()
+
 
 class WorkerKeyboardInterrupt(Exception):
     """Keyboard interrupt exception for worker processes."""
 
 
-def InitWorker():
-    signal.signal(signal.SIGINT, signal.SIG_IGN)
-
-
-def DoWorkWrapper(mirror, opt, cmd, shell, config, args):
-    """A wrapper around the DoWork() method.
-
-    Catch the KeyboardInterrupt exceptions here and re-raise them as a
-    different, ``Exception``-based exception to stop it flooding the console
-    with stacktraces and making the parent hang indefinitely.
-
-    """
-    cnt, project = args
-    try:
-        return DoWork(project, mirror, opt, cmd, shell, cnt, config)
-    except KeyboardInterrupt:
-        print("%s: Worker interrupted" % project.name)
-        raise WorkerKeyboardInterrupt()
-
-
 def DoWork(project, mirror, opt, cmd, shell, cnt, config):
     env = os.environ.copy()
 

subcmds/gc.py

@@ -0,0 +1,127 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from typing import Set
+
+from command import Command
+import platform_utils
+from progress import Progress
+
+
+class Gc(Command):
+    COMMON = True
+    helpSummary = "Cleaning up internal repo state."
+    helpUsage = """
+%prog
+"""
+
+    def _Options(self, p):
+        p.add_option(
+            "-n",
+            "--dry-run",
+            dest="dryrun",
+            default=False,
+            action="store_true",
+            help="do everything except actually delete",
+        )
+        p.add_option(
+            "-y",
+            "--yes",
+            default=False,
+            action="store_true",
+            help="answer yes to all safe prompts",
+        )
+
+    def _find_git_to_delete(
+        self, to_keep: Set[str], start_dir: str
+    ) -> Set[str]:
+        """Searches no longer needed ".git" directories.
+
+        Scans the file system starting from `start_dir` and removes all
+        directories that end with ".git" that are not in the `to_keep` set.
+        """
+        to_delete = set()
+        for root, dirs, _ in platform_utils.walk(start_dir):
+            for directory in dirs:
+                if not directory.endswith(".git"):
+                    continue
+
+                path = os.path.join(root, directory)
+                if path not in to_keep:
+                    to_delete.add(path)
+
+        return to_delete
+
+    def Execute(self, opt, args):
+        projects = self.GetProjects(
+            args, all_manifests=not opt.this_manifest_only
+        )
+        print(f"Scanning filesystem under {self.repodir}...")
+
+        project_paths = set()
+        project_object_paths = set()
+
+        for project in projects:
+            project_paths.add(project.gitdir)
+            project_object_paths.add(project.objdir)
+
+        to_delete = self._find_git_to_delete(
+            project_paths, os.path.join(self.repodir, "projects")
+        )
+
+        to_delete.update(
+            self._find_git_to_delete(
+                project_object_paths,
+                os.path.join(self.repodir, "project-objects"),
+            )
+        )
+
+        if not to_delete:
+            print("Nothing to clean up.")
+            return
+
+        print("Identified the following projects are no longer used:")
+        print("\n".join(to_delete))
+        print("\n")
+        if not opt.yes:
+            print(
+                "If you proceed, any local commits in those projects will be "
+                "destroyed!"
+            )
+            ask = input("Proceed? [y/N] ")
+            if ask.lower() != "y":
+                return 1
+
+        pm = Progress(
+            "Deleting",
+            len(to_delete),
+            delay=False,
+            quiet=opt.quiet,
+            show_elapsed=True,
+            elide=True,
+        )
+
+        for path in to_delete:
+            if opt.dryrun:
+                print(f"\nWould have deleted ${path}")
+            else:
+                tmp_path = os.path.join(
+                    os.path.dirname(path),
+                    f"to_be_deleted_{os.path.basename(path)}",
+                )
+                platform_utils.rename(path, tmp_path)
+                platform_utils.rmtree(tmp_path)
+            pm.update(msg=path)
+        pm.end()

subcmds/grep.py

@@ -23,7 +23,6 @@ from error import GitError
 from error import InvalidArgumentsError
 from error import SilentRepoExitError
 from git_command import GitCommand
-from project import Project
 from repo_logging import RepoLogger
 
 
@@ -40,7 +39,7 @@ class GrepColoring(Coloring):
 class ExecuteOneResult(NamedTuple):
     """Result from an execute instance."""
 
-    project: Project
+    project_idx: int
     rc: int
     stdout: str
     stderr: str
@@ -262,8 +261,10 @@ contain a line that matches both expressions:
             help="Show only file names not containing matching lines",
         )
 
-    def _ExecuteOne(self, cmd_argv, project):
+    @classmethod
+    def _ExecuteOne(cls, cmd_argv, project_idx):
         """Process one project."""
+        project = cls.get_parallel_context()["projects"][project_idx]
         try:
             p = GitCommand(
                 project,
@@ -274,7 +275,7 @@ contain a line that matches both expressions:
                 verify_command=True,
             )
         except GitError as e:
-            return ExecuteOneResult(project, -1, None, str(e), e)
+            return ExecuteOneResult(project_idx, -1, None, str(e), e)
 
         try:
             error = None
@@ -282,10 +283,12 @@ contain a line that matches both expressions:
         except GitError as e:
             rc = 1
             error = e
-        return ExecuteOneResult(project, rc, p.stdout, p.stderr, error)
+        return ExecuteOneResult(project_idx, rc, p.stdout, p.stderr, error)
 
     @staticmethod
-    def _ProcessResults(full_name, have_rev, opt, _pool, out, results):
+    def _ProcessResults(
+        full_name, have_rev, opt, projects, _pool, out, results
+    ):
         git_failed = False
         bad_rev = False
         have_match = False
@@ -293,9 +296,10 @@ contain a line that matches both expressions:
         errors = []
 
         for result in results:
+            project = projects[result.project_idx]
             if result.rc < 0:
                 git_failed = True
-                out.project("--- project %s ---" % _RelPath(result.project))
+                out.project("--- project %s ---" % _RelPath(project))
                 out.nl()
                 out.fail("%s", result.stderr)
                 out.nl()
@@ -311,9 +315,7 @@ contain a line that matches both expressions:
                     ):
                         bad_rev = True
                     else:
-                        out.project(
-                            "--- project %s ---" % _RelPath(result.project)
-                        )
+                        out.project("--- project %s ---" % _RelPath(project))
                         out.nl()
                         out.fail("%s", result.stderr.strip())
                         out.nl()
@@ -331,13 +333,13 @@ contain a line that matches both expressions:
                     rev, line = line.split(":", 1)
                     out.write("%s", rev)
                     out.write(":")
-                    out.project(_RelPath(result.project))
+                    out.project(_RelPath(project))
                     out.write("/")
                     out.write("%s", line)
                     out.nl()
             elif full_name:
                 for line in r:
-                    out.project(_RelPath(result.project))
+                    out.project(_RelPath(project))
                     out.write("/")
                     out.write("%s", line)
                     out.nl()
@@ -381,16 +383,19 @@ contain a line that matches both expressions:
             cmd_argv.extend(opt.revision)
         cmd_argv.append("--")
 
-        git_failed, bad_rev, have_match, errors = self.ExecuteInParallel(
-            opt.jobs,
-            functools.partial(self._ExecuteOne, cmd_argv),
-            projects,
-            callback=functools.partial(
-                self._ProcessResults, full_name, have_rev, opt
-            ),
-            output=out,
-            ordered=True,
-        )
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = projects
+            git_failed, bad_rev, have_match, errors = self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(self._ExecuteOne, cmd_argv),
+                range(len(projects)),
+                callback=functools.partial(
+                    self._ProcessResults, full_name, have_rev, opt, projects
+                ),
+                output=out,
+                ordered=True,
+                chunksize=1,
+            )
 
         if git_failed:
             raise GrepCommandError(

subcmds/help.py

@@ -150,7 +150,7 @@ Displays detailed usage information about a command.
     def _PrintAllCommandHelp(self):
         for name in sorted(all_commands):
             cmd = all_commands[name](manifest=self.manifest)
-            self._PrintCommandHelp(cmd, header_prefix="[%s] " % (name,))
+            self._PrintCommandHelp(cmd, header_prefix=f"[{name}] ")
 
     def _Options(self, p):
         p.add_option(

subcmds/info.py

@@ -97,7 +97,9 @@ class Info(PagedCommand):
             self.headtext(self.manifest.default.revisionExpr)
         self.out.nl()
         self.heading("Manifest merge branch: ")
-        self.headtext(mergeBranch)
+        # The manifest might not have a merge branch if it isn't in a git repo,
+        # e.g. if `repo init --standalone-manifest` is used.
+        self.headtext(mergeBranch or "")
         self.out.nl()
         self.heading("Manifest groups: ")
         self.headtext(manifestGroups)
@@ -248,7 +250,7 @@ class Info(PagedCommand):
 
             for commit in commits:
                 split = commit.split()
-                self.text("{0:38}{1} ".format("", "-"))
+                self.text(f"{'':38}{'-'} ")
                 self.sha(split[0] + " ")
                 self.text(" ".join(split[1:]))
                 self.out.nl()

subcmds/init.py

@@ -21,10 +21,9 @@ from command import MirrorSafeCommand
 from error import RepoUnhandledExceptionError
 from error import UpdateManifestError
 from git_command import git_require
-from git_command import MIN_GIT_VERSION_HARD
-from git_command import MIN_GIT_VERSION_SOFT
 from repo_logging import RepoLogger
 from wrapper import Wrapper
+from wrapper import WrapperDir
 
 
 logger = RepoLogger(__file__)
@@ -53,6 +52,10 @@ The optional -b argument can be used to select the manifest branch
 to checkout and use.  If no branch is specified, the remote's default
 branch is used.  This is equivalent to using -b HEAD.
 
+The optional --manifest-upstream-branch argument can be used when a commit is
+provided to --manifest-branch (or -b), to specify the name of the git ref in
+which the commit can be found.
+
 The optional -m argument can be used to specify an alternate manifest
 to be used. If no manifest is specified, the manifest default.xml
 will be used.
@@ -136,6 +139,7 @@ to update the working directory files.
         # manifest project is special and is created when instantiating the
         # manifest which happens before we parse options.
         self.manifest.manifestProject.clone_depth = opt.manifest_depth
+        self.manifest.manifestProject.upstream = opt.manifest_upstream_branch
         clone_filter_for_depth = (
             "blob:none" if (_REPO_ALLOW_SHALLOW == "0") else None
         )
@@ -215,7 +219,7 @@ to update the working directory files.
 
             if not opt.quiet:
                 print()
-            print("Your identity is: %s <%s>" % (name, email))
+            print(f"Your identity is: {name} <{email}>")
             print("is this correct [y/N]? ", end="", flush=True)
             a = sys.stdin.readline().strip().lower()
             if a in ("yes", "y", "t", "true"):
@@ -318,6 +322,12 @@ to update the working directory files.
                 " be used with --standalone-manifest."
             )
 
+        if opt.manifest_upstream_branch and opt.manifest_branch is None:
+            self.OptionParser.error(
+                "--manifest-upstream-branch cannot be used without "
+                "--manifest-branch."
+            )
+
         if args:
             if opt.manifest_url:
                 self.OptionParser.error(
@@ -331,13 +341,17 @@ to update the working directory files.
                 self.OptionParser.error("too many arguments to init")
 
     def Execute(self, opt, args):
-        git_require(MIN_GIT_VERSION_HARD, fail=True)
-        if not git_require(MIN_GIT_VERSION_SOFT):
+        wrapper = Wrapper()
+
+        reqs = wrapper.Requirements.from_dir(WrapperDir())
+        git_require(reqs.get_hard_ver("git"), fail=True)
+        min_git_version_soft = reqs.get_soft_ver("git")
+        if not git_require(min_git_version_soft):
             logger.warning(
                 "repo: warning: git-%s+ will soon be required; "
                 "please upgrade your version of git to maintain "
                 "support.",
-                ".".join(str(x) for x in MIN_GIT_VERSION_SOFT),
+                ".".join(str(x) for x in min_git_version_soft),
             )
 
         rp = self.manifest.repoProject
@@ -350,10 +364,9 @@ to update the working directory files.
 
         # Handle new --repo-rev requests.
         if opt.repo_rev:
-            wrapper = Wrapper()
             try:
                 remote_ref, rev = wrapper.check_repo_rev(
-                    rp.gitdir,
+                    rp.worktree,
                     opt.repo_rev,
                     repo_verify=opt.repo_verify,
                     quiet=opt.quiet,

subcmds/list.py

@@ -131,7 +131,7 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
             elif opt.path_only and not opt.name_only:
                 lines.append("%s" % (_getpath(project)))
             else:
-                lines.append("%s : %s" % (_getpath(project), project.name))
+                lines.append(f"{_getpath(project)} : {project.name}")
 
         if lines:
             lines.sort()

subcmds/prune.py

@@ -27,8 +27,10 @@ class Prune(PagedCommand):
 """
     PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
-    def _ExecuteOne(self, project):
+    @classmethod
+    def _ExecuteOne(cls, project_idx):
         """Process one project."""
+        project = cls.get_parallel_context()["projects"][project_idx]
         return project.PruneHeads()
 
     def Execute(self, opt, args):
@@ -41,13 +43,15 @@ class Prune(PagedCommand):
         def _ProcessResults(_pool, _output, results):
             return list(itertools.chain.from_iterable(results))
 
-        all_branches = self.ExecuteInParallel(
-            opt.jobs,
-            self._ExecuteOne,
-            projects,
-            callback=_ProcessResults,
-            ordered=True,
-        )
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = projects
+            all_branches = self.ExecuteInParallel(
+                opt.jobs,
+                self._ExecuteOne,
+                range(len(projects)),
+                callback=_ProcessResults,
+                ordered=True,
+            )
 
         if not all_branches:
             return
@@ -83,9 +87,7 @@ class Prune(PagedCommand):
             )
 
             if not branch.base_exists:
-                print(
-                    "(ignoring: tracking branch is gone: %s)" % (branch.base,)
-                )
+                print(f"(ignoring: tracking branch is gone: {branch.base})")
             else:
                 commits = branch.commits
                 date = branch.date

subcmds/start.py

@@ -21,7 +21,6 @@ from error import RepoExitError
 from git_command import git
 from git_config import IsImmutable
 from progress import Progress
-from project import Project
 from repo_logging import RepoLogger
 
 
@@ -29,7 +28,7 @@ logger = RepoLogger(__file__)
 
 
 class ExecuteOneResult(NamedTuple):
-    project: Project
+    project_idx: int
     error: Exception
 
 
@@ -80,18 +79,20 @@ revision specified in the manifest.
         if not git.check_ref_format("heads/%s" % nb):
             self.OptionParser.error("'%s' is not a valid name" % nb)
 
-    def _ExecuteOne(self, revision, nb, project):
+    @classmethod
+    def _ExecuteOne(cls, revision, nb, default_revisionExpr, project_idx):
         """Start one project."""
         # If the current revision is immutable, such as a SHA1, a tag or
         # a change, then we can't push back to it. Substitute with
         # dest_branch, if defined; or with manifest default revision instead.
         branch_merge = ""
         error = None
+        project = cls.get_parallel_context()["projects"][project_idx]
         if IsImmutable(project.revisionExpr):
             if project.dest_branch:
                 branch_merge = project.dest_branch
             else:
-                branch_merge = self.manifest.default.revisionExpr
+                branch_merge = default_revisionExpr
 
         try:
             project.StartBranch(
@@ -100,7 +101,7 @@ revision specified in the manifest.
         except Exception as e:
             logger.error("error: unable to checkout %s: %s", project.name, e)
             error = e
-        return ExecuteOneResult(project, error)
+        return ExecuteOneResult(project_idx, error)
 
     def Execute(self, opt, args):
         nb = args[0]
@@ -120,19 +121,28 @@ revision specified in the manifest.
         def _ProcessResults(_pool, pm, results):
             for result in results:
                 if result.error:
-                    err_projects.append(result.project)
+                    project = all_projects[result.project_idx]
+                    err_projects.append(project)
                     err.append(result.error)
                 pm.update(msg="")
 
-        self.ExecuteInParallel(
-            opt.jobs,
-            functools.partial(self._ExecuteOne, opt.revision, nb),
-            all_projects,
-            callback=_ProcessResults,
-            output=Progress(
-                "Starting %s" % (nb,), len(all_projects), quiet=opt.quiet
-            ),
-        )
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = all_projects
+            self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(
+                    self._ExecuteOne,
+                    opt.revision,
+                    nb,
+                    self.manifest.default.revisionExpr,
+                ),
+                range(len(all_projects)),
+                callback=_ProcessResults,
+                output=Progress(
+                    f"Starting {nb}", len(all_projects), quiet=opt.quiet
+                ),
+                chunksize=1,
+            )
 
         if err_projects:
             for p in err_projects:

subcmds/status.py

@@ -88,7 +88,8 @@ the following meanings:
             "projects",
         )
 
-    def _StatusHelper(self, quiet, local, project):
+    @classmethod
+    def _StatusHelper(cls, quiet, local, project_idx):
         """Obtains the status for a specific project.
 
         Obtains the status for a project, redirecting the output to
@@ -99,12 +100,13 @@ the following meanings:
             local: a boolean, if True, the path is relative to the local
                 (sub)manifest.  If false, the path is relative to the outermost
                 manifest.
-            project: Project to get status of.
+            project_idx: Project index to get status of.
 
         Returns:
             The status of the project.
         """
         buf = io.StringIO()
+        project = cls.get_parallel_context()["projects"][project_idx]
         ret = project.PrintWorkTreeStatus(
             quiet=quiet, output_redir=buf, local=local
         )
@@ -143,15 +145,18 @@ the following meanings:
                     ret += 1
             return ret
 
-        counter = self.ExecuteInParallel(
-            opt.jobs,
-            functools.partial(
-                self._StatusHelper, opt.quiet, opt.this_manifest_only
-            ),
-            all_projects,
-            callback=_ProcessResults,
-            ordered=True,
-        )
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = all_projects
+            counter = self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(
+                    self._StatusHelper, opt.quiet, opt.this_manifest_only
+                ),
+                range(len(all_projects)),
+                callback=_ProcessResults,
+                ordered=True,
+                chunksize=1,
+            )
 
         if not opt.quiet and len(all_projects) == counter:
             print("nothing to commit (working directory clean)")

subcmds/sync.py

@@ -21,6 +21,7 @@ import multiprocessing
 import netrc
 import optparse
 import os
+from pathlib import Path
 import sys
 import tempfile
 import time
@@ -82,22 +83,65 @@ from wrapper import Wrapper
 
 _ONE_DAY_S = 24 * 60 * 60
 
-# Env var to implicitly turn auto-gc back on.  This was added to allow a user to
-# revert a change in default behavior in v2.29.9.  Remove after 2023-04-01.
-_REPO_AUTO_GC = "REPO_AUTO_GC"
-_AUTO_GC = os.environ.get(_REPO_AUTO_GC) == "1"
-
 _REPO_ALLOW_SHALLOW = os.environ.get("REPO_ALLOW_SHALLOW")
 
 logger = RepoLogger(__file__)
 
 
+def _SafeCheckoutOrder(checkouts: List[Project]) -> List[List[Project]]:
+    """Generate a sequence of checkouts that is safe to perform. The client
+    should checkout everything from n-th index before moving to n+1.
+
+    This is only useful if manifest contains nested projects.
+
+    E.g. if foo, foo/bar and foo/bar/baz are project paths, then foo needs to
+    finish before foo/bar can proceed, and foo/bar needs to finish before
+    foo/bar/baz."""
+    res = [[]]
+    current = res[0]
+
+    # depth_stack contains a current stack of parent paths.
+    depth_stack = []
+    # Checkouts are iterated in the hierarchical order. That way, it can easily
+    # be determined if the previous checkout is parent of the current checkout.
+    # We are splitting by the path separator so the final result is
+    # hierarchical, and not just lexicographical. For example, if the projects
+    # are: foo, foo/bar, foo-bar, lexicographical order produces foo, foo-bar
+    # and foo/bar, which doesn't work.
+    for checkout in sorted(checkouts, key=lambda x: x.relpath.split("/")):
+        checkout_path = Path(checkout.relpath)
+        while depth_stack:
+            try:
+                checkout_path.relative_to(depth_stack[-1])
+            except ValueError:
+                # Path.relative_to returns ValueError if paths are not relative.
+                # TODO(sokcevic): Switch to is_relative_to once min supported
+                # version is py3.9.
+                depth_stack.pop()
+            else:
+                if len(depth_stack) >= len(res):
+                    # Another depth created.
+                    res.append([])
+                break
+
+        current = res[len(depth_stack)]
+        current.append(checkout)
+        depth_stack.append(checkout_path)
+
+    return res
+
+
+def _chunksize(projects: int, jobs: int) -> int:
+    """Calculate chunk size for the given number of projects and jobs."""
+    return min(max(1, projects // jobs), WORKER_BATCH_SIZE)
+
+
 class _FetchOneResult(NamedTuple):
     """_FetchOne return value.
 
     Attributes:
       success (bool): True if successful.
-      project (Project): The fetched project.
+      project_idx (int): The fetched project index.
       start (float): The starting time.time().
       finish (float): The ending time.time().
       remote_fetched (bool): True if the remote was actually queried.
@@ -105,7 +149,7 @@ class _FetchOneResult(NamedTuple):
 
     success: bool
     errors: List[Exception]
-    project: Project
+    project_idx: int
     start: float
     finish: float
     remote_fetched: bool
@@ -138,14 +182,14 @@ class _CheckoutOneResult(NamedTuple):
 
     Attributes:
       success (bool): True if successful.
-      project (Project): The project.
+      project_idx (int): The project index.
       start (float): The starting time.time().
       finish (float): The ending time.time().
     """
 
     success: bool
     errors: List[Exception]
-    project: Project
+    project_idx: int
     start: float
     finish: float
 
@@ -243,6 +287,11 @@ directories if they have previously been linked to a different
 object directory. WARNING: This may cause data to be lost since
 refs may be removed when overwriting.
 
+The --force-checkout option can be used to force git to switch revs even if the
+index or the working tree differs from HEAD, and if there are untracked files.
+WARNING: This may cause data to be lost since uncommitted changes may be
+removed.
+
 The --force-remove-dirty option can be used to remove previously used
 projects with uncommitted changes. WARNING: This may cause data to be
 lost since uncommitted changes may be removed with projects that no longer
@@ -340,6 +389,14 @@ later is required to fix a server side protocol bug.
             "point to a different object directory. WARNING: this "
             "may cause loss of data",
         )
+        p.add_option(
+            "--force-checkout",
+            dest="force_checkout",
+            action="store_true",
+            help="force checkout even if it results in throwing away "
+            "uncommitted modifications. "
+            "WARNING: this may cause loss of data",
+        )
         p.add_option(
             "--force-remove-dirty",
             dest="force_remove_dirty",
@@ -348,6 +405,13 @@ later is required to fix a server side protocol bug.
             "projects no longer exist in the manifest. "
             "WARNING: this may cause loss of data",
         )
+        p.add_option(
+            "--rebase",
+            dest="rebase",
+            action="store_true",
+            help="rebase local commits regardless of whether they are "
+            "published",
+        )
         p.add_option(
             "-l",
             "--local-only",
@@ -528,7 +592,8 @@ later is required to fix a server side protocol bug.
             branch = branch[len(R_HEADS) :]
         return branch
 
-    def _GetCurrentBranchOnly(self, opt, manifest):
+    @classmethod
+    def _GetCurrentBranchOnly(cls, opt, manifest):
         """Returns whether current-branch or use-superproject options are
         enabled.
 
@@ -618,7 +683,7 @@ later is required to fix a server side protocol bug.
 
             if not use_super:
                 continue
-            m.superproject.SetQuiet(opt.quiet)
+            m.superproject.SetQuiet(not opt.verbose)
             print_messages = git_superproject.PrintMessages(
                 opt.use_superproject, m
             )
@@ -646,7 +711,8 @@ later is required to fix a server side protocol bug.
         if need_unload:
             m.outer_client.manifest.Unload()
 
-    def _FetchProjectList(self, opt, projects):
+    @classmethod
+    def _FetchProjectList(cls, opt, projects):
         """Main function of the fetch worker.
 
         The projects we're given share the same underlying git object store, so
@@ -658,21 +724,23 @@ later is required to fix a server side protocol bug.
             opt: Program options returned from optparse.  See _Options().
             projects: Projects to fetch.
         """
-        return [self._FetchOne(opt, x) for x in projects]
+        return [cls._FetchOne(opt, x) for x in projects]
 
-    def _FetchOne(self, opt, project):
+    @classmethod
+    def _FetchOne(cls, opt, project_idx):
         """Fetch git objects for a single project.
 
         Args:
             opt: Program options returned from optparse.  See _Options().
-            project: Project object for the project to fetch.
+            project_idx: Project index for the project to fetch.
 
         Returns:
             Whether the fetch was successful.
         """
+        project = cls.get_parallel_context()["projects"][project_idx]
         start = time.time()
         k = f"{project.name} @ {project.relpath}"
-        self._sync_dict[k] = start
+        cls.get_parallel_context()["sync_dict"][k] = start
         success = False
         remote_fetched = False
         errors = []
@@ -682,7 +750,7 @@ later is required to fix a server side protocol bug.
                 quiet=opt.quiet,
                 verbose=opt.verbose,
                 output_redir=buf,
-                current_branch_only=self._GetCurrentBranchOnly(
+                current_branch_only=cls._GetCurrentBranchOnly(
                     opt, project.manifest
                 ),
                 force_sync=opt.force_sync,
@@ -692,7 +760,7 @@ later is required to fix a server side protocol bug.
                 optimized_fetch=opt.optimized_fetch,
                 retry_fetches=opt.retry_fetches,
                 prune=opt.prune,
-                ssh_proxy=self.ssh_proxy,
+                ssh_proxy=cls.get_parallel_context()["ssh_proxy"],
                 clone_filter=project.manifest.CloneFilter,
                 partial_clone_exclude=project.manifest.PartialCloneExclude,
                 clone_filter_for_depth=project.manifest.CloneFilterForDepth,
@@ -724,24 +792,20 @@ later is required to fix a server side protocol bug.
                 type(e).__name__,
                 e,
             )
-            del self._sync_dict[k]
             errors.append(e)
             raise
+        finally:
+            del cls.get_parallel_context()["sync_dict"][k]
 
         finish = time.time()
-        del self._sync_dict[k]
         return _FetchOneResult(
-            success, errors, project, start, finish, remote_fetched
+            success, errors, project_idx, start, finish, remote_fetched
         )
 
-    @classmethod
-    def _FetchInitChild(cls, ssh_proxy):
-        cls.ssh_proxy = ssh_proxy
-
     def _GetSyncProgressMessage(self):
         earliest_time = float("inf")
         earliest_proj = None
-        items = self._sync_dict.items()
+        items = self.get_parallel_context()["sync_dict"].items()
         for project, t in items:
             if t < earliest_time:
                 earliest_time = t
@@ -749,7 +813,7 @@ later is required to fix a server side protocol bug.
 
         if not earliest_proj:
             # This function is called when sync is still running but in some
-            # cases (by chance), _sync_dict can contain no entries. Return some
+            # cases (by chance), sync_dict can contain no entries. Return some
             # text to indicate that sync is still working.
             return "..working.."
 
@@ -757,10 +821,19 @@ later is required to fix a server side protocol bug.
         jobs = jobs_str(len(items))
         return f"{jobs} | {elapsed_str(elapsed)} {earliest_proj}"
 
+    @classmethod
+    def InitWorker(cls):
+        # Force connect to the manager server now.
+        # This is good because workers are initialized one by one. Without this,
+        # multiple workers may connect to the manager when handling the first
+        # job at the same time. Then the connection may fail if too many
+        # connections are pending and execeeded the socket listening backlog,
+        # especially on MacOS.
+        len(cls.get_parallel_context()["sync_dict"])
+
     def _Fetch(self, projects, opt, err_event, ssh_proxy, errors):
         ret = True
 
-        jobs = opt.jobs_network
         fetched = set()
         remote_fetched = set()
         pm = Progress(
@@ -772,7 +845,6 @@ later is required to fix a server side protocol bug.
             elide=True,
         )
 
-        self._sync_dict = multiprocessing.Manager().dict()
         sync_event = _threading.Event()
 
         def _MonitorSyncLoop():
@@ -783,19 +855,13 @@ later is required to fix a server side protocol bug.
 
         sync_progress_thread = _threading.Thread(target=_MonitorSyncLoop)
         sync_progress_thread.daemon = True
-        sync_progress_thread.start()
 
-        objdir_project_map = dict()
-        for project in projects:
-            objdir_project_map.setdefault(project.objdir, []).append(project)
-        projects_list = list(objdir_project_map.values())
-
-        def _ProcessResults(results_sets):
+        def _ProcessResults(pool, pm, results_sets):
             ret = True
             for results in results_sets:
                 for result in results:
                     success = result.success
-                    project = result.project
+                    project = projects[result.project_idx]
                     start = result.start
                     finish = result.finish
                     self._fetch_times.Set(project, finish - start)
@@ -819,58 +885,50 @@ later is required to fix a server side protocol bug.
                         fetched.add(project.gitdir)
                     pm.update()
                 if not ret and opt.fail_fast:
+                    if pool:
+                        pool.close()
                     break
             return ret
 
-        # We pass the ssh proxy settings via the class.  This allows
-        # multiprocessing to pickle it up when spawning children.  We can't pass
-        # it as an argument to _FetchProjectList below as multiprocessing is
-        # unable to pickle those.
-        Sync.ssh_proxy = None
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = projects
+            self.get_parallel_context()[
+                "sync_dict"
+            ] = multiprocessing.Manager().dict()
 
-        # NB: Multiprocessing is heavy, so don't spin it up for one job.
-        if len(projects_list) == 1 or jobs == 1:
-            self._FetchInitChild(ssh_proxy)
-            if not _ProcessResults(
-                self._FetchProjectList(opt, x) for x in projects_list
-            ):
-                ret = False
-        else:
-            # Favor throughput over responsiveness when quiet.  It seems that
-            # imap() will yield results in batches relative to chunksize, so
-            # even as the children finish a sync, we won't see the result until
-            # one child finishes ~chunksize jobs.  When using a large --jobs
-            # with large chunksize, this can be jarring as there will be a large
-            # initial delay where repo looks like it isn't doing anything and
-            # sits at 0%, but then suddenly completes a lot of jobs all at once.
-            # Since this code is more network bound, we can accept a bit more
-            # CPU overhead with a smaller chunksize so that the user sees more
-            # immediate & continuous feedback.
-            if opt.quiet:
-                chunksize = WORKER_BATCH_SIZE
-            else:
+            objdir_project_map = dict()
+            for index, project in enumerate(projects):
+                objdir_project_map.setdefault(project.objdir, []).append(index)
+            projects_list = list(objdir_project_map.values())
+
+            jobs = max(1, min(opt.jobs_network, len(projects_list)))
+
+            # We pass the ssh proxy settings via the class.  This allows
+            # multiprocessing to pickle it up when spawning children.  We can't
+            # pass it as an argument to _FetchProjectList below as
+            # multiprocessing is unable to pickle those.
+            self.get_parallel_context()["ssh_proxy"] = ssh_proxy
+
+            sync_progress_thread.start()
+            if not opt.quiet:
                 pm.update(inc=0, msg="warming up")
-                chunksize = 4
-            with multiprocessing.Pool(
-                jobs, initializer=self._FetchInitChild, initargs=(ssh_proxy,)
-            ) as pool:
-                results = pool.imap_unordered(
+            try:
+                ret = self.ExecuteInParallel(
+                    jobs,
                     functools.partial(self._FetchProjectList, opt),
                     projects_list,
-                    chunksize=chunksize,
+                    callback=_ProcessResults,
+                    output=pm,
+                    # Use chunksize=1 to avoid the chance that some workers are
+                    # idle while other workers still have more than one job in
+                    # their chunk queue.
+                    chunksize=1,
+                    initializer=self.InitWorker,
                 )
-                if not _ProcessResults(results):
-                    ret = False
-                    pool.close()
+            finally:
+                sync_event.set()
+                sync_progress_thread.join()
 
-        # Cleanup the reference now that we're done with it, and we're going to
-        # release any resources it points to.  If we don't, later
-        # multiprocessing usage (e.g. checkouts) will try to pickle and then
-        # crash.
-        del Sync.ssh_proxy
-
-        sync_event.set()
-        pm.end()
         self._fetch_times.Save()
         self._local_sync_state.Save()
 
@@ -911,7 +969,9 @@ later is required to fix a server side protocol bug.
         if not success:
             err_event.set()
 
-        _PostRepoFetch(rp, opt.repo_verify)
+        # Call self update, unless requested not to
+        if os.environ.get("REPO_SKIP_SELF_UPDATE", "0") == "0":
+            _PostRepoFetch(rp, opt.repo_verify)
         if opt.network_only:
             # Bail out now; the rest touches the working tree.
             if err_event.is_set():
@@ -956,17 +1016,32 @@ later is required to fix a server side protocol bug.
 
         return _FetchMainResult(all_projects)
 
-    def _CheckoutOne(self, detach_head, force_sync, project):
+    @classmethod
+    def _CheckoutOne(
+        cls,
+        detach_head,
+        force_sync,
+        force_checkout,
+        force_rebase,
+        verbose,
+        project_idx,
+    ):
         """Checkout work tree for one project
 
         Args:
             detach_head: Whether to leave a detached HEAD.
-            force_sync: Force checking out of the repo.
-            project: Project object for the project to checkout.
+            force_sync: Force checking out of .git directory (e.g. overwrite
+            existing git directory that was previously linked to a different
+            object directory).
+            force_checkout: Force checking out of the repo content.
+            force_rebase: Force rebase.
+            verbose: Whether to show verbose messages.
+            project_idx: Project index for the project to checkout.
 
         Returns:
             Whether the fetch was successful.
         """
+        project = cls.get_parallel_context()["projects"][project_idx]
         start = time.time()
         syncbuf = SyncBuffer(
             project.manifest.manifestProject.config, detach_head=detach_head
@@ -975,7 +1050,12 @@ later is required to fix a server side protocol bug.
         errors = []
         try:
             project.Sync_LocalHalf(
-                syncbuf, force_sync=force_sync, errors=errors
+                syncbuf,
+                force_sync=force_sync,
+                force_checkout=force_checkout,
+                force_rebase=force_rebase,
+                errors=errors,
+                verbose=verbose,
             )
             success = syncbuf.Finish()
         except GitError as e:
@@ -995,7 +1075,7 @@ later is required to fix a server side protocol bug.
         if not success:
             logger.error("error: Cannot checkout %s", project.name)
         finish = time.time()
-        return _CheckoutOneResult(success, errors, project, start, finish)
+        return _CheckoutOneResult(success, errors, project_idx, start, finish)
 
     def _Checkout(self, all_projects, opt, err_results, checkout_errors):
         """Checkout projects listed in all_projects
@@ -1013,7 +1093,9 @@ later is required to fix a server side protocol bug.
             ret = True
             for result in results:
                 success = result.success
-                project = result.project
+                project = self.get_parallel_context()["projects"][
+                    result.project_idx
+                ]
                 start = result.start
                 finish = result.finish
                 self.event_log.AddSync(
@@ -1039,15 +1121,29 @@ later is required to fix a server side protocol bug.
                 pm.update(msg=project.name)
             return ret
 
-        proc_res = self.ExecuteInParallel(
-            opt.jobs_checkout,
-            functools.partial(
-                self._CheckoutOne, opt.detach_head, opt.force_sync
-            ),
-            all_projects,
-            callback=_ProcessResults,
-            output=Progress("Checking out", len(all_projects), quiet=opt.quiet),
-        )
+        for projects in _SafeCheckoutOrder(all_projects):
+            with self.ParallelContext():
+                self.get_parallel_context()["projects"] = projects
+                proc_res = self.ExecuteInParallel(
+                    opt.jobs_checkout,
+                    functools.partial(
+                        self._CheckoutOne,
+                        opt.detach_head,
+                        opt.force_sync,
+                        opt.force_checkout,
+                        opt.rebase,
+                        opt.verbose,
+                    ),
+                    range(len(projects)),
+                    callback=_ProcessResults,
+                    output=Progress(
+                        "Checking out", len(all_projects), quiet=opt.quiet
+                    ),
+                    # Use chunksize=1 to avoid the chance that some workers are
+                    # idle while other workers still have more than one job in
+                    # their chunk queue.
+                    chunksize=1,
+                )
 
         self._local_sync_state.Save()
         return proc_res and not err_results
@@ -1288,7 +1384,7 @@ later is required to fix a server side protocol bug.
                             groups=None,
                         )
                         project.DeleteWorktree(
-                            quiet=opt.quiet, force=opt.force_remove_dirty
+                            verbose=opt.verbose, force=opt.force_remove_dirty
                         )
 
         new_project_paths.sort()
@@ -1346,7 +1442,10 @@ later is required to fix a server side protocol bug.
             for need_remove_file in need_remove_files:
                 # Try to remove the updated copyfile or linkfile.
                 # So, if the file is not exist, nothing need to do.
-                platform_utils.remove(need_remove_file, missing_ok=True)
+                platform_utils.remove(
+                    os.path.join(self.client.topdir, need_remove_file),
+                    missing_ok=True,
+                )
 
         # Create copy-link-files.json, save dest path of "copyfile" and
         # "linkfile".
@@ -1394,7 +1493,7 @@ later is required to fix a server side protocol bug.
 
             if username and password:
                 manifest_server = manifest_server.replace(
-                    "://", "://%s:%s@" % (username, password), 1
+                    "://", f"://{username}:{password}@", 1
                 )
 
         transport = PersistentTransport(manifest_server)
@@ -1411,6 +1510,19 @@ later is required to fix a server side protocol bug.
                     [success, manifest_str] = server.GetApprovedManifest(
                         branch, target
                     )
+                elif (
+                    "TARGET_PRODUCT" in os.environ
+                    and "TARGET_BUILD_VARIANT" in os.environ
+                    and "TARGET_RELEASE" in os.environ
+                ):
+                    target = "%s-%s-%s" % (
+                        os.environ["TARGET_PRODUCT"],
+                        os.environ["TARGET_RELEASE"],
+                        os.environ["TARGET_BUILD_VARIANT"],
+                    )
+                    [success, manifest_str] = server.GetApprovedManifest(
+                        branch, target
+                    )
                 elif (
                     "TARGET_PRODUCT" in os.environ
                     and "TARGET_BUILD_VARIANT" in os.environ
@@ -1500,7 +1612,7 @@ later is required to fix a server side protocol bug.
             buf = TeeStringIO(sys.stdout)
             try:
                 result = mp.Sync_NetworkHalf(
-                    quiet=opt.quiet,
+                    quiet=not opt.verbose,
                     output_redir=buf,
                     verbose=opt.verbose,
                     current_branch_only=self._GetCurrentBranchOnly(
@@ -1533,16 +1645,17 @@ later is required to fix a server side protocol bug.
             syncbuf = SyncBuffer(mp.config)
             start = time.time()
             mp.Sync_LocalHalf(
-                syncbuf, submodules=mp.manifest.HasSubmodules, errors=errors
+                syncbuf,
+                submodules=mp.manifest.HasSubmodules,
+                errors=errors,
+                verbose=opt.verbose,
             )
             clean = syncbuf.Finish()
             self.event_log.AddSync(
                 mp, event_log.TASK_SYNC_LOCAL, start, time.time(), clean
             )
             if not clean:
-                raise UpdateManifestError(
-                    aggregate_errors=errors, project=mp.name
-                )
+                raise UpdateManifestError(aggregate_errors=errors)
             self._ReloadManifest(manifest_name, mp.manifest)
 
     def ValidateOptions(self, opt, args):
@@ -1573,16 +1686,6 @@ later is required to fix a server side protocol bug.
         if opt.prune is None:
             opt.prune = True
 
-        if opt.auto_gc is None and _AUTO_GC:
-            logger.error(
-                "Will run `git gc --auto` because %s is set. %s is deprecated "
-                "and will be removed in a future release.  Use `--auto-gc` "
-                "instead.",
-                _REPO_AUTO_GC,
-                _REPO_AUTO_GC,
-            )
-            opt.auto_gc = True
-
     def _ValidateOptionsWithManifest(self, opt, mp):
         """Like ValidateOptions, but after we've updated the manifest.
 
@@ -1626,7 +1729,7 @@ later is required to fix a server side protocol bug.
         errors = []
         try:
             self._ExecuteHelper(opt, args, errors)
-        except RepoExitError:
+        except (RepoExitError, RepoChangedException):
             raise
         except (KeyboardInterrupt, Exception) as e:
             raise RepoUnhandledExceptionError(e, aggregate_errors=errors)
@@ -2011,7 +2114,7 @@ class LocalSyncState:
         delete = set()
         for path in self._state:
             gitdir = os.path.join(self._manifest.topdir, path, ".git")
-            if not os.path.exists(gitdir):
+            if not os.path.exists(gitdir) or os.path.islink(gitdir):
                 delete.add(path)
         if not delete:
             return

subcmds/upload.py

@@ -218,9 +218,14 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
     def _Options(self, p):
         p.add_option(
             "-t",
+            "--topic-branch",
             dest="auto_topic",
             action="store_true",
-            help="send local branch name to Gerrit Code Review",
+            help="set the topic to the local branch name",
+        )
+        p.add_option(
+            "--topic",
+            help="set topic for the change",
         )
         p.add_option(
             "--hashtag",
@@ -244,6 +249,12 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
             default=[],
             help="add a label when uploading",
         )
+        p.add_option(
+            "--pd",
+            "--patchset-description",
+            dest="patchset_description",
+            help="description for patchset",
+        )
         p.add_option(
             "--re",
             "--reviewers",
@@ -543,42 +554,14 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
         people = copy.deepcopy(original_people)
         self._AppendAutoList(branch, people)
 
-        # Check if there are local changes that may have been forgotten.
-        changes = branch.project.UncommitedFiles()
-        if opt.ignore_untracked_files:
-            untracked = set(branch.project.UntrackedFiles())
-            changes = [x for x in changes if x not in untracked]
-
-        if changes:
-            key = "review.%s.autoupload" % branch.project.remote.review
-            answer = branch.project.config.GetBoolean(key)
-
-            # If they want to auto upload, let's not ask because it
-            # could be automated.
-            if answer is None:
-                print()
-                print(
-                    "Uncommitted changes in %s (did you forget to "
-                    "amend?):" % branch.project.name
-                )
-                print("\n".join(changes))
-                print("Continue uploading? (y/N) ", end="", flush=True)
-                if opt.yes:
-                    print("<--yes>")
-                    a = "yes"
-                else:
-                    a = sys.stdin.readline().strip().lower()
-                if a not in ("y", "yes", "t", "true", "on"):
-                    print("skipping upload", file=sys.stderr)
-                    branch.uploaded = False
-                    branch.error = "User aborted"
-                    return
-
         # Check if topic branches should be sent to the server during
         # upload.
-        if opt.auto_topic is not True:
-            key = "review.%s.uploadtopic" % branch.project.remote.review
-            opt.auto_topic = branch.project.config.GetBoolean(key)
+        if opt.topic is None:
+            if opt.auto_topic is not True:
+                key = "review.%s.uploadtopic" % branch.project.remote.review
+                opt.auto_topic = branch.project.config.GetBoolean(key)
+            if opt.auto_topic:
+                opt.topic = branch.name
 
         def _ExpandCommaList(value):
             """Split |value| up into comma delimited entries."""
@@ -620,19 +603,22 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
             full_dest = destination
             if not full_dest.startswith(R_HEADS):
                 full_dest = R_HEADS + full_dest
+            full_revision = branch.project.revisionExpr
+            if not full_revision.startswith(R_HEADS):
+                full_revision = R_HEADS + full_revision
 
             # If the merge branch of the local branch is different from
             # the project's revision AND destination, this might not be
             # intentional.
             if (
                 merge_branch
-                and merge_branch != branch.project.revisionExpr
+                and merge_branch != full_revision
                 and merge_branch != full_dest
             ):
                 print(
                     f"For local branch {branch.name}: merge branch "
                     f"{merge_branch} does not match destination branch "
-                    f"{destination}"
+                    f"{destination} and revision {branch.project.revisionExpr}"
                 )
                 print("skipping upload.")
                 print(
@@ -645,7 +631,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
         branch.UploadForReview(
             people,
             dryrun=opt.dryrun,
-            auto_topic=opt.auto_topic,
+            topic=opt.topic,
             hashtags=hashtags,
             labels=labels,
             private=opt.private,
@@ -655,6 +641,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
             dest_branch=destination,
             validate_certs=opt.validate_certs,
             push_options=opt.push_options,
+            patchset_description=opt.patchset_description,
         )
 
         branch.uploaded = True
@@ -729,16 +716,17 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
         merge_branch = p.stdout.strip()
         return merge_branch
 
-    @staticmethod
-    def _GatherOne(opt, project):
+    @classmethod
+    def _GatherOne(cls, opt, project_idx):
         """Figure out the upload status for |project|."""
+        project = cls.get_parallel_context()["projects"][project_idx]
         if opt.current_branch:
             cbr = project.CurrentBranch
             up_branch = project.GetUploadableBranch(cbr)
             avail = [up_branch] if up_branch else None
         else:
             avail = project.GetUploadableBranches(opt.branch)
-        return (project, avail)
+        return (project_idx, avail)
 
     def Execute(self, opt, args):
         projects = self.GetProjects(
@@ -748,7 +736,8 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
         def _ProcessResults(_pool, _out, results):
             pending = []
             for result in results:
-                project, avail = result
+                project_idx, avail = result
+                project = projects[project_idx]
                 if avail is None:
                     logger.error(
                         'repo: error: %s: Unable to upload branch "%s". '
@@ -759,15 +748,17 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
                         project.manifest.branch,
                     )
                 elif avail:
-                    pending.append(result)
+                    pending.append((project, avail))
             return pending
 
-        pending = self.ExecuteInParallel(
-            opt.jobs,
-            functools.partial(self._GatherOne, opt),
-            projects,
-            callback=_ProcessResults,
-        )
+        with self.ParallelContext():
+            self.get_parallel_context()["projects"] = projects
+            pending = self.ExecuteInParallel(
+                opt.jobs,
+                functools.partial(self._GatherOne, opt),
+                range(len(projects)),
+                callback=_ProcessResults,
+            )
 
         if not pending:
             if opt.branch is None:

subcmds/version.py

@@ -42,35 +42,28 @@ class Version(Command, MirrorSafeCommand):
         # These might not be the same.  Report them both.
         src_ver = RepoSourceVersion()
         rp_ver = rp.bare_git.describe(HEAD)
-        print("repo version %s" % rp_ver)
-        print("       (from %s)" % rem.url)
-        print("       (tracking %s)" % branch.merge)
-        print("       (%s)" % rp.bare_git.log("-1", "--format=%cD", HEAD))
+        print(f"repo version {rp_ver}")
+        print(f"       (from {rem.url})")
+        print(f"       (tracking {branch.merge})")
+        print(f"       ({rp.bare_git.log('-1', '--format=%cD', HEAD)})")
 
         if self.wrapper_path is not None:
-            print("repo launcher version %s" % self.wrapper_version)
-            print("       (from %s)" % self.wrapper_path)
+            print(f"repo launcher version {self.wrapper_version}")
+            print(f"       (from {self.wrapper_path})")
 
             if src_ver != rp_ver:
-                print("       (currently at %s)" % src_ver)
+                print(f"       (currently at {src_ver})")
 
-        print("repo User-Agent %s" % user_agent.repo)
-        print("git %s" % git.version_tuple().full)
-        print("git User-Agent %s" % user_agent.git)
-        print("Python %s" % sys.version)
+        print(f"repo User-Agent {user_agent.repo}")
+        print(f"git {git.version_tuple().full}")
+        print(f"git User-Agent {user_agent.git}")
+        print(f"Python {sys.version}")
         uname = platform.uname()
         if sys.version_info.major < 3:
             # Python 3 returns a named tuple, but Python 2 is simpler.
             print(uname)
         else:
-            print(
-                "OS %s %s (%s)" % (uname.system, uname.release, uname.version)
-            )
-            print(
-                "CPU %s (%s)"
-                % (
-                    uname.machine,
-                    uname.processor if uname.processor else "unknown",
-                )
-            )
+            print(f"OS {uname.system} {uname.release} ({uname.version})")
+            processor = uname.processor if uname.processor else "unknown"
+            print(f"CPU {uname.machine} ({processor})")
         print("Bug reports:", Wrapper().BUG_URL)

tests/conftest.py

@@ -72,3 +72,12 @@ def tmp_home_dir(monkeypatch, tmp_path_factory):
     the function scope.
     """
     return _set_home(monkeypatch, tmp_path_factory.mktemp("home"))
+
+
+@pytest.fixture(autouse=True)
+def setup_user_identity(monkeysession, scope="session"):
+    """Set env variables for author and committer name and email."""
+    monkeysession.setenv("GIT_AUTHOR_NAME", "Foo Bar")
+    monkeysession.setenv("GIT_COMMITTER_NAME", "Foo Bar")
+    monkeysession.setenv("GIT_AUTHOR_EMAIL", "foo@bar.baz")
+    monkeysession.setenv("GIT_COMMITTER_EMAIL", "foo@bar.baz")

tests/fixtures/gitc_config

@@ -1 +0,0 @@
-gitc_dir=/test/usr/local/google/gitc

tests/fixtures/test.gitconfig

@@ -11,3 +11,11 @@
 	intk = 10k
 	intm = 10m
 	intg = 10g
+
+[color "status"]
+	one = yellow
+	two = magenta cyan
+	three = black red ul
+	reset = reset
+	none
+	empty =

tests/test_color.py

@@ -0,0 +1,74 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for the color.py module."""
+
+import os
+import unittest
+
+import color
+import git_config
+
+
+def fixture(*paths):
+    """Return a path relative to test/fixtures."""
+    return os.path.join(os.path.dirname(__file__), "fixtures", *paths)
+
+
+class ColoringTests(unittest.TestCase):
+    """tests of the Coloring class."""
+
+    def setUp(self):
+        """Create a GitConfig object using the test.gitconfig fixture."""
+        config_fixture = fixture("test.gitconfig")
+        self.config = git_config.GitConfig(config_fixture)
+        color.SetDefaultColoring("true")
+        self.color = color.Coloring(self.config, "status")
+
+    def test_Color_Parse_all_params_none(self):
+        """all params are None"""
+        val = self.color._parse(None, None, None, None)
+        self.assertEqual("", val)
+
+    def test_Color_Parse_first_parameter_none(self):
+        """check fg & bg & attr"""
+        val = self.color._parse(None, "black", "red", "ul")
+        self.assertEqual("\x1b[4;30;41m", val)
+
+    def test_Color_Parse_one_entry(self):
+        """check fg"""
+        val = self.color._parse("one", None, None, None)
+        self.assertEqual("\033[33m", val)
+
+    def test_Color_Parse_two_entry(self):
+        """check fg & bg"""
+        val = self.color._parse("two", None, None, None)
+        self.assertEqual("\033[35;46m", val)
+
+    def test_Color_Parse_three_entry(self):
+        """check fg & bg & attr"""
+        val = self.color._parse("three", None, None, None)
+        self.assertEqual("\033[4;30;41m", val)
+
+    def test_Color_Parse_reset_entry(self):
+        """check reset entry"""
+        val = self.color._parse("reset", None, None, None)
+        self.assertEqual("\033[m", val)
+
+    def test_Color_Parse_empty_entry(self):
+        """check empty entry"""
+        val = self.color._parse("none", "blue", "white", "dim")
+        self.assertEqual("\033[2;34;47m", val)
+        val = self.color._parse("empty", "green", "white", "bold")
+        self.assertEqual("\033[1;32;47m", val)

tests/test_git_command.py

@@ -19,12 +19,7 @@ import os
 import re
 import subprocess
 import unittest
-
-
-try:
-    from unittest import mock
-except ImportError:
-    import mock
+from unittest import mock
 
 import git_command
 import wrapper

tests/test_git_config.py

@@ -100,7 +100,7 @@ class GitConfigReadOnlyTests(unittest.TestCase):
             ("intg", 10737418240),
         )
         for key, value in TESTS:
-            self.assertEqual(value, self.config.GetInt("section.%s" % (key,)))
+            self.assertEqual(value, self.config.GetInt(f"section.{key}"))
 
 
 class GitConfigReadWriteTests(unittest.TestCase):

tests/test_git_superproject.py

@@ -34,7 +34,7 @@ class SuperprojectTestCase(unittest.TestCase):
     PARENT_SID_KEY = "GIT_TRACE2_PARENT_SID"
     PARENT_SID_VALUE = "parent_sid"
     SELF_SID_REGEX = r"repo-\d+T\d+Z-.*"
-    FULL_SID_REGEX = r"^%s/%s" % (PARENT_SID_VALUE, SELF_SID_REGEX)
+    FULL_SID_REGEX = rf"^{PARENT_SID_VALUE}/{SELF_SID_REGEX}"
 
     def setUp(self):
         """Set up superproject every time."""

tests/test_git_trace2_event_log.py

@@ -61,7 +61,7 @@ class EventLogTestCase(unittest.TestCase):
     PARENT_SID_KEY = "GIT_TRACE2_PARENT_SID"
     PARENT_SID_VALUE = "parent_sid"
     SELF_SID_REGEX = r"repo-\d+T\d+Z-.*"
-    FULL_SID_REGEX = r"^%s/%s" % (PARENT_SID_VALUE, SELF_SID_REGEX)
+    FULL_SID_REGEX = rf"^{PARENT_SID_VALUE}/{SELF_SID_REGEX}"
 
     def setUp(self):
         """Load the event_log module every time."""
@@ -150,7 +150,7 @@ class EventLogTestCase(unittest.TestCase):
         <version event>
         <start event>
         """
-        self._event_log_module.StartEvent()
+        self._event_log_module.StartEvent([])
         with tempfile.TemporaryDirectory(prefix="event_log_tests") as tempdir:
             log_path = self._event_log_module.Write(path=tempdir)
             self._log_data = self.readLog(log_path)
@@ -213,10 +213,8 @@ class EventLogTestCase(unittest.TestCase):
         <version event>
         <command event>
         """
-        name = "repo"
-        subcommands = ["init" "this"]
         self._event_log_module.CommandEvent(
-            name="repo", subcommands=subcommands
+            name="repo", subcommands=["init", "this"]
         )
         with tempfile.TemporaryDirectory(prefix="event_log_tests") as tempdir:
             log_path = self._event_log_module.Write(path=tempdir)
@@ -225,12 +223,10 @@ class EventLogTestCase(unittest.TestCase):
         self.assertEqual(len(self._log_data), 2)
         command_event = self._log_data[1]
         self.verifyCommonKeys(self._log_data[0], expected_event_name="version")
-        self.verifyCommonKeys(command_event, expected_event_name="command")
+        self.verifyCommonKeys(command_event, expected_event_name="cmd_name")
         # Check for 'command' event specific fields.
         self.assertIn("name", command_event)
-        self.assertIn("subcommands", command_event)
-        self.assertEqual(command_event["name"], name)
-        self.assertEqual(command_event["subcommands"], subcommands)
+        self.assertEqual(command_event["name"], "repo-init-this")
 
     def test_def_params_event_repo_config(self):
         """Test 'def_params' event data outputs only repo config keys.
@@ -382,17 +378,17 @@ class EventLogTestCase(unittest.TestCase):
             socket_path = os.path.join(tempdir, "server.sock")
             server_ready = threading.Condition()
             # Start "server" listening on Unix domain socket at socket_path.
+            server_thread = threading.Thread(
+                target=serverLoggingThread,
+                args=(socket_path, server_ready, received_traces),
+            )
             try:
-                server_thread = threading.Thread(
-                    target=serverLoggingThread,
-                    args=(socket_path, server_ready, received_traces),
-                )
                 server_thread.start()
 
                 with server_ready:
                     server_ready.wait(timeout=120)
 
-                self._event_log_module.StartEvent()
+                self._event_log_module.StartEvent([])
                 path = self._event_log_module.Write(
                     path=f"af_unix:{socket_path}"
                 )

tests/test_manifest_xml.py

@@ -198,13 +198,13 @@ class ValueTests(unittest.TestCase):
     def test_bool_true(self):
         """Check XmlBool true values."""
         for value in ("yes", "true", "1"):
-            node = self._get_node('<node a="%s"/>' % (value,))
+            node = self._get_node(f'<node a="{value}"/>')
             self.assertTrue(manifest_xml.XmlBool(node, "a"))
 
     def test_bool_false(self):
         """Check XmlBool false values."""
         for value in ("no", "false", "0"):
-            node = self._get_node('<node a="%s"/>' % (value,))
+            node = self._get_node(f'<node a="{value}"/>')
             self.assertFalse(manifest_xml.XmlBool(node, "a"))
 
     def test_int_default(self):
@@ -220,7 +220,7 @@ class ValueTests(unittest.TestCase):
     def test_int_good(self):
         """Check XmlInt numeric handling."""
         for value in (-1, 0, 1, 50000):
-            node = self._get_node('<node a="%s"/>' % (value,))
+            node = self._get_node(f'<node a="{value}"/>')
             self.assertEqual(value, manifest_xml.XmlInt(node, "a"))
 
     def test_int_invalid(self):
@@ -1049,6 +1049,91 @@ class RemoveProjectElementTests(ManifestParseTestCase):
         self.assertTrue(found_proj1_path1)
         self.assertTrue(found_proj2)
 
+    def test_base_revision_checks_on_patching(self):
+        manifest_fail_wrong_tag = self.getXmlManifest(
+            """
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="tag.002" />
+  <project name="project1" path="tests/path1" />
+  <extend-project name="project1" revision="new_hash" base-rev="tag.001" />
+</manifest>
+"""
+        )
+        with self.assertRaises(error.ManifestParseError):
+            manifest_fail_wrong_tag.ToXml()
+
+        manifest_fail_remove = self.getXmlManifest(
+            """
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="project1" path="tests/path1" revision="hash1" />
+  <remove-project name="project1" base-rev="wrong_hash" />
+</manifest>
+"""
+        )
+        with self.assertRaises(error.ManifestParseError):
+            manifest_fail_remove.ToXml()
+
+        manifest_fail_extend = self.getXmlManifest(
+            """
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="project1" path="tests/path1" revision="hash1" />
+  <extend-project name="project1" revision="new_hash" base-rev="wrong_hash" />
+</manifest>
+"""
+        )
+        with self.assertRaises(error.ManifestParseError):
+            manifest_fail_extend.ToXml()
+
+        manifest_fail_unknown = self.getXmlManifest(
+            """
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="project1" path="tests/path1" />
+  <extend-project name="project1" revision="new_hash" base-rev="any_hash" />
+</manifest>
+"""
+        )
+        with self.assertRaises(error.ManifestParseError):
+            manifest_fail_unknown.ToXml()
+
+        manifest_ok = self.getXmlManifest(
+            """
+<manifest>
+  <remote name="default-remote" fetch="http://localhost" />
+  <default remote="default-remote" revision="refs/heads/main" />
+  <project name="project1" path="tests/path1" revision="hash1" />
+  <project name="project2" path="tests/path2" revision="hash2" />
+  <project name="project3" path="tests/path3" revision="hash3" />
+  <project name="project4" path="tests/path4" revision="hash4" />
+
+  <remove-project name="project1" />
+  <remove-project name="project2" base-rev="hash2" />
+  <project name="project2" path="tests/path2" revision="new_hash2" />
+  <extend-project name="project3" base-rev="hash3" revision="new_hash3" />
+  <extend-project name="project3" base-rev="new_hash3" revision="newer_hash3" />
+  <remove-project path="tests/path4" base-rev="hash4" />
+</manifest>
+"""
+        )
+        found_proj2 = False
+        found_proj3 = False
+        for proj in manifest_ok.projects:
+            if proj.name == "project2":
+                found_proj2 = True
+            if proj.name == "project3":
+                found_proj3 = True
+            self.assertNotEqual(proj.name, "project1")
+            self.assertNotEqual(proj.name, "project4")
+        self.assertTrue(found_proj2)
+        self.assertTrue(found_proj3)
+        self.assertTrue(len(manifest_ok.projects) == 2)
+
 
 class ExtendProjectElementTests(ManifestParseTestCase):
     """Tests for <extend-project>."""
@@ -1128,3 +1213,79 @@ class ExtendProjectElementTests(ManifestParseTestCase):
         )
         self.assertEqual(len(manifest.projects), 1)
         self.assertEqual(manifest.projects[0].upstream, "bar")
+
+
+class NormalizeUrlTests(ManifestParseTestCase):
+    """Tests for normalize_url() in manifest_xml.py"""
+
+    def test_has_trailing_slash(self):
+        url = "http://foo.com/bar/baz/"
+        self.assertEqual(
+            "http://foo.com/bar/baz", manifest_xml.normalize_url(url)
+        )
+
+        url = "http://foo.com/bar/"
+        self.assertEqual("http://foo.com/bar", manifest_xml.normalize_url(url))
+
+    def test_has_leading_slash(self):
+        """SCP-like syntax except a / comes before the : which git disallows."""
+        url = "/git@foo.com:bar/baf"
+        self.assertEqual(url, manifest_xml.normalize_url(url))
+
+        url = "gi/t@foo.com:bar/baf"
+        self.assertEqual(url, manifest_xml.normalize_url(url))
+
+        url = "git@fo/o.com:bar/baf"
+        self.assertEqual(url, manifest_xml.normalize_url(url))
+
+    def test_has_no_scheme(self):
+        """Deal with cases where we have no scheme, but we also
+        aren't dealing with the git SCP-like syntax
+        """
+        url = "foo.com/baf/bat"
+        self.assertEqual(url, manifest_xml.normalize_url(url))
+
+        url = "foo.com/baf"
+        self.assertEqual(url, manifest_xml.normalize_url(url))
+
+        url = "git@foo.com/baf/bat"
+        self.assertEqual(url, manifest_xml.normalize_url(url))
+
+        url = "git@foo.com/baf"
+        self.assertEqual(url, manifest_xml.normalize_url(url))
+
+        url = "/file/path/here"
+        self.assertEqual(url, manifest_xml.normalize_url(url))
+
+    def test_has_no_scheme_matches_scp_like_syntax(self):
+        url = "git@foo.com:bar/baf"
+        self.assertEqual(
+            "ssh://git@foo.com/bar/baf", manifest_xml.normalize_url(url)
+        )
+
+        url = "git@foo.com:bar/"
+        self.assertEqual(
+            "ssh://git@foo.com/bar", manifest_xml.normalize_url(url)
+        )
+
+    def test_remote_url_resolution(self):
+        remote = manifest_xml._XmlRemote(
+            name="foo",
+            fetch="git@github.com:org2/",
+            manifestUrl="git@github.com:org2/custom_manifest.git",
+        )
+        self.assertEqual("ssh://git@github.com/org2", remote.resolvedFetchUrl)
+
+        remote = manifest_xml._XmlRemote(
+            name="foo",
+            fetch="ssh://git@github.com/org2/",
+            manifestUrl="git@github.com:org2/custom_manifest.git",
+        )
+        self.assertEqual("ssh://git@github.com/org2", remote.resolvedFetchUrl)
+
+        remote = manifest_xml._XmlRemote(
+            name="foo",
+            fetch="git@github.com:org2/",
+            manifestUrl="ssh://git@github.com/org2/custom_manifest.git",
+        )
+        self.assertEqual("ssh://git@github.com/org2", remote.resolvedFetchUrl)

tests/test_project.py

@@ -107,6 +107,16 @@ class ReviewableBranchTests(unittest.TestCase):
             self.assertTrue(rb.date)
 
 
+class ProjectTests(unittest.TestCase):
+    """Check Project behavior."""
+
+    def test_encode_patchset_description(self):
+        self.assertEqual(
+            project.Project._encode_patchset_description("abcd00!! +"),
+            "abcd00%21%21_%2b",
+        )
+
+
 class CopyLinkTestCase(unittest.TestCase):
     """TestCase for stub repo client checkouts.
 
@@ -151,7 +161,7 @@ class CopyLinkTestCase(unittest.TestCase):
                     # "".
                     break
                 result = os.path.exists(path)
-                msg.append("\tos.path.exists(%s): %s" % (path, result))
+                msg.append(f"\tos.path.exists({path}): {result}")
                 if result:
                     msg.append("\tcontents: %r" % os.listdir(path))
                     break

tests/test_repo_logging.py

@@ -13,9 +13,14 @@
 # limitations under the License.
 
 """Unit test for repo_logging module."""
+
+import contextlib
+import io
+import logging
 import unittest
 from unittest import mock
 
+from color import SetDefaultColoring
 from error import RepoExitError
 from repo_logging import RepoLogger
 
@@ -62,3 +67,35 @@ class TestRepoLogger(unittest.TestCase):
                 mock.call("Repo command failed: %s", "RepoExitError"),
             ]
         )
+
+    def test_log_with_format_string(self):
+        """Test different log levels with format strings."""
+
+        # Set color output to "always" for consistent test results.
+        # This ensures the logger's behavior is uniform across different
+        # environments and git configurations.
+        SetDefaultColoring("always")
+
+        # Regex pattern to match optional ANSI color codes.
+        # \033    - Escape character
+        # \[      - Opening square bracket
+        # [0-9;]* - Zero or more digits or semicolons
+        # m       - Ending 'm' character
+        # ?       - Makes the entire group optional
+        opt_color = r"(\033\[[0-9;]*m)?"
+
+        for level in (logging.INFO, logging.WARN, logging.ERROR):
+            name = logging.getLevelName(level)
+
+            with self.subTest(level=level, name=name):
+                output = io.StringIO()
+
+                with contextlib.redirect_stderr(output):
+                    logger = RepoLogger(__name__)
+                    logger.log(level, "%s", "100% pass")
+
+                self.assertRegex(
+                    output.getvalue().strip(),
+                    f"^{opt_color}100% pass{opt_color}$",
+                    f"failed for level {name}",
+                )

tests/test_subcmds_forall.py

@@ -0,0 +1,156 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for the forall subcmd."""
+
+from io import StringIO
+import os
+from shutil import rmtree
+import subprocess
+import tempfile
+import unittest
+from unittest import mock
+
+import git_command
+import manifest_xml
+import project
+import subcmds
+
+
+class AllCommands(unittest.TestCase):
+    """Check registered all_commands."""
+
+    def setUp(self):
+        """Common setup."""
+        self.tempdirobj = tempfile.TemporaryDirectory(prefix="forall_tests")
+        self.tempdir = self.tempdirobj.name
+        self.repodir = os.path.join(self.tempdir, ".repo")
+        self.manifest_dir = os.path.join(self.repodir, "manifests")
+        self.manifest_file = os.path.join(
+            self.repodir, manifest_xml.MANIFEST_FILE_NAME
+        )
+        self.local_manifest_dir = os.path.join(
+            self.repodir, manifest_xml.LOCAL_MANIFESTS_DIR_NAME
+        )
+        os.mkdir(self.repodir)
+        os.mkdir(self.manifest_dir)
+
+    def tearDown(self):
+        """Common teardown."""
+        rmtree(self.tempdir, ignore_errors=True)
+
+    def initTempGitTree(self, git_dir):
+        """Create a new empty git checkout for testing."""
+
+        # Tests need to assume, that main is default branch at init,
+        # which is not supported in config until 2.28.
+        cmd = ["git", "init", "-q"]
+        if git_command.git_require((2, 28, 0)):
+            cmd += ["--initial-branch=main"]
+        else:
+            # Use template dir for init
+            templatedir = os.path.join(self.tempdirobj.name, ".test-template")
+            os.makedirs(templatedir)
+            with open(os.path.join(templatedir, "HEAD"), "w") as fp:
+                fp.write("ref: refs/heads/main\n")
+            cmd += ["--template", templatedir]
+        cmd += [git_dir]
+        subprocess.check_call(cmd)
+
+    def getXmlManifestWith8Projects(self):
+        """Create and return a setup of 8 projects with enough dummy
+        files and setup to execute forall."""
+
+        # Set up a manifest git dir for parsing to work
+        gitdir = os.path.join(self.repodir, "manifests.git")
+        os.mkdir(gitdir)
+        with open(os.path.join(gitdir, "config"), "w") as fp:
+            fp.write(
+                """[remote "origin"]
+                    url = https://localhost:0/manifest
+                    verbose = false
+                """
+            )
+
+        # Add the manifest data
+        manifest_data = """
+                <manifest>
+                    <remote name="origin" fetch="http://localhost" />
+                    <default remote="origin" revision="refs/heads/main" />
+                    <project name="project1" path="tests/path1" />
+                    <project name="project2" path="tests/path2" />
+                    <project name="project3" path="tests/path3" />
+                    <project name="project4" path="tests/path4" />
+                    <project name="project5" path="tests/path5" />
+                    <project name="project6" path="tests/path6" />
+                    <project name="project7" path="tests/path7" />
+                    <project name="project8" path="tests/path8" />
+                </manifest>
+            """
+        with open(self.manifest_file, "w", encoding="utf-8") as fp:
+            fp.write(manifest_data)
+
+        # Set up 8 empty projects to match the manifest
+        for x in range(1, 9):
+            os.makedirs(
+                os.path.join(
+                    self.repodir, "projects/tests/path" + str(x) + ".git"
+                )
+            )
+            os.makedirs(
+                os.path.join(
+                    self.repodir, "project-objects/project" + str(x) + ".git"
+                )
+            )
+            git_path = os.path.join(self.tempdir, "tests/path" + str(x))
+            self.initTempGitTree(git_path)
+
+        return manifest_xml.XmlManifest(self.repodir, self.manifest_file)
+
+    # Use mock to capture stdout from the forall run
+    @unittest.mock.patch("sys.stdout", new_callable=StringIO)
+    def test_forall_all_projects_called_once(self, mock_stdout):
+        """Test that all projects get a command run once each."""
+
+        manifest_with_8_projects = self.getXmlManifestWith8Projects()
+
+        cmd = subcmds.forall.Forall()
+        cmd.manifest = manifest_with_8_projects
+
+        # Use echo project names as the test of forall
+        opts, args = cmd.OptionParser.parse_args(["-c", "echo $REPO_PROJECT"])
+        opts.verbose = False
+
+        # Mock to not have the Execute fail on remote check
+        with mock.patch.object(
+            project.Project, "GetRevisionId", return_value="refs/heads/main"
+        ):
+            # Run the forall command
+            cmd.Execute(opts, args)
+
+            # Verify that we got every project name in the prints
+            for x in range(1, 9):
+                self.assertIn("project" + str(x), mock_stdout.getvalue())
+
+            # Split the captured output into lines to count them
+            line_count = 0
+            for line in mock_stdout.getvalue().split("\n"):
+                # A commented out print to stderr as a reminder
+                # that stdout is mocked, include sys and uncomment if needed
+                # print(line, file=sys.stderr)
+                if len(line) > 0:
+                    line_count += 1
+
+            # Verify that we didn't get more lines than expected
+            assert line_count == 8

tests/test_subcmds_sync.py

@@ -265,6 +265,119 @@ class LocalSyncState(unittest.TestCase):
         self.assertIsNone(self.state.GetFetchTime(projA))
         self.assertEqual(self.state.GetFetchTime(projB), 7)
 
+    def test_prune_removed_and_symlinked_projects(self):
+        """Removed projects that still exists on disk as symlink are pruned."""
+        with open(self.state._path, "w") as f:
+            f.write(
+                """
+            {
+              "projA": {
+                "last_fetch": 5
+              },
+              "projB": {
+                "last_fetch": 7
+              }
+            }
+            """
+            )
+
+        def mock_exists(path):
+            return True
+
+        def mock_islink(path):
+            if "projB" in path:
+                return True
+            return False
+
+        projA = mock.MagicMock(relpath="projA")
+        projB = mock.MagicMock(relpath="projB")
+        self.state = self._new_state()
+        self.assertEqual(self.state.GetFetchTime(projA), 5)
+        self.assertEqual(self.state.GetFetchTime(projB), 7)
+        with mock.patch("os.path.exists", side_effect=mock_exists):
+            with mock.patch("os.path.islink", side_effect=mock_islink):
+                self.state.PruneRemovedProjects()
+        self.assertIsNone(self.state.GetFetchTime(projB))
+
+        self.state = self._new_state()
+        self.assertIsNone(self.state.GetFetchTime(projB))
+        self.assertEqual(self.state.GetFetchTime(projA), 5)
+
+
+class FakeProject:
+    def __init__(self, relpath):
+        self.relpath = relpath
+
+    def __str__(self):
+        return f"project: {self.relpath}"
+
+    def __repr__(self):
+        return str(self)
+
+
+class SafeCheckoutOrder(unittest.TestCase):
+    def test_no_nested(self):
+        p_f = FakeProject("f")
+        p_foo = FakeProject("foo")
+        out = sync._SafeCheckoutOrder([p_f, p_foo])
+        self.assertEqual(out, [[p_f, p_foo]])
+
+    def test_basic_nested(self):
+        p_foo = p_foo = FakeProject("foo")
+        p_foo_bar = FakeProject("foo/bar")
+        out = sync._SafeCheckoutOrder([p_foo, p_foo_bar])
+        self.assertEqual(out, [[p_foo], [p_foo_bar]])
+
+    def test_complex_nested(self):
+        p_foo = FakeProject("foo")
+        p_foobar = FakeProject("foobar")
+        p_foo_dash_bar = FakeProject("foo-bar")
+        p_foo_bar = FakeProject("foo/bar")
+        p_foo_bar_baz_baq = FakeProject("foo/bar/baz/baq")
+        p_bar = FakeProject("bar")
+        out = sync._SafeCheckoutOrder(
+            [
+                p_foo_bar_baz_baq,
+                p_foo,
+                p_foobar,
+                p_foo_dash_bar,
+                p_foo_bar,
+                p_bar,
+            ]
+        )
+        self.assertEqual(
+            out,
+            [
+                [p_bar, p_foo, p_foo_dash_bar, p_foobar],
+                [p_foo_bar],
+                [p_foo_bar_baz_baq],
+            ],
+        )
+
+
+class Chunksize(unittest.TestCase):
+    """Tests for _chunksize."""
+
+    def test_single_project(self):
+        """Single project."""
+        self.assertEqual(sync._chunksize(1, 1), 1)
+
+    def test_low_project_count(self):
+        """Multiple projects, low number of projects to sync."""
+        self.assertEqual(sync._chunksize(10, 1), 10)
+        self.assertEqual(sync._chunksize(10, 2), 5)
+        self.assertEqual(sync._chunksize(10, 4), 2)
+        self.assertEqual(sync._chunksize(10, 8), 1)
+        self.assertEqual(sync._chunksize(10, 16), 1)
+
+    def test_high_project_count(self):
+        """Multiple projects, high number of projects to sync."""
+        self.assertEqual(sync._chunksize(2800, 1), 32)
+        self.assertEqual(sync._chunksize(2800, 16), 32)
+        self.assertEqual(sync._chunksize(2800, 32), 32)
+        self.assertEqual(sync._chunksize(2800, 64), 32)
+        self.assertEqual(sync._chunksize(2800, 128), 21)
+
 
 class GetPreciousObjectsState(unittest.TestCase):
     """Tests for _GetPreciousObjectsState."""

tests/test_wrapper.py

@@ -72,84 +72,11 @@ class RepoWrapperUnitTest(RepoWrapperTestCase):
 
     def test_init_parser(self):
         """Make sure 'init' GetParser works."""
-        parser = self.wrapper.GetParser(gitc_init=False)
+        parser = self.wrapper.GetParser()
         opts, args = parser.parse_args([])
         self.assertEqual([], args)
         self.assertIsNone(opts.manifest_url)
 
-    def test_gitc_init_parser(self):
-        """Make sure 'gitc-init' GetParser raises."""
-        with self.assertRaises(SystemExit):
-            self.wrapper.GetParser(gitc_init=True)
-
-    def test_get_gitc_manifest_dir_no_gitc(self):
-        """
-        Test reading a missing gitc config file
-        """
-        self.wrapper.GITC_CONFIG_FILE = fixture("missing_gitc_config")
-        val = self.wrapper.get_gitc_manifest_dir()
-        self.assertEqual(val, "")
-
-    def test_get_gitc_manifest_dir(self):
-        """
-        Test reading the gitc config file and parsing the directory
-        """
-        self.wrapper.GITC_CONFIG_FILE = fixture("gitc_config")
-        val = self.wrapper.get_gitc_manifest_dir()
-        self.assertEqual(val, "/test/usr/local/google/gitc")
-
-    def test_gitc_parse_clientdir_no_gitc(self):
-        """
-        Test parsing the gitc clientdir without gitc running
-        """
-        self.wrapper.GITC_CONFIG_FILE = fixture("missing_gitc_config")
-        self.assertEqual(self.wrapper.gitc_parse_clientdir("/something"), None)
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/test"), "test"
-        )
-
-    def test_gitc_parse_clientdir(self):
-        """
-        Test parsing the gitc clientdir
-        """
-        self.wrapper.GITC_CONFIG_FILE = fixture("gitc_config")
-        self.assertEqual(self.wrapper.gitc_parse_clientdir("/something"), None)
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/test"), "test"
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/test/"), "test"
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/test/extra"),
-            "test",
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir(
-                "/test/usr/local/google/gitc/test"
-            ),
-            "test",
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir(
-                "/test/usr/local/google/gitc/test/"
-            ),
-            "test",
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir(
-                "/test/usr/local/google/gitc/test/extra"
-            ),
-            "test",
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/"), None
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir("/test/usr/local/google/gitc/"),
-            None,
-        )
-
 
 class SetGitTrace2ParentSid(RepoWrapperTestCase):
     """Check SetGitTrace2ParentSid behavior."""

tox.ini

@@ -30,6 +30,7 @@ python =
 
 [testenv]
 deps =
+    -c constraints.txt
     black
     flake8
     isort
@@ -44,20 +45,19 @@ setenv =
 [testenv:lint]
 skip_install = true
 deps =
+    -c constraints.txt
     black
     flake8
 commands =
-    black --check {posargs:.}
+    black --check {posargs:. repo run_tests release/update-hooks release/update-manpages}
     flake8
 
 [testenv:format]
 skip_install = true
 deps =
+    -c constraints.txt
     black
     flake8
 commands =
-    black {posargs:.}
+    black {posargs:. repo run_tests release/update-hooks release/update-manpages}
     flake8
-
-[pytest]
-timeout = 300

wrapper.py

@@ -18,8 +18,12 @@ import importlib.util
 import os
 
 
+def WrapperDir():
+    return os.path.dirname(__file__)
+
+
 def WrapperPath():
-    return os.path.join(os.path.dirname(__file__), "repo")
+    return os.path.join(WrapperDir(), "repo")
 
 
 @functools.lru_cache(maxsize=None)