command.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import contextlib
 import multiprocessing
 import optparse
 import os
@@ -71,14 +70,6 @@ class Command:
     # migrated subcommands can set it to False.
     MULTI_MANIFEST_SUPPORT = True
 
-    # Shared data across parallel execution workers.
-    _parallel_context = None
-
-    @classmethod
-    def get_parallel_context(cls):
-        assert cls._parallel_context is not None
-        return cls._parallel_context
-
     def __init__(
         self,
         repodir=None,
@@ -251,39 +242,9 @@ class Command:
         """Perform the action, after option parsing is complete."""
         raise NotImplementedError
 
-    @classmethod
+    @staticmethod
-    @contextlib.contextmanager
-    def ParallelContext(cls):
-        """Obtains the context, which is shared to ExecuteInParallel workers.
-
-        Callers can store data in the context dict before invocation of
-        ExecuteInParallel. The dict will then be shared to child workers of
-        ExecuteInParallel.
-        """
-        assert cls._parallel_context is None
-        cls._parallel_context = {}
-        try:
-            yield
-        finally:
-            cls._parallel_context = None
-
-    @classmethod
-    def _InitParallelWorker(cls, context, initializer):
-        cls._parallel_context = context
-        if initializer:
-            initializer()
-
-    @classmethod
     def ExecuteInParallel(
-        cls,
+        jobs, func, inputs, callback, output=None, ordered=False
-        jobs,
-        func,
-        inputs,
-        callback,
-        output=None,
-        ordered=False,
-        chunksize=WORKER_BATCH_SIZE,
-        initializer=None,
     ):
         """Helper for managing parallel execution boiler plate.
 
@@ -308,9 +269,6 @@ class Command:
             output: An output manager. May be progress.Progess or
                 color.Coloring.
             ordered: Whether the jobs should be processed in order.
-            chunksize: The number of jobs processed in batch by parallel
-                workers.
-            initializer: Worker initializer.
 
         Returns:
             The |callback| function's results are returned.
@@ -320,16 +278,12 @@ class Command:
             if len(inputs) == 1 or jobs == 1:
                 return callback(None, output, (func(x) for x in inputs))
             else:
-                with multiprocessing.Pool(
+                with multiprocessing.Pool(jobs) as pool:
-                    jobs,
-                    initializer=cls._InitParallelWorker,
-                    initargs=(cls._parallel_context, initializer),
-                ) as pool:
                     submit = pool.imap if ordered else pool.imap_unordered
                     return callback(
                         pool,
                         output,
-                        submit(func, inputs, chunksize=chunksize),
+                        submit(func, inputs, chunksize=WORKER_BATCH_SIZE),
                     )
         finally:
             if isinstance(output, progress.Progress):
@@ -547,3 +501,7 @@ class MirrorSafeCommand:
     """Command permits itself to run within a mirror, and does not require a
     working directory.
     """
+
+
+class GitcClientCommand:
+    """Command that requires the local client to be a GITC client."""

constraints.txt

@@ -1,2 +1 @@
-# NB: Keep in sync with run_tests.vpython3.
+black<24
-black<26

docs/internal-fs-layout.md

@@ -141,7 +141,7 @@ Instead, you should use standard Git workflows like [git worktree] or
     (e.g. a local mirror & a public review server) while avoiding duplicating
     the content.  However, this can run into problems if different remotes use
     the same path on their respective servers.  Best to avoid that.
-*   `modules/`: Like `projects/`, but for git submodules.
+*   `subprojects/`: Like `projects/`, but for git submodules.
 *   `subproject-objects/`: Like `project-objects/`, but for git submodules.
 *   `worktrees/`: Bare checkouts of every project synced by the manifest.  The
     filesystem layout matches the `<project name=...` setting in the manifest

docs/manifest-format.md

@@ -107,13 +107,11 @@ following DTD:
   <!ATTLIST extend-project remote CDATA #IMPLIED>
   <!ATTLIST extend-project dest-branch CDATA #IMPLIED>
   <!ATTLIST extend-project upstream CDATA #IMPLIED>
-  <!ATTLIST extend-project base-rev CDATA #IMPLIED>
 
   <!ELEMENT remove-project EMPTY>
   <!ATTLIST remove-project name     CDATA #IMPLIED>
   <!ATTLIST remove-project path     CDATA #IMPLIED>
   <!ATTLIST remove-project optional CDATA #IMPLIED>
-  <!ATTLIST remove-project base-rev CDATA #IMPLIED>
 
   <!ELEMENT repo-hooks EMPTY>
   <!ATTLIST repo-hooks in-project CDATA #REQUIRED>
@@ -231,7 +229,26 @@ At most one manifest-server may be specified. The url attribute
 is used to specify the URL of a manifest server, which is an
 XML RPC service.
 
-See the [smart sync documentation](./smart-sync.md) for more details.
+The manifest server should implement the following RPC methods:
+
+    GetApprovedManifest(branch, target)
+
+Return a manifest in which each project is pegged to a known good revision
+for the current branch and target. This is used by repo sync when the
+--smart-sync option is given.
+
+The target to use is defined by environment variables TARGET_PRODUCT
+and TARGET_BUILD_VARIANT. These variables are used to create a string
+of the form $TARGET_PRODUCT-$TARGET_BUILD_VARIANT, e.g. passion-userdebug.
+If one of those variables or both are not present, the program will call
+GetApprovedManifest without the target parameter and the manifest server
+should choose a reasonable default target.
+
+    GetManifest(tag)
+
+Return a manifest in which each project is pegged to the revision at
+the specified tag. This is used by repo sync when the --smart-tag option
+is given.
 
 
 ### Element submanifest
@@ -416,14 +433,6 @@ project.  Same syntax as the corresponding element of `project`.
 Attribute `upstream`: If specified, overrides the upstream of the original
 project.  Same syntax as the corresponding element of `project`.
 
-Attribute `base-rev`: If specified, adds a check against the revision
-to be extended. Manifest parse will fail and give a list of mismatch extends
-if the revisions being extended have changed since base-rev was set.
-Intended for use with layered manifests using hash revisions to prevent
-patch branches hiding newer upstream revisions. Also compares named refs
-like branches or tags but is misleading if branches are used as base-rev.
-Same syntax as the corresponding element of `project`.
-
 ### Element annotation
 
 Zero or more annotation elements may be specified as children of a
@@ -487,14 +496,6 @@ name. Logic otherwise behaves like both are specified.
 Attribute `optional`: Set to true to ignore remove-project elements with no
 matching `project` element.
 
-Attribute `base-rev`: If specified, adds a check against the revision
-to be removed. Manifest parse will fail and give a list of mismatch removes
-if the revisions being removed have changed since base-rev was set.
-Intended for use with layered manifests using hash revisions to prevent
-patch branches hiding newer upstream revisions. Also compares named refs
-like branches or tags but is misleading if branches are used as base-rev.
-Same syntax as the corresponding element of `project`.
-
 ### Element repo-hooks
 
 NB: See the [practical documentation](./repo-hooks.md) for using repo hooks.

docs/release-process.md

@@ -96,9 +96,6 @@ If that tag is valid, then repo will warn and use that commit instead.
 
 If that tag cannot be verified, it gives up and forces the user to resolve.
 
-If env variable `REPO_SKIP_SELF_UPDATE` is defined, this will
-bypass the self update algorithm.
-
 ### Force an update
 
 The `repo selfupdate` command can be used to force an immediate update.

docs/smart-sync.md

@@ -1,129 +0,0 @@
-# repo Smart Syncing
-
-Repo normally fetches & syncs manifests from the same URL specified during
-`repo init`, and that often fetches the latest revisions of all projects in
-the manifest.  This flow works well for tracking and developing with the
-latest code, but often it's desirable to sync to other points.  For example,
-to get a local build matching a specific release or build to reproduce bugs
-reported by other people.
-
-Repo's sync subcommand has support for fetching manifests from a server over
-an XML-RPC connection.  The local configuration and network API are defined by
-repo, but individual projects have to host their own server for the client to
-communicate with.
-
-This process is called "smart syncing" -- instead of blindly fetching the latest
-revision of all projects and getting an unknown state to develop against, the
-client passes a request to the server and is given a matching manifest that
-typically specifies specific commits for every project to fetch a known source
-state.
-
-[TOC]
-
-## Manifest Configuration
-
-The manifest specifies the server to communicate with via the
-the [`<manifest-server>` element](manifest-format.md#Element-manifest_server)
-element.  This is how the client knows what service to talk to.
-
-```xml
-  <manifest-server url="https://example.com/your/manifest/server/url" />
-```
-
-If the URL starts with `persistent-`, then the
-[`git-remote-persistent-https` helper](https://github.com/git/git/blob/HEAD/contrib/persistent-https/README)
-is used to communicate with the server.
-
-## Credentials
-
-Credentials may be specified directly in typical `username:password`
-[URI syntax](https://en.wikipedia.org/wiki/URI#Syntax) in the
-`<manifest-server>` element directly in the manifest.
-
-If they are not specified, `repo sync` has `--manifest-server-username=USERNAME`
-and `--manifest-server-password=PASSWORD` options.
-
-If those are not used, then repo will look up the host in your
-[`~/.netrc`](https://docs.python.org/3/library/netrc.html) database.
-
-When making the connection, cookies matching the host are automatically loaded
-from the cookiejar specified in
-[Git's `http.cookiefile` setting](https://git-scm.com/docs/git-config#Documentation/git-config.txt-httpcookieFile).
-
-## Manifest Server
-
-Unfortunately, there are no public reference implementations.  Google has an
-internal one for Android, but it is written using Google's internal systems,
-so wouldn't be that helpful as a reference.
-
-That said, the XML-RPC API is pretty simple, so any standard XML-RPC server
-example would do.  Google's internal server uses Python's
-[xmlrpc.server.SimpleXMLRPCDispatcher](https://docs.python.org/3/library/xmlrpc.server.html).
-
-## Network API
-
-The manifest server should implement the following RPC methods.
-
-### GetApprovedManifest
-
-> `GetApprovedManifest(branch: str, target: Optional[str]) -> str`
-
-The meaning of `branch` and `target` is not strictly defined.  The server may
-interpret them however it wants.  The recommended interpretation is that the
-`branch` matches the manifest branch, and `target` is an identifier for your
-project that matches something users would build.
-
-See the client section below for how repo typically generates these values.
-
-The server will return a manifest or an error.  If it's an error, repo will
-show the output directly to the user to provide a limited feedback channel.
-
-If the user's request is ambiguous and could match multiple manifests, the
-server has to decide whether to pick one automatically (and silently such that
-the user won't know there were multiple matches), or return an error and force
-the user to be more specific.
-
-### GetManifest
-
-> `GetManifest(tag: str) -> str`
-
-The meaning of `tag` is not strictly defined.  Projects are encouraged to use
-a system where the tag matches a unique source state.
-
-See the client section below for how repo typically generates these values.
-
-The server will return a manifest or an error.  If it's an error, repo will
-show the output directly to the user to provide a limited feedback channel.
-
-If the user's request is ambiguous and could match multiple manifests, the
-server has to decide whether to pick one automatically (and silently such that
-the user won't know there were multiple matches), or return an error and force
-the user to be more specific.
-
-## Client Options
-
-Once repo has successfully downloaded the manifest from the server, it saves a
-copy into `.repo/manifests/smart_sync_override.xml` so users can examine it.
-The next time `repo sync` is run, this file is automatically replaced or removed
-based on the current set of options.
-
-### --smart-sync
-
-Repo will call `GetApprovedManifest(branch[, target])`.
-
-The `branch` is determined by the current manifest branch as specified by
-`--manifest-branch=BRANCH` when running `repo init`.
-
-The `target` is defined by environment variables in the order below.  If none
-of them match, then `target` is omitted.  These variables were decided as they
-match the settings Android build environments automatically setup.
-
-1.  `${SYNC_TARGET}`: If defined, the value is used directly.
-2.  `${TARGET_PRODUCT}-${TARGET_RELEASE}-${TARGET_BUILD_VARIANT}`: If these
-    variables are all defined, then they are merged with `-` and used.
-3.  `${TARGET_PRODUCT}-${TARGET_BUILD_VARIANT}`: If these variables are all
-    defined, then they are merged with `-` and used.
-
-### --smart-tag=TAG
-
-Repo will call `GetManifest(TAG)`.

error.py

@@ -107,8 +107,8 @@ class GitError(RepoError):
         return self.message
 
 
-class GitAuthError(RepoExitError):
+class GitcUnsupportedError(RepoExitError):
-    """Cannot talk to remote due to auth issue."""
+    """Gitc no longer supported."""
 
 
 class UploadError(RepoError):

event_log.py

@@ -168,10 +168,8 @@ class EventLog:
                 f.write("\n")
 
 
-# An integer id that is unique across this invocation of the program, to be set
+# An integer id that is unique across this invocation of the program.
-# by the first Add event. We can't set it here since it results in leaked
+_EVENT_ID = multiprocessing.Value("i", 1)
-# resources (see: https://issues.gerritcodereview.com/353656374).
-_EVENT_ID = None
 
 
 def _NextEventId():
@@ -180,12 +178,6 @@ def _NextEventId():
     Returns:
         A unique, to this invocation of the program, integer id.
     """
-    global _EVENT_ID
-    if _EVENT_ID is None:
-        # There is a small chance of race condition - two parallel processes
-        # setting up _EVENT_ID. However, we expect TASK_COMMAND to happen before
-        # mp kicks in.
-        _EVENT_ID = multiprocessing.Value("i", 1)
     with _EVENT_ID.get_lock():
         val = _EVENT_ID.value
         _EVENT_ID.value += 1

git_command.py

@@ -238,9 +238,9 @@ def _build_env(
             s = p + " " + s
         env["GIT_CONFIG_PARAMETERS"] = s
     if "GIT_ALLOW_PROTOCOL" not in env:
-        env["GIT_ALLOW_PROTOCOL"] = (
+        env[
-            "file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc"
+            "GIT_ALLOW_PROTOCOL"
-        )
+        ] = "file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc"
     env["GIT_HTTP_USER_AGENT"] = user_agent.git
 
     if objdir:
@@ -313,15 +313,12 @@ class GitCommand:
             cwd = None
         command_name = cmdv[0]
         command.append(command_name)
-
+        # Need to use the --progress flag for fetch/clone so output will be
-        if command_name in ("fetch", "clone"):
+        # displayed as by default git only does progress output if stderr is a
-            env["GIT_TERMINAL_PROMPT"] = "0"
+        # TTY.
-            # Need to use the --progress flag for fetch/clone so output will be
+        if sys.stderr.isatty() and command_name in ("fetch", "clone"):
-            # displayed as by default git only does progress output if stderr is
+            if "--progress" not in cmdv and "--quiet" not in cmdv:
-            # a TTY.
+                command.append("--progress")
-            if sys.stderr.isatty():
-                if "--progress" not in cmdv and "--quiet" not in cmdv:
-                    command.append("--progress")
         command.extend(cmdv[1:])
 
         event_log = (
@@ -350,9 +347,9 @@ class GitCommand:
                         "Project": e.project,
                         "CommandName": command_name,
                         "Message": str(e),
-                        "ReturnCode": (
+                        "ReturnCode": str(e.git_rc)
-                            str(e.git_rc) if e.git_rc is not None else None
+                        if e.git_rc is not None
-                        ),
+                        else None,
                         "IsError": log_as_error,
                     }
                 )

git_config.py

@@ -90,20 +90,6 @@ class GitConfig:
 
     @staticmethod
     def _getUserConfig():
-        """Get the user-specific config file.
-
-        Prefers the XDG config location if available, with fallback to
-        ~/.gitconfig
-
-        This matches git behavior:
-        https://git-scm.com/docs/git-config#FILES
-        """
-        xdg_config_home = os.getenv(
-            "XDG_CONFIG_HOME", os.path.expanduser("~/.config")
-        )
-        xdg_config_file = os.path.join(xdg_config_home, "git", "config")
-        if os.path.exists(xdg_config_file):
-            return xdg_config_file
         return os.path.expanduser("~/.gitconfig")
 
     @classmethod

git_superproject.py

@@ -307,6 +307,8 @@ class Superproject:
             )
             return SyncResult(False, False)
 
+        _PrintBetaNotice()
+
         should_exit = True
         if not self._remote_url:
             self._LogWarning(
@@ -450,6 +452,16 @@ class Superproject:
         return UpdateProjectsResult(manifest_path, False)
 
 
+@functools.lru_cache(maxsize=10)
+def _PrintBetaNotice():
+    """Print the notice of beta status."""
+    print(
+        "NOTICE: --use-superproject is in beta; report any issues to the "
+        "address described in `repo version`",
+        file=sys.stderr,
+    )
+
+
 @functools.lru_cache(maxsize=None)
 def _UseSuperprojectFromConfiguration():
     """Returns the user choice of whether to use superproject."""

git_trace2_event_log_base.py

@@ -130,10 +130,10 @@ class BaseEventLog:
             "time": datetime.datetime.now(datetime.timezone.utc).isoformat(),
         }
 
-    def StartEvent(self, argv):
+    def StartEvent(self):
         """Append a 'start' event to the current log."""
         start_event = self._CreateEventDict("start")
-        start_event["argv"] = argv
+        start_event["argv"] = sys.argv
         self._log.append(start_event)
 
     def ExitEvent(self, result):
@@ -159,11 +159,9 @@ class BaseEventLog:
             name: Name of the primary command (ex: repo, git)
             subcommands: List of the sub-commands (ex: version, init, sync)
         """
-        command_event = self._CreateEventDict("cmd_name")
+        command_event = self._CreateEventDict("command")
-        name = f"{name}-"
-        name += "-".join(subcommands)
         command_event["name"] = name
-        command_event["hierarchy"] = name
+        command_event["subcommands"] = subcommands
         self._log.append(command_event)
 
     def LogConfigEvents(self, config, event_dict_name):

main.py

@@ -45,6 +45,7 @@ from command import InteractiveCommand
 from command import MirrorSafeCommand
 from editor import Editor
 from error import DownloadError
+from error import GitcUnsupportedError
 from error import InvalidProjectGroupsError
 from error import ManifestInvalidRevisionError
 from error import ManifestParseError
@@ -307,6 +308,10 @@ class _Repo:
                 outer_client=outer_client,
             )
 
+        if Wrapper().gitc_parse_clientdir(os.getcwd()):
+            logger.error("GITC is not supported.")
+            raise GitcUnsupportedError()
+
         try:
             cmd = self.commands[name](
                 repodir=self.repodir,
@@ -352,7 +357,7 @@ class _Repo:
         start = time.time()
         cmd_event = cmd.event_log.Add(name, event_log.TASK_COMMAND, start)
         cmd.event_log.SetParent(cmd_event)
-        git_trace2_event_log.StartEvent(["repo", name] + argv)
+        git_trace2_event_log.StartEvent()
         git_trace2_event_log.CommandEvent(name="repo", subcommands=[name])
 
         def execute_command_helper():

man/repo-gc.1

@@ -1,43 +0,0 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "December 2024" "repo gc" "Repo Manual"
-.SH NAME
-repo \- repo gc - manual page for repo gc
-.SH SYNOPSIS
-.B repo
-\fI\,gc\/\fR
-.SH DESCRIPTION
-Summary
-.PP
-Cleaning up internal repo state.
-.SH OPTIONS
-.TP
-\fB\-h\fR, \fB\-\-help\fR
-show this help message and exit
-.TP
-\fB\-n\fR, \fB\-\-dry\-run\fR
-do everything except actually delete
-.TP
-\fB\-y\fR, \fB\-\-yes\fR
-answer yes to all safe prompts
-.SS Logging options:
-.TP
-\fB\-v\fR, \fB\-\-verbose\fR
-show all output
-.TP
-\fB\-q\fR, \fB\-\-quiet\fR
-only show errors
-.SS Multi\-manifest options:
-.TP
-\fB\-\-outer\-manifest\fR
-operate starting at the outermost manifest
-.TP
-\fB\-\-no\-outer\-manifest\fR
-do not operate on outer manifests
-.TP
-\fB\-\-this\-manifest\-only\fR
-only operate on this (sub)manifest
-.TP
-\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
-operate on this manifest and its submanifests
-.PP
-Run `repo help gc` to view the detailed manual.

man/repo-init.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "September 2024" "repo init" "Repo Manual"
+.TH REPO "1" "October 2022" "repo init" "Repo Manual"
 .SH NAME
 repo \- repo init - manual page for repo init
 .SH SYNOPSIS
@@ -28,11 +28,6 @@ manifest repository location
 \fB\-b\fR REVISION, \fB\-\-manifest\-branch\fR=\fI\,REVISION\/\fR
 manifest branch or revision (use HEAD for default)
 .TP
-\fB\-\-manifest\-upstream\-branch\fR=\fI\,BRANCH\/\fR
-when a commit is provided to \fB\-\-manifest\-branch\fR, this
-is the name of the git ref in which the commit can be
-found
-.TP
 \fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml
 initial manifest file
 .TP
@@ -168,10 +163,6 @@ The optional \fB\-b\fR argument can be used to select the manifest branch to che
 and use. If no branch is specified, the remote's default branch is used. This is
 equivalent to using \fB\-b\fR HEAD.
 .PP
-The optional \fB\-\-manifest\-upstream\-branch\fR argument can be used when a commit is
-provided to \fB\-\-manifest\-branch\fR (or \fB\-b\fR), to specify the name of the git ref in
-which the commit can be found.
-.PP
 The optional \fB\-m\fR argument can be used to specify an alternate manifest to be
 used. If no manifest is specified, the manifest default.xml will be used.
 .PP

man/repo-manifest.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "December 2024" "repo manifest" "Repo Manual"
+.TH REPO "1" "April 2024" "repo manifest" "Repo Manual"
 .SH NAME
 repo \- repo manifest - manual page for repo manifest
 .SH SYNOPSIS
@@ -192,13 +192,11 @@ CDATA #IMPLIED>
 <!ATTLIST extend\-project remote CDATA #IMPLIED>
 <!ATTLIST extend\-project dest\-branch CDATA #IMPLIED>
 <!ATTLIST extend\-project upstream CDATA #IMPLIED>
-<!ATTLIST extend\-project base\-rev CDATA #IMPLIED>
 .IP
 <!ELEMENT remove\-project EMPTY>
 <!ATTLIST remove\-project name     CDATA #IMPLIED>
 <!ATTLIST remove\-project path     CDATA #IMPLIED>
 <!ATTLIST remove\-project optional CDATA #IMPLIED>
-<!ATTLIST remove\-project base\-rev CDATA #IMPLIED>
 .IP
 <!ELEMENT repo\-hooks EMPTY>
 <!ATTLIST repo\-hooks in\-project CDATA #REQUIRED>
@@ -497,14 +495,6 @@ project. Same syntax as the corresponding element of `project`.
 Attribute `upstream`: If specified, overrides the upstream of the original
 project. Same syntax as the corresponding element of `project`.
 .PP
-Attribute `base\-rev`: If specified, adds a check against the revision to be
-extended. Manifest parse will fail and give a list of mismatch extends if the
-revisions being extended have changed since base\-rev was set. Intended for use
-with layered manifests using hash revisions to prevent patch branches hiding
-newer upstream revisions. Also compares named refs like branches or tags but is
-misleading if branches are used as base\-rev. Same syntax as the corresponding
-element of `project`.
-.PP
 Element annotation
 .PP
 Zero or more annotation elements may be specified as children of a project or
@@ -566,14 +556,6 @@ Logic otherwise behaves like both are specified.
 Attribute `optional`: Set to true to ignore remove\-project elements with no
 matching `project` element.
 .PP
-Attribute `base\-rev`: If specified, adds a check against the revision to be
-removed. Manifest parse will fail and give a list of mismatch removes if the
-revisions being removed have changed since base\-rev was set. Intended for use
-with layered manifests using hash revisions to prevent patch branches hiding
-newer upstream revisions. Also compares named refs like branches or tags but is
-misleading if branches are used as base\-rev. Same syntax as the corresponding
-element of `project`.
-.PP
 Element repo\-hooks
 .PP
 NB: See the [practical documentation](./repo\-hooks.md) for using repo hooks.

man/repo-smartsync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "September 2024" "repo smartsync" "Repo Manual"
+.TH REPO "1" "April 2024" "repo smartsync" "Repo Manual"
 .SH NAME
 repo \- repo smartsync - manual page for repo smartsync
 .SH SYNOPSIS
@@ -47,10 +47,6 @@ force remove projects with uncommitted modifications
 if projects no longer exist in the manifest. WARNING:
 this may cause loss of data
 .TP
-\fB\-\-rebase\fR
-rebase local commits regardless of whether they are
-published
-.TP
 \fB\-l\fR, \fB\-\-local\-only\fR
 only update working tree, don't fetch
 .TP

man/repo-sync.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "September 2024" "repo sync" "Repo Manual"
+.TH REPO "1" "April 2024" "repo sync" "Repo Manual"
 .SH NAME
 repo \- repo sync - manual page for repo sync
 .SH SYNOPSIS
@@ -47,10 +47,6 @@ force remove projects with uncommitted modifications
 if projects no longer exist in the manifest. WARNING:
 this may cause loss of data
 .TP
-\fB\-\-rebase\fR
-rebase local commits regardless of whether they are
-published
-.TP
 \fB\-l\fR, \fB\-\-local\-only\fR
 only update working tree, don't fetch
 .TP

man/repo.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "December 2024" "repo" "Repo Manual"
+.TH REPO "1" "April 2024" "repo" "Repo Manual"
 .SH NAME
 repo \- repository management tool built on top of git
 .SH SYNOPSIS
@@ -79,9 +79,6 @@ Download and checkout a change
 forall
 Run a shell command in each project
 .TP
-gc
-Cleaning up internal repo state.
-.TP
 grep
 Print lines matching a pattern
 .TP

manifest_xml.py

@@ -1014,9 +1014,9 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
     def SetManifestOverride(self, path):
         """Override manifestFile.  The caller must call Unload()"""
-        self._outer_client.manifest.manifestFileOverrides[self.path_prefix] = (
+        self._outer_client.manifest.manifestFileOverrides[
-            path
+            self.path_prefix
-        )
+        ] = path
 
     @property
     def UseLocalManifests(self):
@@ -1445,7 +1445,6 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
 
         repo_hooks_project = None
         enabled_repo_hooks = None
-        failed_revision_changes = []
         for node in itertools.chain(*node_list):
             if node.nodeName == "project":
                 project = self._ParseProject(node)
@@ -1472,7 +1471,6 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                     remote = self._get_remote(node)
                 dest_branch = node.getAttribute("dest-branch")
                 upstream = node.getAttribute("upstream")
-                base_revision = node.getAttribute("base-rev")
 
                 named_projects = self._projects[name]
                 if dest_path and not path and len(named_projects) > 1:
@@ -1486,13 +1484,6 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                     if groups:
                         p.groups.extend(groups)
                     if revision:
-                        if base_revision:
-                            if p.revisionExpr != base_revision:
-                                failed_revision_changes.append(
-                                    "extend-project name %s mismatch base "
-                                    "%s vs revision %s"
-                                    % (name, base_revision, p.revisionExpr)
-                                )
                         p.SetRevision(revision)
 
                     if remote_name:
@@ -1567,7 +1558,6 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
             if node.nodeName == "remove-project":
                 name = node.getAttribute("name")
                 path = node.getAttribute("path")
-                base_revision = node.getAttribute("base-rev")
 
                 # Name or path needed.
                 if not name and not path:
@@ -1581,13 +1571,6 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                 for projname, projects in list(self._projects.items()):
                     for p in projects:
                         if name == projname and not path:
-                            if base_revision:
-                                if p.revisionExpr != base_revision:
-                                    failed_revision_changes.append(
-                                        "remove-project name %s mismatch base "
-                                        "%s vs revision %s"
-                                        % (name, base_revision, p.revisionExpr)
-                                    )
                             del self._paths[p.relpath]
                             if not removed_project:
                                 del self._projects[name]
@@ -1595,17 +1578,6 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                         elif path == p.relpath and (
                             name == projname or not name
                         ):
-                            if base_revision:
-                                if p.revisionExpr != base_revision:
-                                    failed_revision_changes.append(
-                                        "remove-project path %s mismatch base "
-                                        "%s vs revision %s"
-                                        % (
-                                            p.relpath,
-                                            base_revision,
-                                            p.revisionExpr,
-                                        )
-                                    )
                             self._projects[projname].remove(p)
                             del self._paths[p.relpath]
                             removed_project = p.name
@@ -1625,13 +1597,6 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
                         "project: %s" % node.toxml()
                     )
 
-        if failed_revision_changes:
-            raise ManifestParseError(
-                "revision base check failed, rebase patches and update "
-                "base revs for: ",
-                failed_revision_changes,
-            )
-
         # Store repo hooks project information.
         if repo_hooks_project:
             # Store a reference to the Project.
@@ -2056,12 +2021,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         path = path.rstrip("/")
         name = name.rstrip("/")
         relpath = self._JoinRelpath(parent.relpath, path)
-        subprojects = os.path.join(parent.gitdir, "subprojects", f"{path}.git")
+        gitdir = os.path.join(parent.gitdir, "subprojects", "%s.git" % path)
-        modules = os.path.join(parent.gitdir, "modules", path)
-        if platform_utils.isdir(subprojects):
-            gitdir = subprojects
-        else:
-            gitdir = modules
         objdir = os.path.join(
             parent.gitdir, "subproject-objects", "%s.git" % name
         )
@@ -2112,22 +2072,22 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
         # implementation:
         # https://eclipse.googlesource.com/jgit/jgit/+/9110037e3e9461ff4dac22fee84ef3694ed57648/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java#884
         BAD_CODEPOINTS = {
-            "\u200c",  # ZERO WIDTH NON-JOINER
+            "\u200C",  # ZERO WIDTH NON-JOINER
-            "\u200d",  # ZERO WIDTH JOINER
+            "\u200D",  # ZERO WIDTH JOINER
-            "\u200e",  # LEFT-TO-RIGHT MARK
+            "\u200E",  # LEFT-TO-RIGHT MARK
-            "\u200f",  # RIGHT-TO-LEFT MARK
+            "\u200F",  # RIGHT-TO-LEFT MARK
-            "\u202a",  # LEFT-TO-RIGHT EMBEDDING
+            "\u202A",  # LEFT-TO-RIGHT EMBEDDING
-            "\u202b",  # RIGHT-TO-LEFT EMBEDDING
+            "\u202B",  # RIGHT-TO-LEFT EMBEDDING
-            "\u202c",  # POP DIRECTIONAL FORMATTING
+            "\u202C",  # POP DIRECTIONAL FORMATTING
-            "\u202d",  # LEFT-TO-RIGHT OVERRIDE
+            "\u202D",  # LEFT-TO-RIGHT OVERRIDE
-            "\u202e",  # RIGHT-TO-LEFT OVERRIDE
+            "\u202E",  # RIGHT-TO-LEFT OVERRIDE
-            "\u206a",  # INHIBIT SYMMETRIC SWAPPING
+            "\u206A",  # INHIBIT SYMMETRIC SWAPPING
-            "\u206b",  # ACTIVATE SYMMETRIC SWAPPING
+            "\u206B",  # ACTIVATE SYMMETRIC SWAPPING
-            "\u206c",  # INHIBIT ARABIC FORM SHAPING
+            "\u206C",  # INHIBIT ARABIC FORM SHAPING
-            "\u206d",  # ACTIVATE ARABIC FORM SHAPING
+            "\u206D",  # ACTIVATE ARABIC FORM SHAPING
-            "\u206e",  # NATIONAL DIGIT SHAPES
+            "\u206E",  # NATIONAL DIGIT SHAPES
-            "\u206f",  # NOMINAL DIGIT SHAPES
+            "\u206F",  # NOMINAL DIGIT SHAPES
-            "\ufeff",  # ZERO WIDTH NO-BREAK SPACE
+            "\uFEFF",  # ZERO WIDTH NO-BREAK SPACE
         }
         if BAD_CODEPOINTS & path_codepoints:
             # This message is more expansive than reality, but should be fine.

pager.py

@@ -40,7 +40,7 @@ def RunPager(globalConfig):
 
 
 def TerminatePager():
-    global pager_process
+    global pager_process, old_stdout, old_stderr
     if pager_process:
         sys.stdout.flush()
         sys.stderr.flush()

platform_utils.py

@@ -156,12 +156,6 @@ def remove(path, missing_ok=False):
                 os.rmdir(longpath)
             else:
                 os.remove(longpath)
-        elif (
-            e.errno == errno.EROFS
-            and missing_ok
-            and not os.path.exists(longpath)
-        ):
-            pass
         elif missing_ok and e.errno == errno.ENOENT:
             pass
         else:

progress.py

@@ -100,7 +100,6 @@ class Progress:
         self._show = not delay
         self._units = units
         self._elide = elide and _TTY
-        self._quiet = quiet
 
         # Only show the active jobs section if we run more than one in parallel.
         self._show_jobs = False
@@ -115,7 +114,13 @@ class Progress:
         )
         self._update_thread.daemon = True
 
-        if not quiet and show_elapsed:
+        # When quiet, never show any output.  It's a bit hacky, but reusing the
+        # existing logic that delays initial output keeps the rest of the class
+        # clean.  Basically we set the start time to years in the future.
+        if quiet:
+            self._show = False
+            self._start += 2**32
+        elif show_elapsed:
             self._update_thread.start()
 
     def _update_loop(self):
@@ -155,7 +160,7 @@ class Progress:
             msg = self._last_msg
         self._last_msg = msg
 
-        if not _TTY or IsTraceToStderr() or self._quiet:
+        if not _TTY or IsTraceToStderr():
             return
 
         elapsed_sec = time.time() - self._start
@@ -197,7 +202,7 @@ class Progress:
 
     def end(self):
         self._update_event.set()
-        if not _TTY or IsTraceToStderr() or self._quiet:
+        if not _TTY or IsTraceToStderr() or not self._show:
             return
 
         duration = duration_str(time.time() - self._start)

project.py

@@ -32,7 +32,6 @@ import urllib.parse
 
 from color import Coloring
 from error import DownloadError
-from error import GitAuthError
 from error import GitError
 from error import ManifestInvalidPathError
 from error import ManifestInvalidRevisionError
@@ -576,6 +575,7 @@ class Project:
         dest_branch=None,
         optimized_fetch=False,
         retry_fetches=0,
+        old_revision=None,
     ):
         """Init a Project object.
 
@@ -608,6 +608,7 @@ class Project:
                 only fetch from the remote if the sha1 is not present locally.
             retry_fetches: Retry remote fetches n times upon receiving transient
                 error with exponential backoff and jitter.
+            old_revision: saved git commit id for open GITC projects.
         """
         self.client = self.manifest = manifest
         self.name = name
@@ -637,15 +638,12 @@ class Project:
         self.linkfiles = []
         self.annotations = []
         self.dest_branch = dest_branch
+        self.old_revision = old_revision
 
         # This will be filled in if a project is later identified to be the
         # project containing repo hooks.
         self.enabled_repo_hooks = []
 
-        # This will be updated later if the project has submodules and
-        # if they will be synced.
-        self.has_subprojects = False
-
     def RelPath(self, local=True):
         """Return the path for the project relative to a manifest.
 
@@ -1564,11 +1562,6 @@ class Project:
             return
 
         self._InitWorkTree(force_sync=force_sync, submodules=submodules)
-        # TODO(https://git-scm.com/docs/git-worktree#_bugs): Re-evaluate if
-        # submodules can be init when using worktrees once its support is
-        # complete.
-        if self.has_subprojects and not self.use_git_worktrees:
-            self._InitSubmodules()
         all_refs = self.bare_ref.all
         self.CleanPublishedCache(all_refs)
         revid = self.GetRevisionId(all_refs)
@@ -1701,8 +1694,6 @@ class Project:
                             project=self.name,
                         )
                     )
-                    return
-                syncbuf.later1(self, _doff, not verbose)
                 return
             elif pub == head:
                 # All published commits are merged, and thus we are a
@@ -2197,27 +2188,24 @@ class Project:
 
         def get_submodules(gitdir, rev):
             # Parse .gitmodules for submodule sub_paths and sub_urls.
-            sub_paths, sub_urls, sub_shallows = parse_gitmodules(gitdir, rev)
+            sub_paths, sub_urls = parse_gitmodules(gitdir, rev)
             if not sub_paths:
                 return []
             # Run `git ls-tree` to read SHAs of submodule object, which happen
             # to be revision of submodule repository.
             sub_revs = git_ls_tree(gitdir, rev, sub_paths)
             submodules = []
-            for sub_path, sub_url, sub_shallow in zip(
+            for sub_path, sub_url in zip(sub_paths, sub_urls):
-                sub_paths, sub_urls, sub_shallows
-            ):
                 try:
                     sub_rev = sub_revs[sub_path]
                 except KeyError:
                     # Ignore non-exist submodules.
                     continue
-                submodules.append((sub_rev, sub_path, sub_url, sub_shallow))
+                submodules.append((sub_rev, sub_path, sub_url))
             return submodules
 
         re_path = re.compile(r"^submodule\.(.+)\.path=(.*)$")
         re_url = re.compile(r"^submodule\.(.+)\.url=(.*)$")
-        re_shallow = re.compile(r"^submodule\.(.+)\.shallow=(.*)$")
 
         def parse_gitmodules(gitdir, rev):
             cmd = ["cat-file", "blob", "%s:.gitmodules" % rev]
@@ -2231,9 +2219,9 @@ class Project:
                     gitdir=gitdir,
                 )
             except GitError:
-                return [], [], []
+                return [], []
             if p.Wait() != 0:
-                return [], [], []
+                return [], []
 
             gitmodules_lines = []
             fd, temp_gitmodules_path = tempfile.mkstemp()
@@ -2250,17 +2238,16 @@ class Project:
                     gitdir=gitdir,
                 )
                 if p.Wait() != 0:
-                    return [], [], []
+                    return [], []
                 gitmodules_lines = p.stdout.split("\n")
             except GitError:
-                return [], [], []
+                return [], []
             finally:
                 platform_utils.remove(temp_gitmodules_path)
 
             names = set()
             paths = {}
             urls = {}
-            shallows = {}
             for line in gitmodules_lines:
                 if not line:
                     continue
@@ -2274,16 +2261,10 @@ class Project:
                     names.add(m.group(1))
                     urls[m.group(1)] = m.group(2)
                     continue
-                m = re_shallow.match(line)
-                if m:
-                    names.add(m.group(1))
-                    shallows[m.group(1)] = m.group(2)
-                    continue
             names = sorted(names)
             return (
                 [paths.get(name, "") for name in names],
                 [urls.get(name, "") for name in names],
-                [shallows.get(name, "") for name in names],
             )
 
         def git_ls_tree(gitdir, rev, paths):
@@ -2312,9 +2293,7 @@ class Project:
 
         try:
             rev = self.GetRevisionId()
-        except (GitError, ManifestInvalidRevisionError):
+        except GitError:
-            # The git repo may be outdated (i.e. not fetched yet) and querying
-            # its submodules using the revision may not work; so return here.
             return []
         return get_submodules(self.gitdir, rev)
 
@@ -2324,7 +2303,7 @@ class Project:
             # If git repo does not exist yet, querying its submodules will
             # mess up its states; so return here.
             return result
-        for rev, path, url, shallow in self._GetSubmodules():
+        for rev, path, url in self._GetSubmodules():
             name = self.manifest.GetSubprojectName(self, path)
             (
                 relpath,
@@ -2346,7 +2325,6 @@ class Project:
                 review=self.remote.review,
                 revision=self.remote.revision,
             )
-            clone_depth = 1 if shallow.lower() == "true" else None
             subproject = Project(
                 manifest=self.manifest,
                 name=name,
@@ -2363,13 +2341,10 @@ class Project:
                 sync_s=self.sync_s,
                 sync_tags=self.sync_tags,
                 parent=self,
-                clone_depth=clone_depth,
                 is_derived=True,
             )
             result.append(subproject)
             result.extend(subproject.GetDerivedSubprojects())
-        if result:
-            self.has_subprojects = True
         return result
 
     def EnableRepositoryExtension(self, key, value="true", version=1):
@@ -2418,25 +2393,26 @@ class Project:
         try:
             # if revision (sha or tag) is not present then following function
             # throws an error.
-            revs = [f"{self.revisionExpr}^0"]
-            upstream_rev = None
-            if self.upstream:
-                upstream_rev = self.GetRemote().ToLocal(self.upstream)
-                revs.append(upstream_rev)
-
             self.bare_git.rev_list(
                 "-1",
                 "--missing=allow-any",
-                *revs,
+                "%s^0" % self.revisionExpr,
                 "--",
                 log_as_error=False,
             )
-
             if self.upstream:
+                rev = self.GetRemote().ToLocal(self.upstream)
+                self.bare_git.rev_list(
+                    "-1",
+                    "--missing=allow-any",
+                    "%s^0" % rev,
+                    "--",
+                    log_as_error=False,
+                )
                 self.bare_git.merge_base(
                     "--is-ancestor",
                     self.revisionExpr,
-                    upstream_rev,
+                    rev,
                     log_as_error=False,
                 )
             return True
@@ -2686,10 +2662,7 @@ class Project:
             # TODO(b/360889369#comment24): git may gc commits incorrectly.
             # Until the root cause is fixed, retry fetch with --refetch which
             # will bring the repository into a good state.
-            elif gitcmd.stdout and (
+            elif gitcmd.stdout and "could not parse commit" in gitcmd.stdout:
-                "could not parse commit" in gitcmd.stdout
-                or "unable to parse commit" in gitcmd.stdout
-            ):
                 cmd.insert(1, "--refetch")
                 print(
                     "could not parse commit error, retrying with refetch",
@@ -2722,47 +2695,12 @@ class Project:
                 )
                 # Continue right away so we don't sleep as we shouldn't need to.
                 continue
-            elif (
-                ret == 128
-                and gitcmd.stdout
-                and "fatal: could not read Username" in gitcmd.stdout
-            ):
-                # User needs to be authenticated, and Git wants to prompt for
-                # username and password.
-                print(
-                    "git requires authentication, but repo cannot perform "
-                    "interactive authentication. Check git credentials.",
-                    file=output_redir,
-                )
-                break
-            elif (
-                ret == 128
-                and gitcmd.stdout
-                and "remote helper 'sso' aborted session" in gitcmd.stdout
-            ):
-                # User needs to be authenticated, and Git wants to prompt for
-                # username and password.
-                print(
-                    "git requires authentication, but repo cannot perform "
-                    "interactive authentication.",
-                    file=output_redir,
-                )
-                raise GitAuthError(gitcmd.stdout)
-                break
             elif current_branch_only and is_sha1 and ret == 128:
                 # Exit code 128 means "couldn't find the ref you asked for"; if
                 # we're in sha1 mode, we just tried sync'ing from the upstream
                 # field; it doesn't exist, thus abort the optimization attempt
                 # and do a full sync.
                 break
-            elif depth and is_sha1 and ret == 1:
-                # In sha1 mode, when depth is enabled, syncing the revision
-                # from upstream may not work because some servers only allow
-                # fetching named refs. Fetching a specific sha1 may result
-                # in an error like 'server does not allow request for
-                # unadvertised object'. In this case, attempt a full sync
-                # without depth.
-                break
             elif ret < 0:
                 # Git died with a signal, exit immediately.
                 break
@@ -2883,14 +2821,7 @@ class Project:
 
         # We do not use curl's --retry option since it generally doesn't
         # actually retry anything; code 18 for example, it will not retry on.
-        cmd = [
+        cmd = ["curl", "--fail", "--output", tmpPath, "--netrc", "--location"]
-            "curl",
-            "--fail",
-            "--output",
-            tmpPath,
-            "--netrc-optional",
-            "--location",
-        ]
         if quiet:
             cmd += ["--silent", "--show-error"]
         if os.path.exists(tmpPath):
@@ -3035,17 +2966,6 @@ class Project:
                 project=self.name,
             )
 
-    def _InitSubmodules(self, quiet=True):
-        """Initialize the submodules for the project."""
-        cmd = ["submodule", "init"]
-        if quiet:
-            cmd.append("-q")
-        if GitCommand(self, cmd).Wait() != 0:
-            raise GitError(
-                f"{self.name} submodule init",
-                project=self.name,
-            )
-
     def _Rebase(self, upstream, onto=None):
         cmd = ["rebase"]
         if onto is not None:
@@ -3421,29 +3341,24 @@ class Project:
             setting = fp.read()
             assert setting.startswith("gitdir:")
             git_worktree_path = setting.split(":", 1)[1].strip()
-
+        # Some platforms (e.g. Windows) won't let us update dotgit in situ
-        # `gitdir` maybe be either relative or absolute depending on the
+        # because of file permissions.  Delete it and recreate it from scratch
-        # behavior of the local copy of git, so only convert the path to
+        # to avoid.
-        # relative if it needs to be converted.
+        platform_utils.remove(dotgit)
-        if os.path.isabs(git_worktree_path):
+        # Use relative path from checkout->worktree & maintain Unix line endings
-            # Some platforms (e.g. Windows) won't let us update dotgit in situ
+        # on all OS's to match git behavior.
-            # because of file permissions.  Delete it and recreate it from
+        with open(dotgit, "w", newline="\n") as fp:
-            # scratch to avoid.
+            print(
-            platform_utils.remove(dotgit)
+                "gitdir:",
-            # Use relative path from checkout->worktree & maintain Unix line
+                os.path.relpath(git_worktree_path, self.worktree),
-            # endings on all OS's to match git behavior.
+                file=fp,
-            with open(dotgit, "w", newline="\n") as fp:
+            )
-                print(
+        # Use relative path from worktree->checkout & maintain Unix line endings
-                    "gitdir:",
+        # on all OS's to match git behavior.
-                    os.path.relpath(git_worktree_path, self.worktree),
+        with open(
-                    file=fp,
+            os.path.join(git_worktree_path, "gitdir"), "w", newline="\n"
-                )
+        ) as fp:
-            # Use relative path from worktree->checkout & maintain Unix line
+            print(os.path.relpath(dotgit, git_worktree_path), file=fp)
-            # endings on all OS's to match git behavior.
-            with open(
-                os.path.join(git_worktree_path, "gitdir"), "w", newline="\n"
-            ) as fp:
-                print(os.path.relpath(dotgit, git_worktree_path), file=fp)
 
         self._InitMRef()
 
@@ -3464,11 +3379,6 @@ class Project:
         """
         dotgit = os.path.join(self.worktree, ".git")
 
-        # If bare checkout of the submodule is stored under the subproject dir,
-        # migrate it.
-        if self.parent:
-            self._MigrateOldSubmoduleDir()
-
         # If using an old layout style (a directory), migrate it.
         if not platform_utils.islink(dotgit) and platform_utils.isdir(dotgit):
             self._MigrateOldWorkTreeGitDir(dotgit, project=self.name)
@@ -3479,76 +3389,34 @@ class Project:
                 self._InitGitWorktree()
                 self._CopyAndLinkFiles()
         else:
-            # Remove old directory symbolic links for submodules.
-            if self.parent and platform_utils.islink(dotgit):
-                platform_utils.remove(dotgit)
-                init_dotgit = True
-
             if not init_dotgit:
                 # See if the project has changed.
-                self._removeBadGitDirLink(dotgit)
+                if os.path.realpath(self.gitdir) != os.path.realpath(dotgit):
+                    platform_utils.remove(dotgit)
 
             if init_dotgit or not os.path.exists(dotgit):
-                self._createDotGit(dotgit)
+                os.makedirs(self.worktree, exist_ok=True)
+                platform_utils.symlink(
+                    os.path.relpath(self.gitdir, self.worktree), dotgit
+                )
 
             if init_dotgit:
                 _lwrite(
-                    os.path.join(self.gitdir, HEAD), f"{self.GetRevisionId()}\n"
+                    os.path.join(dotgit, HEAD), "%s\n" % self.GetRevisionId()
                 )
 
                 # Finish checking out the worktree.
                 cmd = ["read-tree", "--reset", "-u", "-v", HEAD]
-                try:
+                if GitCommand(self, cmd).Wait() != 0:
-                    if GitCommand(self, cmd).Wait() != 0:
+                    raise GitError(
-                        raise GitError(
+                        "Cannot initialize work tree for " + self.name,
-                            "Cannot initialize work tree for " + self.name,
+                        project=self.name,
-                            project=self.name,
+                    )
-                        )
-                except Exception as e:
-                    # Something went wrong with read-tree (perhaps fetching
-                    # missing blobs), so remove .git to avoid half initialized
-                    # workspace from which repo can't recover on its own.
-                    platform_utils.remove(dotgit)
-                    raise e
 
                 if submodules:
                     self._SyncSubmodules(quiet=True)
                 self._CopyAndLinkFiles()
 
-    def _createDotGit(self, dotgit):
-        """Initialize .git path.
-
-        For submodule projects, create a '.git' file using the gitfile
-        mechanism, and for the rest, create a symbolic link.
-        """
-        os.makedirs(self.worktree, exist_ok=True)
-        if self.parent:
-            _lwrite(
-                dotgit,
-                f"gitdir: {os.path.relpath(self.gitdir, self.worktree)}\n",
-            )
-        else:
-            platform_utils.symlink(
-                os.path.relpath(self.gitdir, self.worktree), dotgit
-            )
-
-    def _removeBadGitDirLink(self, dotgit):
-        """Verify .git is initialized correctly, otherwise delete it."""
-        if self.parent and os.path.isfile(dotgit):
-            with open(dotgit) as fp:
-                setting = fp.read()
-            if not setting.startswith("gitdir:"):
-                raise GitError(
-                    f"'.git' in {self.worktree} must start with 'gitdir:'",
-                    project=self.name,
-                )
-            gitdir = setting.split(":", 1)[1].strip()
-            dotgit_path = os.path.normpath(os.path.join(self.worktree, gitdir))
-        else:
-            dotgit_path = os.path.realpath(dotgit)
-        if os.path.realpath(self.gitdir) != dotgit_path:
-            platform_utils.remove(dotgit)
-
     @classmethod
     def _MigrateOldWorkTreeGitDir(cls, dotgit, project=None):
         """Migrate the old worktree .git/ dir style to a symlink.
@@ -3637,28 +3505,6 @@ class Project:
             dotgit,
         )
 
-    def _MigrateOldSubmoduleDir(self):
-        """Move the old bare checkout in 'subprojects' to 'modules'
-        as bare checkouts of submodules are now in 'modules' dir.
-        """
-        subprojects = os.path.join(self.parent.gitdir, "subprojects")
-        if not platform_utils.isdir(subprojects):
-            return
-
-        modules = os.path.join(self.parent.gitdir, "modules")
-        old = self.gitdir
-        new = os.path.splitext(self.gitdir.replace(subprojects, modules))[0]
-
-        if all(map(platform_utils.isdir, [old, new])):
-            platform_utils.rmtree(old, ignore_errors=True)
-        else:
-            os.makedirs(modules, exist_ok=True)
-            platform_utils.rename(old, new)
-        self.gitdir = new
-        self.UpdatePaths(self.relpath, self.worktree, self.gitdir, self.objdir)
-        if platform_utils.isdir(subprojects) and not os.listdir(subprojects):
-            platform_utils.rmtree(subprojects, ignore_errors=True)
-
     def _get_symlink_error_message(self):
         if platform_utils.isWindows():
             return (

pyproject.toml

@@ -16,8 +16,3 @@
 line-length = 80
 # NB: Keep in sync with tox.ini.
 target-version = ['py36', 'py37', 'py38', 'py39', 'py310', 'py311'] #, 'py312'
-
-[tool.pytest.ini_options]
-markers = """
-    skip_cq: Skip tests in the CQ. Should be rarely used!
-"""

release/util.py

@@ -16,7 +16,6 @@
 
 import os
 import re
-import shlex
 import subprocess
 import sys
 
@@ -36,7 +35,12 @@ KEYID_ECC = "E1F9040D7A3F6DAFAC897CD3D3B95DA243E48A39"
 
 def cmdstr(cmd):
     """Get a nicely quoted shell command."""
-    return " ".join(shlex.quote(x) for x in cmd)
+    ret = []
+    for arg in cmd:
+        if not re.match(r"^[a-zA-Z0-9/_.=-]+$", arg):
+            arg = f'"{arg}"'
+        ret.append(arg)
+    return " ".join(ret)
 
 
 def run(opts, cmd, check=True, **kwargs):

repo

@@ -27,7 +27,6 @@ import platform
 import shlex
 import subprocess
 import sys
-from typing import NamedTuple
 
 
 # These should never be newer than the main.py version since this needs to be a
@@ -57,14 +56,9 @@ class Trace:
 trace = Trace()
 
 
-def cmdstr(cmd):
-    """Get a nicely quoted shell command."""
-    return " ".join(shlex.quote(x) for x in cmd)
-
-
 def exec_command(cmd):
     """Execute |cmd| or return None on failure."""
-    trace.print(":", cmdstr(cmd))
+    trace.print(":", " ".join(cmd))
     try:
         if platform.system() == "Windows":
             ret = subprocess.call(cmd)
@@ -130,7 +124,7 @@ if not REPO_REV:
 BUG_URL = "https://issues.gerritcodereview.com/issues/new?component=1370071"
 
 # increment this whenever we make important changes to this script
-VERSION = (2, 54)
+VERSION = (2, 45)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (2, 3)
@@ -221,8 +215,11 @@ repodir = ".repo"  # name of repo's private directory
 S_repo = "repo"  # special repo repository
 S_manifests = "manifests"  # special manifest repository
 REPO_MAIN = S_repo + "/main.py"  # main script
+GITC_CONFIG_FILE = "/gitc/.config"
+GITC_FS_ROOT_DIR = "/gitc/manifest-rw/"
 
 
+import collections
 import errno
 import json
 import optparse
@@ -238,9 +235,12 @@ home_dot_repo = os.path.join(repo_config_dir, ".repoconfig")
 gpg_dir = os.path.join(home_dot_repo, "gnupg")
 
 
-def GetParser():
+def GetParser(gitc_init=False):
     """Setup the CLI parser."""
-    usage = "repo init [options] [-u] url"
+    if gitc_init:
+        sys.exit("repo: fatal: GITC not supported.")
+    else:
+        usage = "repo init [options] [-u] url"
 
     parser = optparse.OptionParser(usage=usage)
     InitParser(parser)
@@ -282,12 +282,6 @@ def InitParser(parser):
         metavar="REVISION",
         help="manifest branch or revision (use HEAD for default)",
     )
-    group.add_option(
-        "--manifest-upstream-branch",
-        help="when a commit is provided to --manifest-branch, this "
-        "is the name of the git ref in which the commit can be found",
-        metavar="BRANCH",
-    )
     group.add_option(
         "-m",
         "--manifest-name",
@@ -487,6 +481,16 @@ def InitParser(parser):
     return parser
 
 
+# This is a poor replacement for subprocess.run until we require Python 3.6+.
+RunResult = collections.namedtuple(
+    "RunResult", ("returncode", "stdout", "stderr")
+)
+
+
+class RunError(Exception):
+    """Error when running a command failed."""
+
+
 def run_command(cmd, **kwargs):
     """Run |cmd| and return its output."""
     check = kwargs.pop("check", False)
@@ -511,7 +515,7 @@ def run_command(cmd, **kwargs):
     # Run & package the results.
     proc = subprocess.Popen(cmd, **kwargs)
     (stdout, stderr) = proc.communicate(input=cmd_input)
-    dbg = ": " + cmdstr(cmd)
+    dbg = ": " + " ".join(cmd)
     if cmd_input is not None:
         dbg += " 0<|"
     if stdout == subprocess.PIPE:
@@ -521,9 +525,7 @@ def run_command(cmd, **kwargs):
     elif stderr == subprocess.STDOUT:
         dbg += " 2>&1"
     trace.print(dbg)
-    ret = subprocess.CompletedProcess(
+    ret = RunResult(proc.returncode, decode(stdout), decode(stderr))
-        cmd, proc.returncode, decode(stdout), decode(stderr)
-    )
 
     # If things failed, print useful debugging output.
     if check and ret.returncode:
@@ -544,13 +546,56 @@ def run_command(cmd, **kwargs):
 
         _print_output("stdout", ret.stdout)
         _print_output("stderr", ret.stderr)
-        # This will raise subprocess.CalledProcessError for us.
+        raise RunError(ret)
-        ret.check_returncode()
 
     return ret
 
 
+_gitc_manifest_dir = None
+
+
+def get_gitc_manifest_dir():
+    global _gitc_manifest_dir
+    if _gitc_manifest_dir is None:
+        _gitc_manifest_dir = ""
+        try:
+            with open(GITC_CONFIG_FILE) as gitc_config:
+                for line in gitc_config:
+                    match = re.match("gitc_dir=(?P<gitc_manifest_dir>.*)", line)
+                    if match:
+                        _gitc_manifest_dir = match.group("gitc_manifest_dir")
+        except OSError:
+            pass
+    return _gitc_manifest_dir
+
+
+def gitc_parse_clientdir(gitc_fs_path):
+    """Parse a path in the GITC FS and return its client name.
+
+    Args:
+        gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR.
+
+    Returns:
+        The GITC client name.
+    """
+    if gitc_fs_path == GITC_FS_ROOT_DIR:
+        return None
+    if not gitc_fs_path.startswith(GITC_FS_ROOT_DIR):
+        manifest_dir = get_gitc_manifest_dir()
+        if manifest_dir == "":
+            return None
+        if manifest_dir[-1] != "/":
+            manifest_dir += "/"
+        if gitc_fs_path == manifest_dir:
+            return None
+        if not gitc_fs_path.startswith(manifest_dir):
+            return None
+        return gitc_fs_path.split(manifest_dir)[1].split("/")[0]
+    return gitc_fs_path.split(GITC_FS_ROOT_DIR)[1].split("/")[0]
+
+
 class CloneFailure(Exception):
+
     """Indicate the remote clone of repo itself failed."""
 
 
@@ -587,9 +632,9 @@ def check_repo_rev(dst, rev, repo_verify=True, quiet=False):
     return (remote_ref, rev)
 
 
-def _Init(args):
+def _Init(args, gitc_init=False):
     """Installs repo by cloning it over the network."""
-    parser = GetParser()
+    parser = GetParser(gitc_init=gitc_init)
     opt, args = parser.parse_args(args)
     if args:
         if not opt.manifest_url:
@@ -669,20 +714,15 @@ def run_git(*args, **kwargs):
             file=sys.stderr,
         )
         sys.exit(1)
-    except subprocess.CalledProcessError:
+    except RunError:
         raise CloneFailure()
 
 
-class GitVersion(NamedTuple):
+# The git version info broken down into components for easy analysis.
-    """The git version info broken down into components for easy analysis.
+# Similar to Python's sys.version_info.
-
+GitVersion = collections.namedtuple(
-    Similar to Python's sys.version_info.
+    "GitVersion", ("major", "minor", "micro", "full")
-    """
+)
-
-    major: int
-    minor: int
-    micro: int
-    full: int
 
 
 def ParseGitVersion(ver_str=None):
@@ -848,11 +888,10 @@ def _GetRepoConfig(name):
         return None
     else:
         print(
-            f"repo: error: git {cmdstr(cmd)} failed:\n{ret.stderr}",
+            f"repo: error: git {' '.join(cmd)} failed:\n{ret.stderr}",
             file=sys.stderr,
         )
-        # This will raise subprocess.CalledProcessError for us.
+        raise RunError()
-        ret.check_returncode()
 
 
 def _InitHttp():
@@ -1119,7 +1158,7 @@ class _Options:
 def _ExpandAlias(name):
     """Look up user registered aliases."""
     # We don't resolve aliases for existing subcommands.  This matches git.
-    if name in {"help", "init"}:
+    if name in {"gitc-init", "help", "init"}:
         return name, []
 
     alias = _GetRepoConfig(f"alias.{name}")
@@ -1247,6 +1286,10 @@ class Requirements:
 
 
 def _Usage():
+    gitc_usage = ""
+    if get_gitc_manifest_dir():
+        gitc_usage = "  gitc-init Initialize a GITC Client.\n"
+
     print(
         """usage: repo COMMAND [ARGS]
 
@@ -1255,7 +1298,9 @@ repo is not yet installed.  Use "repo init" to install it here.
 The most commonly used repo commands are:
 
   init      Install repo in the current working directory
-  help      Display detailed help on a command
+"""
+        + gitc_usage
+        + """  help      Display detailed help on a command
 
 For access to the full online help, install repo ("repo init").
 """
@@ -1266,8 +1311,8 @@ For access to the full online help, install repo ("repo init").
 
 def _Help(args):
     if args:
-        if args[0] in {"init"}:
+        if args[0] in {"init", "gitc-init"}:
-            parser = GetParser()
+            parser = GetParser(gitc_init=args[0] == "gitc-init")
             parser.print_help()
             sys.exit(0)
         else:
@@ -1284,11 +1329,10 @@ def _Help(args):
 
 def _Version():
     """Show version information."""
-    git_version = ParseGitVersion()
     print("<repo not installed>")
     print(f"repo launcher version {'.'.join(str(x) for x in VERSION)}")
     print(f"       (from {__file__})")
-    print(f"git {git_version.full}" if git_version else "git not installed")
+    print(f"git {ParseGitVersion().full}")
     print(f"Python {sys.version}")
     uname = platform.uname()
     print(f"OS {uname.system} {uname.release} ({uname.version})")
@@ -1321,11 +1365,11 @@ def _RunSelf(wrapper_path):
     my_main = os.path.join(my_dir, "main.py")
     my_git = os.path.join(my_dir, ".git")
 
-    if os.path.isfile(my_main):
+    if os.path.isfile(my_main) and os.path.isdir(my_git):
         for name in ["git_config.py", "project.py", "subcmds"]:
             if not os.path.exists(os.path.join(my_dir, name)):
                 return None, None
-        return my_main, my_git if os.path.isdir(my_git) else None
+        return my_main, my_git
     return None, None
 
 
@@ -1356,11 +1400,23 @@ def main(orig_args):
     # We run this early as we run some git commands ourselves.
     SetGitTrace2ParentSid()
 
-    repo_main, rel_repo_dir = _FindRepo()
+    repo_main, rel_repo_dir = None, None
+    # Don't use the local repo copy, make sure to switch to the gitc client first.
+    if cmd != "gitc-init":
+        repo_main, rel_repo_dir = _FindRepo()
 
     wrapper_path = os.path.abspath(__file__)
     my_main, my_git = _RunSelf(wrapper_path)
 
+    cwd = os.getcwd()
+    if get_gitc_manifest_dir() and cwd.startswith(get_gitc_manifest_dir()):
+        print(
+            "error: repo cannot be used in the GITC local manifest directory."
+            "\nIf you want to work on this GITC client please rerun this "
+            "command from the corresponding client under /gitc/",
+            file=sys.stderr,
+        )
+        sys.exit(1)
     if not repo_main:
         # Only expand aliases here since we'll be parsing the CLI ourselves.
         # If we had repo_main, alias expansion would happen in main.py.
@@ -1375,11 +1431,11 @@ def main(orig_args):
             _Version()
         if not cmd:
             _NotInstalled()
-        if cmd == "init":
+        if cmd == "init" or cmd == "gitc-init":
             if my_git:
                 _SetDefaultsTo(my_git)
             try:
-                _Init(args)
+                _Init(args, gitc_init=(cmd == "gitc-init"))
             except CloneFailure:
                 path = os.path.join(repodir, S_repo)
                 print(

run_tests

@@ -15,57 +15,16 @@
 
 """Wrapper to run linters and pytest with the right settings."""
 
-import functools
 import os
 import subprocess
 import sys
-from typing import List
+
+import pytest
 
 
 ROOT_DIR = os.path.dirname(os.path.realpath(__file__))
 
 
-@functools.lru_cache()
-def is_ci() -> bool:
-    """Whether we're running in our CI system."""
-    return os.getenv("LUCI_CQ") == "yes"
-
-
-def run_pytest(argv: List[str]) -> int:
-    """Returns the exit code from pytest."""
-    if is_ci():
-        argv = ["-m", "not skip_cq"] + argv
-
-    return subprocess.run(
-        [sys.executable, "-m", "pytest"] + argv,
-        check=False,
-        cwd=ROOT_DIR,
-    ).returncode
-
-
-def run_pytest_py38(argv: List[str]) -> int:
-    """Returns the exit code from pytest under Python 3.8."""
-    if is_ci():
-        argv = ["-m", "not skip_cq"] + argv
-
-    try:
-        return subprocess.run(
-            [
-                "vpython3",
-                "-vpython-spec",
-                "run_tests.vpython3.8",
-                "-m",
-                "pytest",
-            ]
-            + argv,
-            check=False,
-            cwd=ROOT_DIR,
-        ).returncode
-    except FileNotFoundError:
-        # Skip if the user doesn't have vpython from depot_tools.
-        return 0
-
-
 def run_black():
     """Returns the exit code from black."""
     # Black by default only matches .py files.  We have to list standalone
@@ -79,40 +38,32 @@ def run_black():
     return subprocess.run(
         [sys.executable, "-m", "black", "--check", ROOT_DIR] + extra_programs,
         check=False,
-        cwd=ROOT_DIR,
     ).returncode
 
 
 def run_flake8():
     """Returns the exit code from flake8."""
     return subprocess.run(
-        [sys.executable, "-m", "flake8", ROOT_DIR],
+        [sys.executable, "-m", "flake8", ROOT_DIR], check=False
-        check=False,
-        cwd=ROOT_DIR,
     ).returncode
 
 
 def run_isort():
     """Returns the exit code from isort."""
     return subprocess.run(
-        [sys.executable, "-m", "isort", "--check", ROOT_DIR],
+        [sys.executable, "-m", "isort", "--check", ROOT_DIR], check=False
-        check=False,
-        cwd=ROOT_DIR,
     ).returncode
 
 
 def main(argv):
     """The main entry."""
     checks = (
-        functools.partial(run_pytest, argv),
+        lambda: pytest.main(argv),
-        functools.partial(run_pytest_py38, argv),
         run_black,
         run_flake8,
         run_isort,
     )
-    # Run all the tests all the time to get full feedback.  Don't exit on the
+    return 0 if all(not c() for c in checks) else 1
-    # first error as that makes it more difficult to iterate in the CQ.
-    return 1 if sum(c() for c in checks) else 0
 
 
 if __name__ == "__main__":

run_tests.vpython3

@@ -5,92 +5,97 @@
 # List of available wheels:
 #   https://chromium.googlesource.com/infra/infra/+/main/infra/tools/dockerbuild/wheels.md
 
-python_version: "3.11"
+python_version: "3.8"
 
 wheel: <
   name: "infra/python/wheels/pytest-py3"
-  version: "version:8.3.4"
+  version: "version:6.2.2"
 >
 
-# Required by pytest==8.3.4
+# Required by pytest==6.2.2
 wheel: <
   name: "infra/python/wheels/py-py2_py3"
-  version: "version:1.11.0"
+  version: "version:1.10.0"
 >
 
-# Required by pytest==8.3.4
+# Required by pytest==6.2.2
 wheel: <
   name: "infra/python/wheels/iniconfig-py3"
   version: "version:1.1.1"
 >
 
-# Required by pytest==8.3.4
+# Required by pytest==6.2.2
 wheel: <
   name: "infra/python/wheels/packaging-py3"
   version: "version:23.0"
 >
 
-# Required by pytest==8.3.4
+# Required by pytest==6.2.2
 wheel: <
   name: "infra/python/wheels/pluggy-py3"
-  version: "version:1.5.0"
+  version: "version:0.13.1"
 >
 
-# Required by pytest==8.3.4
+# Required by pytest==6.2.2
 wheel: <
   name: "infra/python/wheels/toml-py3"
   version: "version:0.10.1"
 >
 
-# Required by pytest==8.3.4
+# Required by pytest==6.2.2
 wheel: <
   name: "infra/python/wheels/pyparsing-py3"
   version: "version:3.0.7"
 >
 
-# Required by pytest==8.3.4
+# Required by pytest==6.2.2
 wheel: <
   name: "infra/python/wheels/attrs-py2_py3"
   version: "version:21.4.0"
 >
 
-# NB: Keep in sync with constraints.txt.
+# Required by packaging==16.8
 wheel: <
-  name: "infra/python/wheels/black-py3"
+  name: "infra/python/wheels/six-py2_py3"
-  version: "version:25.1.0"
+  version: "version:1.16.0"
 >
 
-# Required by black==25.1.0
+wheel: <
+  name: "infra/python/wheels/black-py3"
+  version: "version:23.1.0"
+>
+
+# Required by black==23.1.0
 wheel: <
   name: "infra/python/wheels/mypy-extensions-py3"
   version: "version:0.4.3"
 >
 
-# Required by black==25.1.0
+# Required by black==23.1.0
 wheel: <
   name: "infra/python/wheels/tomli-py3"
   version: "version:2.0.1"
 >
 
-# Required by black==25.1.0
+# Required by black==23.1.0
 wheel: <
   name: "infra/python/wheels/platformdirs-py3"
   version: "version:2.5.2"
 >
 
-# Required by black==25.1.0
+# Required by black==23.1.0
 wheel: <
   name: "infra/python/wheels/pathspec-py3"
   version: "version:0.9.0"
 >
 
-# Required by black==25.1.0
+# Required by black==23.1.0
 wheel: <
   name: "infra/python/wheels/typing-extensions-py3"
   version: "version:4.3.0"
 >
 
-# Required by black==25.1.0
+# Required by black==23.1.0
 wheel: <
   name: "infra/python/wheels/click-py3"
   version: "version:8.0.3"

run_tests.vpython3.8

@@ -1,67 +0,0 @@
-# This is a vpython "spec" file.
-#
-# Read more about `vpython` and how to modify this file here:
-#   https://chromium.googlesource.com/infra/infra/+/main/doc/users/vpython.md
-# List of available wheels:
-#   https://chromium.googlesource.com/infra/infra/+/main/infra/tools/dockerbuild/wheels.md
-
-python_version: "3.8"
-
-wheel: <
-  name: "infra/python/wheels/pytest-py3"
-  version: "version:8.3.4"
->
-
-# Required by pytest==8.3.4
-wheel: <
-  name: "infra/python/wheels/py-py2_py3"
-  version: "version:1.11.0"
->
-
-# Required by pytest==8.3.4
-wheel: <
-  name: "infra/python/wheels/iniconfig-py3"
-  version: "version:1.1.1"
->
-
-# Required by pytest==8.3.4
-wheel: <
-  name: "infra/python/wheels/packaging-py3"
-  version: "version:23.0"
->
-
-# Required by pytest==8.3.4
-wheel: <
-  name: "infra/python/wheels/pluggy-py3"
-  version: "version:1.5.0"
->
-
-# Required by pytest==8.3.4
-wheel: <
-  name: "infra/python/wheels/toml-py3"
-  version: "version:0.10.1"
->
-
-# Required by pytest==8.3.4
-wheel: <
-  name: "infra/python/wheels/tomli-py3"
-  version: "version:2.1.0"
->
-
-# Required by pytest==8.3.4
-wheel: <
-  name: "infra/python/wheels/pyparsing-py3"
-  version: "version:3.0.7"
->
-
-# Required by pytest==8.3.4
-wheel: <
-  name: "infra/python/wheels/attrs-py2_py3"
-  version: "version:21.4.0"
->
-
-# Required by pytest==8.3.4
-wheel: <
-  name: "infra/python/wheels/exceptiongroup-py3"
-  version: "version:1.1.2"
->

subcmds/abandon.py

@@ -70,10 +70,8 @@ It is equivalent to "git branch -D <branchname>".
         else:
             args.insert(0, "'All local branches'")
 
-    @classmethod
+    def _ExecuteOne(self, all_branches, nb, project):
-    def _ExecuteOne(cls, all_branches, nb, project_idx):
         """Abandon one project."""
-        project = cls.get_parallel_context()["projects"][project_idx]
         if all_branches:
             branches = project.GetBranches()
         else:
@@ -91,7 +89,7 @@ It is equivalent to "git branch -D <branchname>".
             if status is not None:
                 ret[name] = status
 
-        return (ret, project_idx, errors)
+        return (ret, project, errors)
 
     def Execute(self, opt, args):
         nb = args[0].split()
@@ -104,8 +102,7 @@ It is equivalent to "git branch -D <branchname>".
         _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
 
         def _ProcessResults(_pool, pm, states):
-            for results, project_idx, errors in states:
+            for results, project, errors in states:
-                project = all_projects[project_idx]
                 for branch, status in results.items():
                     if status:
                         success[branch].append(project)
@@ -114,18 +111,15 @@ It is equivalent to "git branch -D <branchname>".
                 aggregate_errors.extend(errors)
                 pm.update(msg="")
 
-        with self.ParallelContext():
+        self.ExecuteInParallel(
-            self.get_parallel_context()["projects"] = all_projects
+            opt.jobs,
-            self.ExecuteInParallel(
+            functools.partial(self._ExecuteOne, opt.all, nb),
-                opt.jobs,
+            all_projects,
-                functools.partial(self._ExecuteOne, opt.all, nb),
+            callback=_ProcessResults,
-                range(len(all_projects)),
+            output=Progress(
-                callback=_ProcessResults,
+                f"Abandon {nb}", len(all_projects), quiet=opt.quiet
-                output=Progress(
+            ),
-                    f"Abandon {nb}", len(all_projects), quiet=opt.quiet
+        )
-                ),
-                chunksize=1,
-            )
 
         width = max(
             itertools.chain(

subcmds/branches.py

@@ -98,22 +98,6 @@ is shown, then the branch appears in all projects.
 """
     PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
-    @classmethod
-    def _ExpandProjectToBranches(cls, project_idx):
-        """Expands a project into a list of branch names & associated info.
-
-        Args:
-            project_idx: project.Project index
-
-        Returns:
-            List[Tuple[str, git_config.Branch, int]]
-        """
-        branches = []
-        project = cls.get_parallel_context()["projects"][project_idx]
-        for name, b in project.GetBranches().items():
-            branches.append((name, b, project_idx))
-        return branches
-
     def Execute(self, opt, args):
         projects = self.GetProjects(
             args, all_manifests=not opt.this_manifest_only
@@ -123,20 +107,17 @@ is shown, then the branch appears in all projects.
         project_cnt = len(projects)
 
         def _ProcessResults(_pool, _output, results):
-            for name, b, project_idx in itertools.chain.from_iterable(results):
+            for name, b in itertools.chain.from_iterable(results):
-                b.project = projects[project_idx]
                 if name not in all_branches:
                     all_branches[name] = BranchInfo(name)
                 all_branches[name].add(b)
 
-        with self.ParallelContext():
+        self.ExecuteInParallel(
-            self.get_parallel_context()["projects"] = projects
+            opt.jobs,
-            self.ExecuteInParallel(
+            expand_project_to_branches,
-                opt.jobs,
+            projects,
-                self._ExpandProjectToBranches,
+            callback=_ProcessResults,
-                range(len(projects)),
+        )
-                callback=_ProcessResults,
-            )
 
         names = sorted(all_branches)
 
@@ -167,10 +148,7 @@ is shown, then the branch appears in all projects.
             else:
                 published = " "
 
-            # A branch name can contain a percent sign, so we need to escape it.
+            hdr("%c%c %-*s" % (current, published, width, name))
-            # Escape after f-string formatting to properly account for leading
-            # spaces.
-            hdr(f"{current}{published} {name:{width}}".replace("%", "%%"))
             out.write(" |")
 
             _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)
@@ -213,3 +191,19 @@ is shown, then the branch appears in all projects.
             else:
                 out.write(" in all projects")
             out.nl()
+
+
+def expand_project_to_branches(project):
+    """Expands a project into a list of branch names & associated information.
+
+    Args:
+        project: project.Project
+
+    Returns:
+        List[Tuple[str, git_config.Branch]]
+    """
+    branches = []
+    for name, b in project.GetBranches().items():
+        b.project = project
+        branches.append((name, b))
+    return branches

subcmds/checkout.py

@@ -20,6 +20,7 @@ from command import DEFAULT_LOCAL_JOBS
 from error import GitError
 from error import RepoExitError
 from progress import Progress
+from project import Project
 from repo_logging import RepoLogger
 
 
@@ -29,7 +30,7 @@ logger = RepoLogger(__file__)
 class CheckoutBranchResult(NamedTuple):
     # Whether the Project is on the branch (i.e. branch exists and no errors)
     result: bool
-    project_idx: int
+    project: Project
     error: Exception
 
 
@@ -61,17 +62,15 @@ The command is equivalent to:
         if not args:
             self.Usage()
 
-    @classmethod
+    def _ExecuteOne(self, nb, project):
-    def _ExecuteOne(cls, nb, project_idx):
         """Checkout one project."""
         error = None
         result = None
-        project = cls.get_parallel_context()["projects"][project_idx]
         try:
             result = project.CheckoutBranch(nb)
         except GitError as e:
             error = e
-        return CheckoutBranchResult(result, project_idx, error)
+        return CheckoutBranchResult(result, project, error)
 
     def Execute(self, opt, args):
         nb = args[0]
@@ -84,25 +83,22 @@ The command is equivalent to:
 
         def _ProcessResults(_pool, pm, results):
             for result in results:
-                project = all_projects[result.project_idx]
                 if result.error is not None:
                     err.append(result.error)
-                    err_projects.append(project)
+                    err_projects.append(result.project)
                 elif result.result:
-                    success.append(project)
+                    success.append(result.project)
                 pm.update(msg="")
 
-        with self.ParallelContext():
+        self.ExecuteInParallel(
-            self.get_parallel_context()["projects"] = all_projects
+            opt.jobs,
-            self.ExecuteInParallel(
+            functools.partial(self._ExecuteOne, nb),
-                opt.jobs,
+            all_projects,
-                functools.partial(self._ExecuteOne, nb),
+            callback=_ProcessResults,
-                range(len(all_projects)),
+            output=Progress(
-                callback=_ProcessResults,
+                f"Checkout {nb}", len(all_projects), quiet=opt.quiet
-                output=Progress(
+            ),
-                    f"Checkout {nb}", len(all_projects), quiet=opt.quiet
+        )
-                ),
-            )
 
         if err_projects:
             for p in err_projects:

subcmds/diff.py

@@ -40,8 +40,7 @@ to the Unix 'patch' command.
             help="paths are relative to the repository root",
         )
 
-    @classmethod
+    def _ExecuteOne(self, absolute, local, project):
-    def _ExecuteOne(cls, absolute, local, project_idx):
         """Obtains the diff for a specific project.
 
         Args:
@@ -49,13 +48,12 @@ to the Unix 'patch' command.
             local: a boolean, if True, the path is relative to the local
                 (sub)manifest.  If false, the path is relative to the outermost
                 manifest.
-            project_idx: Project index to get status of.
+            project: Project to get status of.
 
         Returns:
             The status of the project.
         """
         buf = io.StringIO()
-        project = cls.get_parallel_context()["projects"][project_idx]
         ret = project.PrintWorkTreeDiff(absolute, output_redir=buf, local=local)
         return (ret, buf.getvalue())
 
@@ -73,15 +71,12 @@ to the Unix 'patch' command.
                     ret = 1
             return ret
 
-        with self.ParallelContext():
+        return self.ExecuteInParallel(
-            self.get_parallel_context()["projects"] = all_projects
+            opt.jobs,
-            return self.ExecuteInParallel(
+            functools.partial(
-                opt.jobs,
+                self._ExecuteOne, opt.absolute, opt.this_manifest_only
-                functools.partial(
+            ),
-                    self._ExecuteOne, opt.absolute, opt.this_manifest_only
+            all_projects,
-                ),
+            callback=_ProcessResults,
-                range(len(all_projects)),
+            ordered=True,
-                callback=_ProcessResults,
+        )
-                ordered=True,
-                chunksize=1,
-            )

subcmds/diffmanifests.py

@@ -233,9 +233,9 @@ synced and their revisions won't be found.
             )
             self.printRevision = self.out.nofmt_printer("revision", fg="yellow")
         else:
-            self.printProject = self.printAdded = self.printRemoved = (
+            self.printProject = (
-                self.printRevision
+                self.printAdded
-            ) = self.printText
+            ) = self.printRemoved = self.printRevision = self.printText
 
         manifest1 = RepoClient(self.repodir)
         manifest1.Override(args[0], load_local_manifests=False)

subcmds/forall.py

@@ -15,6 +15,7 @@
 import errno
 import functools
 import io
+import multiprocessing
 import os
 import re
 import signal
@@ -25,6 +26,7 @@ from color import Coloring
 from command import Command
 from command import DEFAULT_LOCAL_JOBS
 from command import MirrorSafeCommand
+from command import WORKER_BATCH_SIZE
 from error import ManifestInvalidRevisionError
 from repo_logging import RepoLogger
 
@@ -239,6 +241,7 @@ without iterating through the remaining projects.
                     cmd.insert(cmd.index(cn) + 1, "--color")
 
         mirror = self.manifest.IsMirror
+        rc = 0
 
         smart_sync_manifest_name = "smart_sync_override.xml"
         smart_sync_manifest_path = os.path.join(
@@ -261,44 +264,35 @@ without iterating through the remaining projects.
 
         os.environ["REPO_COUNT"] = str(len(projects))
 
-        def _ProcessResults(_pool, _output, results):
-            rc = 0
-            first = True
-            for r, output in results:
-                if output:
-                    if first:
-                        first = False
-                    elif opt.project_header:
-                        print()
-                    # To simplify the DoWorkWrapper, take care of automatic
-                    # newlines.
-                    end = "\n"
-                    if output[-1] == "\n":
-                        end = ""
-                    print(output, end=end)
-                rc = rc or r
-                if r != 0 and opt.abort_on_errors:
-                    raise Exception("Aborting due to previous error")
-            return rc
-
         try:
             config = self.manifest.manifestProject.config
-            with self.ParallelContext():
+            with multiprocessing.Pool(opt.jobs, InitWorker) as pool:
-                self.get_parallel_context()["projects"] = projects
+                results_it = pool.imap(
-                rc = self.ExecuteInParallel(
-                    opt.jobs,
                     functools.partial(
-                        self.DoWorkWrapper, mirror, opt, cmd, shell, config
+                        DoWorkWrapper, mirror, opt, cmd, shell, config
                     ),
-                    range(len(projects)),
+                    enumerate(projects),
-                    callback=_ProcessResults,
+                    chunksize=WORKER_BATCH_SIZE,
-                    ordered=True,
-                    initializer=self.InitWorker,
-                    chunksize=1,
                 )
+                first = True
+                for r, output in results_it:
+                    if output:
+                        if first:
+                            first = False
+                        elif opt.project_header:
+                            print()
+                        # To simplify the DoWorkWrapper, take care of automatic
+                        # newlines.
+                        end = "\n"
+                        if output[-1] == "\n":
+                            end = ""
+                        print(output, end=end)
+                    rc = rc or r
+                    if r != 0 and opt.abort_on_errors:
+                        raise Exception("Aborting due to previous error")
         except (KeyboardInterrupt, WorkerKeyboardInterrupt):
             # Catch KeyboardInterrupt raised inside and outside of workers
-            rc = errno.EINTR
+            rc = rc or errno.EINTR
         except Exception as e:
             # Catch any other exceptions raised
             logger.error(
@@ -306,35 +300,35 @@ without iterating through the remaining projects.
                 type(e).__name__,
                 e,
             )
-            rc = getattr(e, "errno", 1)
+            rc = rc or getattr(e, "errno", 1)
         if rc != 0:
             sys.exit(rc)
 
-    @classmethod
-    def InitWorker(cls):
-        signal.signal(signal.SIGINT, signal.SIG_IGN)
-
-    @classmethod
-    def DoWorkWrapper(cls, mirror, opt, cmd, shell, config, project_idx):
-        """A wrapper around the DoWork() method.
-
-        Catch the KeyboardInterrupt exceptions here and re-raise them as a
-        different, ``Exception``-based exception to stop it flooding the console
-        with stacktraces and making the parent hang indefinitely.
-
-        """
-        project = cls.get_parallel_context()["projects"][project_idx]
-        try:
-            return DoWork(project, mirror, opt, cmd, shell, project_idx, config)
-        except KeyboardInterrupt:
-            print("%s: Worker interrupted" % project.name)
-            raise WorkerKeyboardInterrupt()
-
 
 class WorkerKeyboardInterrupt(Exception):
     """Keyboard interrupt exception for worker processes."""
 
 
+def InitWorker():
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+
+def DoWorkWrapper(mirror, opt, cmd, shell, config, args):
+    """A wrapper around the DoWork() method.
+
+    Catch the KeyboardInterrupt exceptions here and re-raise them as a
+    different, ``Exception``-based exception to stop it flooding the console
+    with stacktraces and making the parent hang indefinitely.
+
+    """
+    cnt, project = args
+    try:
+        return DoWork(project, mirror, opt, cmd, shell, cnt, config)
+    except KeyboardInterrupt:
+        print("%s: Worker interrupted" % project.name)
+        raise WorkerKeyboardInterrupt()
+
+
 def DoWork(project, mirror, opt, cmd, shell, cnt, config):
     env = os.environ.copy()
 

subcmds/gc.py

@@ -1,294 +0,0 @@
-# Copyright (C) 2024 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-from typing import List, Set
-
-from command import Command
-from git_command import GitCommand
-import platform_utils
-from progress import Progress
-from project import Project
-
-
-class Gc(Command):
-    COMMON = True
-    helpSummary = "Cleaning up internal repo and Git state."
-    helpUsage = """
-%prog
-"""
-
-    def _Options(self, p):
-        p.add_option(
-            "-n",
-            "--dry-run",
-            dest="dryrun",
-            default=False,
-            action="store_true",
-            help="do everything except actually delete",
-        )
-        p.add_option(
-            "-y",
-            "--yes",
-            default=False,
-            action="store_true",
-            help="answer yes to all safe prompts",
-        )
-        p.add_option(
-            "--repack",
-            default=False,
-            action="store_true",
-            help="repack all projects that use partial clone with "
-            "filter=blob:none",
-        )
-
-    def _find_git_to_delete(
-        self, to_keep: Set[str], start_dir: str
-    ) -> Set[str]:
-        """Searches no longer needed ".git" directories.
-
-        Scans the file system starting from `start_dir` and removes all
-        directories that end with ".git" that are not in the `to_keep` set.
-        """
-        to_delete = set()
-        for root, dirs, _ in platform_utils.walk(start_dir):
-            for directory in dirs:
-                if not directory.endswith(".git"):
-                    continue
-
-                path = os.path.join(root, directory)
-                if path not in to_keep:
-                    to_delete.add(path)
-
-        return to_delete
-
-    def delete_unused_projects(self, projects: List[Project], opt):
-        print(f"Scanning filesystem under {self.repodir}...")
-
-        project_paths = set()
-        project_object_paths = set()
-
-        for project in projects:
-            project_paths.add(project.gitdir)
-            project_object_paths.add(project.objdir)
-
-        to_delete = self._find_git_to_delete(
-            project_paths, os.path.join(self.repodir, "projects")
-        )
-
-        to_delete.update(
-            self._find_git_to_delete(
-                project_object_paths,
-                os.path.join(self.repodir, "project-objects"),
-            )
-        )
-
-        if not to_delete:
-            print("Nothing to clean up.")
-            return 0
-
-        print("Identified the following projects are no longer used:")
-        print("\n".join(to_delete))
-        print("")
-        if not opt.yes:
-            print(
-                "If you proceed, any local commits in those projects will be "
-                "destroyed!"
-            )
-            ask = input("Proceed? [y/N] ")
-            if ask.lower() != "y":
-                return 1
-
-        pm = Progress(
-            "Deleting",
-            len(to_delete),
-            delay=False,
-            quiet=opt.quiet,
-            show_elapsed=True,
-            elide=True,
-        )
-
-        for path in to_delete:
-            if opt.dryrun:
-                print(f"\nWould have deleted ${path}")
-            else:
-                tmp_path = os.path.join(
-                    os.path.dirname(path),
-                    f"to_be_deleted_{os.path.basename(path)}",
-                )
-                platform_utils.rename(path, tmp_path)
-                platform_utils.rmtree(tmp_path)
-            pm.update(msg=path)
-        pm.end()
-
-        return 0
-
-    def _generate_promisor_files(self, pack_dir: str):
-        """Generates promisor files for all pack files in the given directory.
-
-        Promisor files are empty files with the same name as the corresponding
-        pack file but with the ".promisor" extension. They are used by Git.
-        """
-        for root, _, files in platform_utils.walk(pack_dir):
-            for file in files:
-                if not file.endswith(".pack"):
-                    continue
-                with open(os.path.join(root, f"{file[:-4]}promisor"), "w"):
-                    pass
-
-    def repack_projects(self, projects: List[Project], opt):
-        repack_projects = []
-        # Find all projects eligible for repacking:
-        # - can't be shared
-        # - have a specific fetch filter
-        for project in projects:
-            if project.config.GetBoolean("extensions.preciousObjects"):
-                continue
-            if not project.clone_depth:
-                continue
-            if project.manifest.CloneFilterForDepth != "blob:none":
-                continue
-
-            repack_projects.append(project)
-
-        if opt.dryrun:
-            print(f"Would have repacked {len(repack_projects)} projects.")
-            return 0
-
-        pm = Progress(
-            "Repacking (this will take a while)",
-            len(repack_projects),
-            delay=False,
-            quiet=opt.quiet,
-            show_elapsed=True,
-            elide=True,
-        )
-
-        for project in repack_projects:
-            pm.update(msg=f"{project.name}")
-
-            pack_dir = os.path.join(project.gitdir, "tmp_repo_repack")
-            if os.path.isdir(pack_dir):
-                platform_utils.rmtree(pack_dir)
-            os.mkdir(pack_dir)
-
-            # Prepare workspace for repacking - remove all unreachable refs and
-            # their objects.
-            GitCommand(
-                project,
-                ["reflog", "expire", "--expire-unreachable=all"],
-                verify_command=True,
-            ).Wait()
-            pm.update(msg=f"{project.name} | gc", inc=0)
-            GitCommand(
-                project,
-                ["gc"],
-                verify_command=True,
-            ).Wait()
-
-            # Get all objects that are reachable from the remote, and pack them.
-            pm.update(msg=f"{project.name} | generating list of objects", inc=0)
-            remote_objects_cmd = GitCommand(
-                project,
-                [
-                    "rev-list",
-                    "--objects",
-                    f"--remotes={project.remote.name}",
-                    "--filter=blob:none",
-                    "--tags",
-                ],
-                capture_stdout=True,
-                verify_command=True,
-            )
-
-            # Get all local objects and pack them.
-            local_head_objects_cmd = GitCommand(
-                project,
-                ["rev-list", "--objects", "HEAD^{tree}"],
-                capture_stdout=True,
-                verify_command=True,
-            )
-            local_objects_cmd = GitCommand(
-                project,
-                [
-                    "rev-list",
-                    "--objects",
-                    "--all",
-                    "--reflog",
-                    "--indexed-objects",
-                    "--not",
-                    f"--remotes={project.remote.name}",
-                    "--tags",
-                ],
-                capture_stdout=True,
-                verify_command=True,
-            )
-
-            remote_objects_cmd.Wait()
-
-            pm.update(msg=f"{project.name} | remote repack", inc=0)
-            GitCommand(
-                project,
-                ["pack-objects", os.path.join(pack_dir, "pack")],
-                input=remote_objects_cmd.stdout,
-                capture_stderr=True,
-                capture_stdout=True,
-                verify_command=True,
-            ).Wait()
-
-            # create promisor file for each pack file
-            self._generate_promisor_files(pack_dir)
-
-            local_head_objects_cmd.Wait()
-            local_objects_cmd.Wait()
-
-            pm.update(msg=f"{project.name} | local repack", inc=0)
-            GitCommand(
-                project,
-                ["pack-objects", os.path.join(pack_dir, "pack")],
-                input=local_head_objects_cmd.stdout + local_objects_cmd.stdout,
-                capture_stderr=True,
-                capture_stdout=True,
-                verify_command=True,
-            ).Wait()
-
-            # Swap the old pack directory with the new one.
-            platform_utils.rename(
-                os.path.join(project.objdir, "objects", "pack"),
-                os.path.join(project.objdir, "objects", "pack_old"),
-            )
-            platform_utils.rename(
-                pack_dir,
-                os.path.join(project.objdir, "objects", "pack"),
-            )
-            platform_utils.rmtree(
-                os.path.join(project.objdir, "objects", "pack_old")
-            )
-
-        pm.end()
-        return 0
-
-    def Execute(self, opt, args):
-        projects: List[Project] = self.GetProjects(
-            args, all_manifests=not opt.this_manifest_only
-        )
-
-        ret = self.delete_unused_projects(projects, opt)
-        if ret != 0:
-            return ret
-
-        if not opt.repack:
-            return
-
-        return self.repack_projects(projects, opt)

subcmds/grep.py

@@ -23,6 +23,7 @@ from error import GitError
 from error import InvalidArgumentsError
 from error import SilentRepoExitError
 from git_command import GitCommand
+from project import Project
 from repo_logging import RepoLogger
 
 
@@ -39,7 +40,7 @@ class GrepColoring(Coloring):
 class ExecuteOneResult(NamedTuple):
     """Result from an execute instance."""
 
-    project_idx: int
+    project: Project
     rc: int
     stdout: str
     stderr: str
@@ -261,10 +262,8 @@ contain a line that matches both expressions:
             help="Show only file names not containing matching lines",
         )
 
-    @classmethod
+    def _ExecuteOne(self, cmd_argv, project):
-    def _ExecuteOne(cls, cmd_argv, project_idx):
         """Process one project."""
-        project = cls.get_parallel_context()["projects"][project_idx]
         try:
             p = GitCommand(
                 project,
@@ -275,7 +274,7 @@ contain a line that matches both expressions:
                 verify_command=True,
             )
         except GitError as e:
-            return ExecuteOneResult(project_idx, -1, None, str(e), e)
+            return ExecuteOneResult(project, -1, None, str(e), e)
 
         try:
             error = None
@@ -283,12 +282,10 @@ contain a line that matches both expressions:
         except GitError as e:
             rc = 1
             error = e
-        return ExecuteOneResult(project_idx, rc, p.stdout, p.stderr, error)
+        return ExecuteOneResult(project, rc, p.stdout, p.stderr, error)
 
     @staticmethod
-    def _ProcessResults(
+    def _ProcessResults(full_name, have_rev, opt, _pool, out, results):
-        full_name, have_rev, opt, projects, _pool, out, results
-    ):
         git_failed = False
         bad_rev = False
         have_match = False
@@ -296,10 +293,9 @@ contain a line that matches both expressions:
         errors = []
 
         for result in results:
-            project = projects[result.project_idx]
             if result.rc < 0:
                 git_failed = True
-                out.project("--- project %s ---" % _RelPath(project))
+                out.project("--- project %s ---" % _RelPath(result.project))
                 out.nl()
                 out.fail("%s", result.stderr)
                 out.nl()
@@ -315,7 +311,9 @@ contain a line that matches both expressions:
                     ):
                         bad_rev = True
                     else:
-                        out.project("--- project %s ---" % _RelPath(project))
+                        out.project(
+                            "--- project %s ---" % _RelPath(result.project)
+                        )
                         out.nl()
                         out.fail("%s", result.stderr.strip())
                         out.nl()
@@ -333,13 +331,13 @@ contain a line that matches both expressions:
                     rev, line = line.split(":", 1)
                     out.write("%s", rev)
                     out.write(":")
-                    out.project(_RelPath(project))
+                    out.project(_RelPath(result.project))
                     out.write("/")
                     out.write("%s", line)
                     out.nl()
             elif full_name:
                 for line in r:
-                    out.project(_RelPath(project))
+                    out.project(_RelPath(result.project))
                     out.write("/")
                     out.write("%s", line)
                     out.nl()
@@ -383,19 +381,16 @@ contain a line that matches both expressions:
             cmd_argv.extend(opt.revision)
         cmd_argv.append("--")
 
-        with self.ParallelContext():
+        git_failed, bad_rev, have_match, errors = self.ExecuteInParallel(
-            self.get_parallel_context()["projects"] = projects
+            opt.jobs,
-            git_failed, bad_rev, have_match, errors = self.ExecuteInParallel(
+            functools.partial(self._ExecuteOne, cmd_argv),
-                opt.jobs,
+            projects,
-                functools.partial(self._ExecuteOne, cmd_argv),
+            callback=functools.partial(
-                range(len(projects)),
+                self._ProcessResults, full_name, have_rev, opt
-                callback=functools.partial(
+            ),
-                    self._ProcessResults, full_name, have_rev, opt, projects
+            output=out,
-                ),
+            ordered=True,
-                output=out,
+        )
-                ordered=True,
-                chunksize=1,
-            )
 
         if git_failed:
             raise GrepCommandError(

subcmds/init.py

@@ -52,10 +52,6 @@ The optional -b argument can be used to select the manifest branch
 to checkout and use.  If no branch is specified, the remote's default
 branch is used.  This is equivalent to using -b HEAD.
 
-The optional --manifest-upstream-branch argument can be used when a commit is
-provided to --manifest-branch (or -b), to specify the name of the git ref in
-which the commit can be found.
-
 The optional -m argument can be used to specify an alternate manifest
 to be used. If no manifest is specified, the manifest default.xml
 will be used.
@@ -139,7 +135,6 @@ to update the working directory files.
         # manifest project is special and is created when instantiating the
         # manifest which happens before we parse options.
         self.manifest.manifestProject.clone_depth = opt.manifest_depth
-        self.manifest.manifestProject.upstream = opt.manifest_upstream_branch
         clone_filter_for_depth = (
             "blob:none" if (_REPO_ALLOW_SHALLOW == "0") else None
         )
@@ -322,12 +317,6 @@ to update the working directory files.
                 " be used with --standalone-manifest."
             )
 
-        if opt.manifest_upstream_branch and opt.manifest_branch is None:
-            self.OptionParser.error(
-                "--manifest-upstream-branch cannot be used without "
-                "--manifest-branch."
-            )
-
         if args:
             if opt.manifest_url:
                 self.OptionParser.error(

subcmds/prune.py

@@ -27,10 +27,8 @@ class Prune(PagedCommand):
 """
     PARALLEL_JOBS = DEFAULT_LOCAL_JOBS
 
-    @classmethod
+    def _ExecuteOne(self, project):
-    def _ExecuteOne(cls, project_idx):
         """Process one project."""
-        project = cls.get_parallel_context()["projects"][project_idx]
         return project.PruneHeads()
 
     def Execute(self, opt, args):
@@ -43,15 +41,13 @@ class Prune(PagedCommand):
         def _ProcessResults(_pool, _output, results):
             return list(itertools.chain.from_iterable(results))
 
-        with self.ParallelContext():
+        all_branches = self.ExecuteInParallel(
-            self.get_parallel_context()["projects"] = projects
+            opt.jobs,
-            all_branches = self.ExecuteInParallel(
+            self._ExecuteOne,
-                opt.jobs,
+            projects,
-                self._ExecuteOne,
+            callback=_ProcessResults,
-                range(len(projects)),
+            ordered=True,
-                callback=_ProcessResults,
+        )
-                ordered=True,
-            )
 
         if not all_branches:
             return

subcmds/start.py

@@ -21,6 +21,7 @@ from error import RepoExitError
 from git_command import git
 from git_config import IsImmutable
 from progress import Progress
+from project import Project
 from repo_logging import RepoLogger
 
 
@@ -28,7 +29,7 @@ logger = RepoLogger(__file__)
 
 
 class ExecuteOneResult(NamedTuple):
-    project_idx: int
+    project: Project
     error: Exception
 
 
@@ -79,20 +80,18 @@ revision specified in the manifest.
         if not git.check_ref_format("heads/%s" % nb):
             self.OptionParser.error("'%s' is not a valid name" % nb)
 
-    @classmethod
+    def _ExecuteOne(self, revision, nb, project):
-    def _ExecuteOne(cls, revision, nb, default_revisionExpr, project_idx):
         """Start one project."""
         # If the current revision is immutable, such as a SHA1, a tag or
         # a change, then we can't push back to it. Substitute with
         # dest_branch, if defined; or with manifest default revision instead.
         branch_merge = ""
         error = None
-        project = cls.get_parallel_context()["projects"][project_idx]
         if IsImmutable(project.revisionExpr):
             if project.dest_branch:
                 branch_merge = project.dest_branch
             else:
-                branch_merge = default_revisionExpr
+                branch_merge = self.manifest.default.revisionExpr
 
         try:
             project.StartBranch(
@@ -101,7 +100,7 @@ revision specified in the manifest.
         except Exception as e:
             logger.error("error: unable to checkout %s: %s", project.name, e)
             error = e
-        return ExecuteOneResult(project_idx, error)
+        return ExecuteOneResult(project, error)
 
     def Execute(self, opt, args):
         nb = args[0]
@@ -121,28 +120,19 @@ revision specified in the manifest.
         def _ProcessResults(_pool, pm, results):
             for result in results:
                 if result.error:
-                    project = all_projects[result.project_idx]
+                    err_projects.append(result.project)
-                    err_projects.append(project)
                     err.append(result.error)
                 pm.update(msg="")
 
-        with self.ParallelContext():
+        self.ExecuteInParallel(
-            self.get_parallel_context()["projects"] = all_projects
+            opt.jobs,
-            self.ExecuteInParallel(
+            functools.partial(self._ExecuteOne, opt.revision, nb),
-                opt.jobs,
+            all_projects,
-                functools.partial(
+            callback=_ProcessResults,
-                    self._ExecuteOne,
+            output=Progress(
-                    opt.revision,
+                f"Starting {nb}", len(all_projects), quiet=opt.quiet
-                    nb,
+            ),
-                    self.manifest.default.revisionExpr,
+        )
-                ),
-                range(len(all_projects)),
-                callback=_ProcessResults,
-                output=Progress(
-                    f"Starting {nb}", len(all_projects), quiet=opt.quiet
-                ),
-                chunksize=1,
-            )
 
         if err_projects:
             for p in err_projects:

subcmds/status.py

@@ -88,8 +88,7 @@ the following meanings:
             "projects",
         )
 
-    @classmethod
+    def _StatusHelper(self, quiet, local, project):
-    def _StatusHelper(cls, quiet, local, project_idx):
         """Obtains the status for a specific project.
 
         Obtains the status for a project, redirecting the output to
@@ -100,13 +99,12 @@ the following meanings:
             local: a boolean, if True, the path is relative to the local
                 (sub)manifest.  If false, the path is relative to the outermost
                 manifest.
-            project_idx: Project index to get status of.
+            project: Project to get status of.
 
         Returns:
             The status of the project.
         """
         buf = io.StringIO()
-        project = cls.get_parallel_context()["projects"][project_idx]
         ret = project.PrintWorkTreeStatus(
             quiet=quiet, output_redir=buf, local=local
         )
@@ -145,18 +143,15 @@ the following meanings:
                     ret += 1
             return ret
 
-        with self.ParallelContext():
+        counter = self.ExecuteInParallel(
-            self.get_parallel_context()["projects"] = all_projects
+            opt.jobs,
-            counter = self.ExecuteInParallel(
+            functools.partial(
-                opt.jobs,
+                self._StatusHelper, opt.quiet, opt.this_manifest_only
-                functools.partial(
+            ),
-                    self._StatusHelper, opt.quiet, opt.this_manifest_only
+            all_projects,
-                ),
+            callback=_ProcessResults,
-                range(len(all_projects)),
+            ordered=True,
-                callback=_ProcessResults,
+        )
-                ordered=True,
-                chunksize=1,
-            )
 
         if not opt.quiet and len(all_projects) == counter:
             print("nothing to commit (working directory clean)")

subcmds/sync.py

@@ -131,17 +131,12 @@ def _SafeCheckoutOrder(checkouts: List[Project]) -> List[List[Project]]:
     return res
 
 
-def _chunksize(projects: int, jobs: int) -> int:
-    """Calculate chunk size for the given number of projects and jobs."""
-    return min(max(1, projects // jobs), WORKER_BATCH_SIZE)
-
-
 class _FetchOneResult(NamedTuple):
     """_FetchOne return value.
 
     Attributes:
       success (bool): True if successful.
-      project_idx (int): The fetched project index.
+      project (Project): The fetched project.
       start (float): The starting time.time().
       finish (float): The ending time.time().
       remote_fetched (bool): True if the remote was actually queried.
@@ -149,7 +144,7 @@ class _FetchOneResult(NamedTuple):
 
     success: bool
     errors: List[Exception]
-    project_idx: int
+    project: Project
     start: float
     finish: float
     remote_fetched: bool
@@ -182,14 +177,14 @@ class _CheckoutOneResult(NamedTuple):
 
     Attributes:
       success (bool): True if successful.
-      project_idx (int): The project index.
+      project (Project): The project.
       start (float): The starting time.time().
       finish (float): The ending time.time().
     """
 
     success: bool
     errors: List[Exception]
-    project_idx: int
+    project: Project
     start: float
     finish: float
 
@@ -350,8 +345,6 @@ later is required to fix a server side protocol bug.
     # value later on.
     PARALLEL_JOBS = 0
 
-    _JOBS_WARN_THRESHOLD = 100
-
     def _Options(self, p, show_smart=True):
         p.add_option(
             "--jobs-network",
@@ -594,8 +587,7 @@ later is required to fix a server side protocol bug.
             branch = branch[len(R_HEADS) :]
         return branch
 
-    @classmethod
+    def _GetCurrentBranchOnly(self, opt, manifest):
-    def _GetCurrentBranchOnly(cls, opt, manifest):
         """Returns whether current-branch or use-superproject options are
         enabled.
 
@@ -713,8 +705,7 @@ later is required to fix a server side protocol bug.
         if need_unload:
             m.outer_client.manifest.Unload()
 
-    @classmethod
+    def _FetchProjectList(self, opt, projects):
-    def _FetchProjectList(cls, opt, projects):
         """Main function of the fetch worker.
 
         The projects we're given share the same underlying git object store, so
@@ -726,23 +717,21 @@ later is required to fix a server side protocol bug.
             opt: Program options returned from optparse.  See _Options().
             projects: Projects to fetch.
         """
-        return [cls._FetchOne(opt, x) for x in projects]
+        return [self._FetchOne(opt, x) for x in projects]
 
-    @classmethod
+    def _FetchOne(self, opt, project):
-    def _FetchOne(cls, opt, project_idx):
         """Fetch git objects for a single project.
 
         Args:
             opt: Program options returned from optparse.  See _Options().
-            project_idx: Project index for the project to fetch.
+            project: Project object for the project to fetch.
 
         Returns:
             Whether the fetch was successful.
         """
-        project = cls.get_parallel_context()["projects"][project_idx]
         start = time.time()
         k = f"{project.name} @ {project.relpath}"
-        cls.get_parallel_context()["sync_dict"][k] = start
+        self._sync_dict[k] = start
         success = False
         remote_fetched = False
         errors = []
@@ -752,7 +741,7 @@ later is required to fix a server side protocol bug.
                 quiet=opt.quiet,
                 verbose=opt.verbose,
                 output_redir=buf,
-                current_branch_only=cls._GetCurrentBranchOnly(
+                current_branch_only=self._GetCurrentBranchOnly(
                     opt, project.manifest
                 ),
                 force_sync=opt.force_sync,
@@ -762,7 +751,7 @@ later is required to fix a server side protocol bug.
                 optimized_fetch=opt.optimized_fetch,
                 retry_fetches=opt.retry_fetches,
                 prune=opt.prune,
-                ssh_proxy=cls.get_parallel_context()["ssh_proxy"],
+                ssh_proxy=self.ssh_proxy,
                 clone_filter=project.manifest.CloneFilter,
                 partial_clone_exclude=project.manifest.PartialCloneExclude,
                 clone_filter_for_depth=project.manifest.CloneFilterForDepth,
@@ -794,20 +783,24 @@ later is required to fix a server side protocol bug.
                 type(e).__name__,
                 e,
             )
+            del self._sync_dict[k]
             errors.append(e)
             raise
-        finally:
-            del cls.get_parallel_context()["sync_dict"][k]
 
         finish = time.time()
+        del self._sync_dict[k]
         return _FetchOneResult(
-            success, errors, project_idx, start, finish, remote_fetched
+            success, errors, project, start, finish, remote_fetched
         )
 
+    @classmethod
+    def _FetchInitChild(cls, ssh_proxy):
+        cls.ssh_proxy = ssh_proxy
+
     def _GetSyncProgressMessage(self):
         earliest_time = float("inf")
         earliest_proj = None
-        items = self.get_parallel_context()["sync_dict"].items()
+        items = self._sync_dict.items()
         for project, t in items:
             if t < earliest_time:
                 earliest_time = t
@@ -815,7 +808,7 @@ later is required to fix a server side protocol bug.
 
         if not earliest_proj:
             # This function is called when sync is still running but in some
-            # cases (by chance), sync_dict can contain no entries. Return some
+            # cases (by chance), _sync_dict can contain no entries. Return some
             # text to indicate that sync is still working.
             return "..working.."
 
@@ -823,19 +816,10 @@ later is required to fix a server side protocol bug.
         jobs = jobs_str(len(items))
         return f"{jobs} | {elapsed_str(elapsed)} {earliest_proj}"
 
-    @classmethod
-    def InitWorker(cls):
-        # Force connect to the manager server now.
-        # This is good because workers are initialized one by one. Without this,
-        # multiple workers may connect to the manager when handling the first
-        # job at the same time. Then the connection may fail if too many
-        # connections are pending and execeeded the socket listening backlog,
-        # especially on MacOS.
-        len(cls.get_parallel_context()["sync_dict"])
-
     def _Fetch(self, projects, opt, err_event, ssh_proxy, errors):
         ret = True
 
+        jobs = opt.jobs_network
         fetched = set()
         remote_fetched = set()
         pm = Progress(
@@ -847,6 +831,7 @@ later is required to fix a server side protocol bug.
             elide=True,
         )
 
+        self._sync_dict = multiprocessing.Manager().dict()
         sync_event = _threading.Event()
 
         def _MonitorSyncLoop():
@@ -857,13 +842,19 @@ later is required to fix a server side protocol bug.
 
         sync_progress_thread = _threading.Thread(target=_MonitorSyncLoop)
         sync_progress_thread.daemon = True
+        sync_progress_thread.start()
 
-        def _ProcessResults(pool, pm, results_sets):
+        objdir_project_map = dict()
+        for project in projects:
+            objdir_project_map.setdefault(project.objdir, []).append(project)
+        projects_list = list(objdir_project_map.values())
+
+        def _ProcessResults(results_sets):
             ret = True
             for results in results_sets:
                 for result in results:
                     success = result.success
-                    project = projects[result.project_idx]
+                    project = result.project
                     start = result.start
                     finish = result.finish
                     self._fetch_times.Set(project, finish - start)
@@ -887,50 +878,58 @@ later is required to fix a server side protocol bug.
                         fetched.add(project.gitdir)
                     pm.update()
                 if not ret and opt.fail_fast:
-                    if pool:
-                        pool.close()
                     break
             return ret
 
-        with self.ParallelContext():
+        # We pass the ssh proxy settings via the class.  This allows
-            self.get_parallel_context()["projects"] = projects
+        # multiprocessing to pickle it up when spawning children.  We can't pass
-            self.get_parallel_context()[
+        # it as an argument to _FetchProjectList below as multiprocessing is
-                "sync_dict"
+        # unable to pickle those.
-            ] = multiprocessing.Manager().dict()
+        Sync.ssh_proxy = None
 
-            objdir_project_map = dict()
+        # NB: Multiprocessing is heavy, so don't spin it up for one job.
-            for index, project in enumerate(projects):
+        if len(projects_list) == 1 or jobs == 1:
-                objdir_project_map.setdefault(project.objdir, []).append(index)
+            self._FetchInitChild(ssh_proxy)
-            projects_list = list(objdir_project_map.values())
+            if not _ProcessResults(
-
+                self._FetchProjectList(opt, x) for x in projects_list
-            jobs = max(1, min(opt.jobs_network, len(projects_list)))
+            ):
-
+                ret = False
-            # We pass the ssh proxy settings via the class.  This allows
+        else:
-            # multiprocessing to pickle it up when spawning children.  We can't
+            # Favor throughput over responsiveness when quiet.  It seems that
-            # pass it as an argument to _FetchProjectList below as
+            # imap() will yield results in batches relative to chunksize, so
-            # multiprocessing is unable to pickle those.
+            # even as the children finish a sync, we won't see the result until
-            self.get_parallel_context()["ssh_proxy"] = ssh_proxy
+            # one child finishes ~chunksize jobs.  When using a large --jobs
-
+            # with large chunksize, this can be jarring as there will be a large
-            sync_progress_thread.start()
+            # initial delay where repo looks like it isn't doing anything and
-            if not opt.quiet:
+            # sits at 0%, but then suddenly completes a lot of jobs all at once.
+            # Since this code is more network bound, we can accept a bit more
+            # CPU overhead with a smaller chunksize so that the user sees more
+            # immediate & continuous feedback.
+            if opt.quiet:
+                chunksize = WORKER_BATCH_SIZE
+            else:
                 pm.update(inc=0, msg="warming up")
-            try:
+                chunksize = 4
-                ret = self.ExecuteInParallel(
+            with multiprocessing.Pool(
-                    jobs,
+                jobs, initializer=self._FetchInitChild, initargs=(ssh_proxy,)
+            ) as pool:
+                results = pool.imap_unordered(
                     functools.partial(self._FetchProjectList, opt),
                     projects_list,
-                    callback=_ProcessResults,
+                    chunksize=chunksize,
-                    output=pm,
-                    # Use chunksize=1 to avoid the chance that some workers are
-                    # idle while other workers still have more than one job in
-                    # their chunk queue.
-                    chunksize=1,
-                    initializer=self.InitWorker,
                 )
-            finally:
+                if not _ProcessResults(results):
-                sync_event.set()
+                    ret = False
-                sync_progress_thread.join()
+                    pool.close()
 
+        # Cleanup the reference now that we're done with it, and we're going to
+        # release any resources it points to.  If we don't, later
+        # multiprocessing usage (e.g. checkouts) will try to pickle and then
+        # crash.
+        del Sync.ssh_proxy
+
+        sync_event.set()
+        pm.end()
         self._fetch_times.Save()
         self._local_sync_state.Save()
 
@@ -971,9 +970,7 @@ later is required to fix a server side protocol bug.
         if not success:
             err_event.set()
 
-        # Call self update, unless requested not to
+        _PostRepoFetch(rp, opt.repo_verify)
-        if os.environ.get("REPO_SKIP_SELF_UPDATE", "0") == "0":
-            _PostRepoFetch(rp, opt.repo_verify)
         if opt.network_only:
             # Bail out now; the rest touches the working tree.
             if err_event.is_set():
@@ -1018,15 +1015,14 @@ later is required to fix a server side protocol bug.
 
         return _FetchMainResult(all_projects)
 
-    @classmethod
     def _CheckoutOne(
-        cls,
+        self,
         detach_head,
         force_sync,
         force_checkout,
         force_rebase,
         verbose,
-        project_idx,
+        project,
     ):
         """Checkout work tree for one project
 
@@ -1038,12 +1034,11 @@ later is required to fix a server side protocol bug.
             force_checkout: Force checking out of the repo content.
             force_rebase: Force rebase.
             verbose: Whether to show verbose messages.
-            project_idx: Project index for the project to checkout.
+            project: Project object for the project to checkout.
 
         Returns:
             Whether the fetch was successful.
         """
-        project = cls.get_parallel_context()["projects"][project_idx]
         start = time.time()
         syncbuf = SyncBuffer(
             project.manifest.manifestProject.config, detach_head=detach_head
@@ -1060,8 +1055,6 @@ later is required to fix a server side protocol bug.
                 verbose=verbose,
             )
             success = syncbuf.Finish()
-        except KeyboardInterrupt:
-            logger.error("Keyboard interrupt while processing %s", project.name)
         except GitError as e:
             logger.error(
                 "error.GitError: Cannot checkout %s: %s", project.name, e
@@ -1079,7 +1072,7 @@ later is required to fix a server side protocol bug.
         if not success:
             logger.error("error: Cannot checkout %s", project.name)
         finish = time.time()
-        return _CheckoutOneResult(success, errors, project_idx, start, finish)
+        return _CheckoutOneResult(success, errors, project, start, finish)
 
     def _Checkout(self, all_projects, opt, err_results, checkout_errors):
         """Checkout projects listed in all_projects
@@ -1097,9 +1090,7 @@ later is required to fix a server side protocol bug.
             ret = True
             for result in results:
                 success = result.success
-                project = self.get_parallel_context()["projects"][
+                project = result.project
-                    result.project_idx
-                ]
                 start = result.start
                 finish = result.finish
                 self.event_log.AddSync(
@@ -1126,28 +1117,22 @@ later is required to fix a server side protocol bug.
             return ret
 
         for projects in _SafeCheckoutOrder(all_projects):
-            with self.ParallelContext():
+            proc_res = self.ExecuteInParallel(
-                self.get_parallel_context()["projects"] = projects
+                opt.jobs_checkout,
-                proc_res = self.ExecuteInParallel(
+                functools.partial(
-                    opt.jobs_checkout,
+                    self._CheckoutOne,
-                    functools.partial(
+                    opt.detach_head,
-                        self._CheckoutOne,
+                    opt.force_sync,
-                        opt.detach_head,
+                    opt.force_checkout,
-                        opt.force_sync,
+                    opt.rebase,
-                        opt.force_checkout,
+                    opt.verbose,
-                        opt.rebase,
+                ),
-                        opt.verbose,
+                projects,
-                    ),
+                callback=_ProcessResults,
-                    range(len(projects)),
+                output=Progress(
-                    callback=_ProcessResults,
+                    "Checking out", len(all_projects), quiet=opt.quiet
-                    output=Progress(
+                ),
-                        "Checking out", len(all_projects), quiet=opt.quiet
+            )
-                    ),
-                    # Use chunksize=1 to avoid the chance that some workers are
-                    # idle while other workers still have more than one job in
-                    # their chunk queue.
-                    chunksize=1,
-                )
 
         self._local_sync_state.Save()
         return proc_res and not err_results
@@ -1446,10 +1431,7 @@ later is required to fix a server side protocol bug.
             for need_remove_file in need_remove_files:
                 # Try to remove the updated copyfile or linkfile.
                 # So, if the file is not exist, nothing need to do.
-                platform_utils.remove(
+                platform_utils.remove(need_remove_file, missing_ok=True)
-                    os.path.join(self.client.topdir, need_remove_file),
-                    missing_ok=True,
-                )
 
         # Create copy-link-files.json, save dest path of "copyfile" and
         # "linkfile".
@@ -1504,7 +1486,6 @@ later is required to fix a server side protocol bug.
         if manifest_server.startswith("persistent-"):
             manifest_server = manifest_server[len("persistent-") :]
 
-        # Changes in behavior should update docs/smart-sync.md accordingly.
         try:
             server = xmlrpc.client.Server(manifest_server, transport=transport)
             if opt.smart_sync:
@@ -1730,24 +1711,6 @@ later is required to fix a server side protocol bug.
         opt.jobs_network = min(opt.jobs_network, jobs_soft_limit)
         opt.jobs_checkout = min(opt.jobs_checkout, jobs_soft_limit)
 
-        # Warn once if effective job counts seem excessively high.
-        # Prioritize --jobs, then --jobs-network, then --jobs-checkout.
-        job_options_to_check = (
-            ("--jobs", opt.jobs),
-            ("--jobs-network", opt.jobs_network),
-            ("--jobs-checkout", opt.jobs_checkout),
-        )
-        for name, value in job_options_to_check:
-            if value > self._JOBS_WARN_THRESHOLD:
-                logger.warning(
-                    "High job count (%d > %d) specified for %s; this may "
-                    "lead to excessive resource usage or diminishing returns.",
-                    value,
-                    self._JOBS_WARN_THRESHOLD,
-                    name,
-                )
-                break
-
     def Execute(self, opt, args):
         errors = []
         try:
@@ -2019,8 +1982,6 @@ def _PostRepoFetch(rp, repo_verify=True, verbose=False):
             # We also have to make sure this will switch to an older commit if
             # that's the latest tag in order to support release rollback.
             try:
-                # Refresh index since reset --keep won't do it.
-                rp.work_git.update_index("-q", "--refresh")
                 rp.work_git.reset("--keep", new_rev)
             except GitError as e:
                 raise RepoUnhandledExceptionError(e)

subcmds/upload.py

@@ -603,22 +603,19 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
             full_dest = destination
             if not full_dest.startswith(R_HEADS):
                 full_dest = R_HEADS + full_dest
-            full_revision = branch.project.revisionExpr
-            if not full_revision.startswith(R_HEADS):
-                full_revision = R_HEADS + full_revision
 
             # If the merge branch of the local branch is different from
             # the project's revision AND destination, this might not be
             # intentional.
             if (
                 merge_branch
-                and merge_branch != full_revision
+                and merge_branch != branch.project.revisionExpr
                 and merge_branch != full_dest
             ):
                 print(
                     f"For local branch {branch.name}: merge branch "
                     f"{merge_branch} does not match destination branch "
-                    f"{destination} and revision {branch.project.revisionExpr}"
+                    f"{destination}"
                 )
                 print("skipping upload.")
                 print(
@@ -716,17 +713,16 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
         merge_branch = p.stdout.strip()
         return merge_branch
 
-    @classmethod
+    @staticmethod
-    def _GatherOne(cls, opt, project_idx):
+    def _GatherOne(opt, project):
         """Figure out the upload status for |project|."""
-        project = cls.get_parallel_context()["projects"][project_idx]
         if opt.current_branch:
             cbr = project.CurrentBranch
             up_branch = project.GetUploadableBranch(cbr)
             avail = [up_branch] if up_branch else None
         else:
             avail = project.GetUploadableBranches(opt.branch)
-        return (project_idx, avail)
+        return (project, avail)
 
     def Execute(self, opt, args):
         projects = self.GetProjects(
@@ -736,8 +732,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
         def _ProcessResults(_pool, _out, results):
             pending = []
             for result in results:
-                project_idx, avail = result
+                project, avail = result
-                project = projects[project_idx]
                 if avail is None:
                     logger.error(
                         'repo: error: %s: Unable to upload branch "%s". '
@@ -748,17 +743,15 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
                         project.manifest.branch,
                     )
                 elif avail:
-                    pending.append((project, avail))
+                    pending.append(result)
             return pending
 
-        with self.ParallelContext():
+        pending = self.ExecuteInParallel(
-            self.get_parallel_context()["projects"] = projects
+            opt.jobs,
-            pending = self.ExecuteInParallel(
+            functools.partial(self._GatherOne, opt),
-                opt.jobs,
+            projects,
-                functools.partial(self._GatherOne, opt),
+            callback=_ProcessResults,
-                range(len(projects)),
+        )
-                callback=_ProcessResults,
-            )
 
         if not pending:
             if opt.branch is None:

tests/fixtures/gitc_config

@@ -0,0 +1 @@
+gitc_dir=/test/usr/local/google/gitc

tests/test_git_command.py

@@ -21,8 +21,6 @@ import subprocess
 import unittest
 from unittest import mock
 
-import pytest
-
 import git_command
 import wrapper
 
@@ -265,7 +263,6 @@ class UserAgentUnitTest(unittest.TestCase):
         m = re.match(r"^[^ ]+$", os_name)
         self.assertIsNotNone(m)
 
-    @pytest.mark.skip_cq("TODO(b/266734831): Find out why this fails in CQ")
     def test_smoke_repo(self):
         """Make sure repo UA returns something useful."""
         ua = git_command.user_agent.repo
@@ -274,7 +271,6 @@ class UserAgentUnitTest(unittest.TestCase):
         m = re.match(r"^git-repo/[^ ]+ ([^ ]+) git/[^ ]+ Python/[0-9.]+", ua)
         self.assertIsNotNone(m)
 
-    @pytest.mark.skip_cq("TODO(b/266734831): Find out why this fails in CQ")
     def test_smoke_git(self):
         """Make sure git UA returns something useful."""
         ua = git_command.user_agent.git

tests/test_git_superproject.py

@@ -21,7 +21,6 @@ import tempfile
 import unittest
 from unittest import mock
 
-import pytest
 from test_manifest_xml import sort_attributes
 
 import git_superproject
@@ -146,7 +145,6 @@ class SuperprojectTestCase(unittest.TestCase):
         )
         self.assertIsNone(manifest.superproject)
 
-    @pytest.mark.skip_cq("TODO(b/266734831): Find out why this takes 8m+ in CQ")
     def test_superproject_get_superproject_invalid_url(self):
         """Test with an invalid url."""
         manifest = self.getXmlManifest(
@@ -170,7 +168,6 @@ class SuperprojectTestCase(unittest.TestCase):
         self.assertFalse(sync_result.success)
         self.assertTrue(sync_result.fatal)
 
-    @pytest.mark.skip_cq("TODO(b/266734831): Find out why this takes 8m+ in CQ")
     def test_superproject_get_superproject_invalid_branch(self):
         """Test with an invalid branch."""
         manifest = self.getXmlManifest(

tests/test_git_trace2_event_log.py

@@ -150,7 +150,7 @@ class EventLogTestCase(unittest.TestCase):
         <version event>
         <start event>
         """
-        self._event_log_module.StartEvent([])
+        self._event_log_module.StartEvent()
         with tempfile.TemporaryDirectory(prefix="event_log_tests") as tempdir:
             log_path = self._event_log_module.Write(path=tempdir)
             self._log_data = self.readLog(log_path)
@@ -213,8 +213,10 @@ class EventLogTestCase(unittest.TestCase):
         <version event>
         <command event>
         """
+        name = "repo"
+        subcommands = ["init" "this"]
         self._event_log_module.CommandEvent(
-            name="repo", subcommands=["init", "this"]
+            name="repo", subcommands=subcommands
         )
         with tempfile.TemporaryDirectory(prefix="event_log_tests") as tempdir:
             log_path = self._event_log_module.Write(path=tempdir)
@@ -223,10 +225,12 @@ class EventLogTestCase(unittest.TestCase):
         self.assertEqual(len(self._log_data), 2)
         command_event = self._log_data[1]
         self.verifyCommonKeys(self._log_data[0], expected_event_name="version")
-        self.verifyCommonKeys(command_event, expected_event_name="cmd_name")
+        self.verifyCommonKeys(command_event, expected_event_name="command")
         # Check for 'command' event specific fields.
         self.assertIn("name", command_event)
-        self.assertEqual(command_event["name"], "repo-init-this")
+        self.assertIn("subcommands", command_event)
+        self.assertEqual(command_event["name"], name)
+        self.assertEqual(command_event["subcommands"], subcommands)
 
     def test_def_params_event_repo_config(self):
         """Test 'def_params' event data outputs only repo config keys.
@@ -378,17 +382,17 @@ class EventLogTestCase(unittest.TestCase):
             socket_path = os.path.join(tempdir, "server.sock")
             server_ready = threading.Condition()
             # Start "server" listening on Unix domain socket at socket_path.
-            server_thread = threading.Thread(
-                target=serverLoggingThread,
-                args=(socket_path, server_ready, received_traces),
-            )
             try:
+                server_thread = threading.Thread(
+                    target=serverLoggingThread,
+                    args=(socket_path, server_ready, received_traces),
+                )
                 server_thread.start()
 
                 with server_ready:
                     server_ready.wait(timeout=120)
 
-                self._event_log_module.StartEvent([])
+                self._event_log_module.StartEvent()
                 path = self._event_log_module.Write(
                     path=f"af_unix:{socket_path}"
                 )

tests/test_manifest_xml.py

@@ -51,7 +51,7 @@ INVALID_FS_PATHS = (
     "foo~",
     "blah/foo~",
     # Block Unicode characters that get normalized out by filesystems.
-    "foo\u200cbar",
+    "foo\u200Cbar",
     # Block newlines.
     "f\n/bar",
     "f\r/bar",
@@ -1049,91 +1049,6 @@ class RemoveProjectElementTests(ManifestParseTestCase):
         self.assertTrue(found_proj1_path1)
         self.assertTrue(found_proj2)
 
-    def test_base_revision_checks_on_patching(self):
-        manifest_fail_wrong_tag = self.getXmlManifest(
-            """
-<manifest>
-  <remote name="default-remote" fetch="http://localhost" />
-  <default remote="default-remote" revision="tag.002" />
-  <project name="project1" path="tests/path1" />
-  <extend-project name="project1" revision="new_hash" base-rev="tag.001" />
-</manifest>
-"""
-        )
-        with self.assertRaises(error.ManifestParseError):
-            manifest_fail_wrong_tag.ToXml()
-
-        manifest_fail_remove = self.getXmlManifest(
-            """
-<manifest>
-  <remote name="default-remote" fetch="http://localhost" />
-  <default remote="default-remote" revision="refs/heads/main" />
-  <project name="project1" path="tests/path1" revision="hash1" />
-  <remove-project name="project1" base-rev="wrong_hash" />
-</manifest>
-"""
-        )
-        with self.assertRaises(error.ManifestParseError):
-            manifest_fail_remove.ToXml()
-
-        manifest_fail_extend = self.getXmlManifest(
-            """
-<manifest>
-  <remote name="default-remote" fetch="http://localhost" />
-  <default remote="default-remote" revision="refs/heads/main" />
-  <project name="project1" path="tests/path1" revision="hash1" />
-  <extend-project name="project1" revision="new_hash" base-rev="wrong_hash" />
-</manifest>
-"""
-        )
-        with self.assertRaises(error.ManifestParseError):
-            manifest_fail_extend.ToXml()
-
-        manifest_fail_unknown = self.getXmlManifest(
-            """
-<manifest>
-  <remote name="default-remote" fetch="http://localhost" />
-  <default remote="default-remote" revision="refs/heads/main" />
-  <project name="project1" path="tests/path1" />
-  <extend-project name="project1" revision="new_hash" base-rev="any_hash" />
-</manifest>
-"""
-        )
-        with self.assertRaises(error.ManifestParseError):
-            manifest_fail_unknown.ToXml()
-
-        manifest_ok = self.getXmlManifest(
-            """
-<manifest>
-  <remote name="default-remote" fetch="http://localhost" />
-  <default remote="default-remote" revision="refs/heads/main" />
-  <project name="project1" path="tests/path1" revision="hash1" />
-  <project name="project2" path="tests/path2" revision="hash2" />
-  <project name="project3" path="tests/path3" revision="hash3" />
-  <project name="project4" path="tests/path4" revision="hash4" />
-
-  <remove-project name="project1" />
-  <remove-project name="project2" base-rev="hash2" />
-  <project name="project2" path="tests/path2" revision="new_hash2" />
-  <extend-project name="project3" base-rev="hash3" revision="new_hash3" />
-  <extend-project name="project3" base-rev="new_hash3" revision="newer_hash3" />
-  <remove-project path="tests/path4" base-rev="hash4" />
-</manifest>
-"""
-        )
-        found_proj2 = False
-        found_proj3 = False
-        for proj in manifest_ok.projects:
-            if proj.name == "project2":
-                found_proj2 = True
-            if proj.name == "project3":
-                found_proj3 = True
-            self.assertNotEqual(proj.name, "project1")
-            self.assertNotEqual(proj.name, "project4")
-        self.assertTrue(found_proj2)
-        self.assertTrue(found_proj3)
-        self.assertTrue(len(manifest_ok.projects) == 2)
-
 
 class ExtendProjectElementTests(ManifestParseTestCase):
     """Tests for <extend-project>."""

tests/test_subcmds_forall.py

@@ -1,156 +0,0 @@
-# Copyright (C) 2024 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Unittests for the forall subcmd."""
-
-from io import StringIO
-import os
-from shutil import rmtree
-import subprocess
-import tempfile
-import unittest
-from unittest import mock
-
-import git_command
-import manifest_xml
-import project
-import subcmds
-
-
-class AllCommands(unittest.TestCase):
-    """Check registered all_commands."""
-
-    def setUp(self):
-        """Common setup."""
-        self.tempdirobj = tempfile.TemporaryDirectory(prefix="forall_tests")
-        self.tempdir = self.tempdirobj.name
-        self.repodir = os.path.join(self.tempdir, ".repo")
-        self.manifest_dir = os.path.join(self.repodir, "manifests")
-        self.manifest_file = os.path.join(
-            self.repodir, manifest_xml.MANIFEST_FILE_NAME
-        )
-        self.local_manifest_dir = os.path.join(
-            self.repodir, manifest_xml.LOCAL_MANIFESTS_DIR_NAME
-        )
-        os.mkdir(self.repodir)
-        os.mkdir(self.manifest_dir)
-
-    def tearDown(self):
-        """Common teardown."""
-        rmtree(self.tempdir, ignore_errors=True)
-
-    def initTempGitTree(self, git_dir):
-        """Create a new empty git checkout for testing."""
-
-        # Tests need to assume, that main is default branch at init,
-        # which is not supported in config until 2.28.
-        cmd = ["git", "init", "-q"]
-        if git_command.git_require((2, 28, 0)):
-            cmd += ["--initial-branch=main"]
-        else:
-            # Use template dir for init
-            templatedir = os.path.join(self.tempdirobj.name, ".test-template")
-            os.makedirs(templatedir)
-            with open(os.path.join(templatedir, "HEAD"), "w") as fp:
-                fp.write("ref: refs/heads/main\n")
-            cmd += ["--template", templatedir]
-        cmd += [git_dir]
-        subprocess.check_call(cmd)
-
-    def getXmlManifestWith8Projects(self):
-        """Create and return a setup of 8 projects with enough dummy
-        files and setup to execute forall."""
-
-        # Set up a manifest git dir for parsing to work
-        gitdir = os.path.join(self.repodir, "manifests.git")
-        os.mkdir(gitdir)
-        with open(os.path.join(gitdir, "config"), "w") as fp:
-            fp.write(
-                """[remote "origin"]
-                    url = https://localhost:0/manifest
-                    verbose = false
-                """
-            )
-
-        # Add the manifest data
-        manifest_data = """
-                <manifest>
-                    <remote name="origin" fetch="http://localhost" />
-                    <default remote="origin" revision="refs/heads/main" />
-                    <project name="project1" path="tests/path1" />
-                    <project name="project2" path="tests/path2" />
-                    <project name="project3" path="tests/path3" />
-                    <project name="project4" path="tests/path4" />
-                    <project name="project5" path="tests/path5" />
-                    <project name="project6" path="tests/path6" />
-                    <project name="project7" path="tests/path7" />
-                    <project name="project8" path="tests/path8" />
-                </manifest>
-            """
-        with open(self.manifest_file, "w", encoding="utf-8") as fp:
-            fp.write(manifest_data)
-
-        # Set up 8 empty projects to match the manifest
-        for x in range(1, 9):
-            os.makedirs(
-                os.path.join(
-                    self.repodir, "projects/tests/path" + str(x) + ".git"
-                )
-            )
-            os.makedirs(
-                os.path.join(
-                    self.repodir, "project-objects/project" + str(x) + ".git"
-                )
-            )
-            git_path = os.path.join(self.tempdir, "tests/path" + str(x))
-            self.initTempGitTree(git_path)
-
-        return manifest_xml.XmlManifest(self.repodir, self.manifest_file)
-
-    # Use mock to capture stdout from the forall run
-    @unittest.mock.patch("sys.stdout", new_callable=StringIO)
-    def test_forall_all_projects_called_once(self, mock_stdout):
-        """Test that all projects get a command run once each."""
-
-        manifest_with_8_projects = self.getXmlManifestWith8Projects()
-
-        cmd = subcmds.forall.Forall()
-        cmd.manifest = manifest_with_8_projects
-
-        # Use echo project names as the test of forall
-        opts, args = cmd.OptionParser.parse_args(["-c", "echo $REPO_PROJECT"])
-        opts.verbose = False
-
-        # Mock to not have the Execute fail on remote check
-        with mock.patch.object(
-            project.Project, "GetRevisionId", return_value="refs/heads/main"
-        ):
-            # Run the forall command
-            cmd.Execute(opts, args)
-
-            # Verify that we got every project name in the prints
-            for x in range(1, 9):
-                self.assertIn("project" + str(x), mock_stdout.getvalue())
-
-            # Split the captured output into lines to count them
-            line_count = 0
-            for line in mock_stdout.getvalue().split("\n"):
-                # A commented out print to stderr as a reminder
-                # that stdout is mocked, include sys and uncomment if needed
-                # print(line, file=sys.stderr)
-                if len(line) > 0:
-                    line_count += 1
-
-            # Verify that we didn't get more lines than expected
-            assert line_count == 8

tests/test_subcmds_sync.py

@@ -355,30 +355,6 @@ class SafeCheckoutOrder(unittest.TestCase):
         )
 
 
-class Chunksize(unittest.TestCase):
-    """Tests for _chunksize."""
-
-    def test_single_project(self):
-        """Single project."""
-        self.assertEqual(sync._chunksize(1, 1), 1)
-
-    def test_low_project_count(self):
-        """Multiple projects, low number of projects to sync."""
-        self.assertEqual(sync._chunksize(10, 1), 10)
-        self.assertEqual(sync._chunksize(10, 2), 5)
-        self.assertEqual(sync._chunksize(10, 4), 2)
-        self.assertEqual(sync._chunksize(10, 8), 1)
-        self.assertEqual(sync._chunksize(10, 16), 1)
-
-    def test_high_project_count(self):
-        """Multiple projects, high number of projects to sync."""
-        self.assertEqual(sync._chunksize(2800, 1), 32)
-        self.assertEqual(sync._chunksize(2800, 16), 32)
-        self.assertEqual(sync._chunksize(2800, 32), 32)
-        self.assertEqual(sync._chunksize(2800, 64), 32)
-        self.assertEqual(sync._chunksize(2800, 128), 21)
-
-
 class GetPreciousObjectsState(unittest.TestCase):
     """Tests for _GetPreciousObjectsState."""
 

tests/test_wrapper.py

@@ -17,7 +17,6 @@
 import io
 import os
 import re
-import subprocess
 import sys
 import tempfile
 import unittest
@@ -73,11 +72,84 @@ class RepoWrapperUnitTest(RepoWrapperTestCase):
 
     def test_init_parser(self):
         """Make sure 'init' GetParser works."""
-        parser = self.wrapper.GetParser()
+        parser = self.wrapper.GetParser(gitc_init=False)
         opts, args = parser.parse_args([])
         self.assertEqual([], args)
         self.assertIsNone(opts.manifest_url)
 
+    def test_gitc_init_parser(self):
+        """Make sure 'gitc-init' GetParser raises."""
+        with self.assertRaises(SystemExit):
+            self.wrapper.GetParser(gitc_init=True)
+
+    def test_get_gitc_manifest_dir_no_gitc(self):
+        """
+        Test reading a missing gitc config file
+        """
+        self.wrapper.GITC_CONFIG_FILE = fixture("missing_gitc_config")
+        val = self.wrapper.get_gitc_manifest_dir()
+        self.assertEqual(val, "")
+
+    def test_get_gitc_manifest_dir(self):
+        """
+        Test reading the gitc config file and parsing the directory
+        """
+        self.wrapper.GITC_CONFIG_FILE = fixture("gitc_config")
+        val = self.wrapper.get_gitc_manifest_dir()
+        self.assertEqual(val, "/test/usr/local/google/gitc")
+
+    def test_gitc_parse_clientdir_no_gitc(self):
+        """
+        Test parsing the gitc clientdir without gitc running
+        """
+        self.wrapper.GITC_CONFIG_FILE = fixture("missing_gitc_config")
+        self.assertEqual(self.wrapper.gitc_parse_clientdir("/something"), None)
+        self.assertEqual(
+            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/test"), "test"
+        )
+
+    def test_gitc_parse_clientdir(self):
+        """
+        Test parsing the gitc clientdir
+        """
+        self.wrapper.GITC_CONFIG_FILE = fixture("gitc_config")
+        self.assertEqual(self.wrapper.gitc_parse_clientdir("/something"), None)
+        self.assertEqual(
+            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/test"), "test"
+        )
+        self.assertEqual(
+            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/test/"), "test"
+        )
+        self.assertEqual(
+            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/test/extra"),
+            "test",
+        )
+        self.assertEqual(
+            self.wrapper.gitc_parse_clientdir(
+                "/test/usr/local/google/gitc/test"
+            ),
+            "test",
+        )
+        self.assertEqual(
+            self.wrapper.gitc_parse_clientdir(
+                "/test/usr/local/google/gitc/test/"
+            ),
+            "test",
+        )
+        self.assertEqual(
+            self.wrapper.gitc_parse_clientdir(
+                "/test/usr/local/google/gitc/test/extra"
+            ),
+            "test",
+        )
+        self.assertEqual(
+            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/"), None
+        )
+        self.assertEqual(
+            self.wrapper.gitc_parse_clientdir("/test/usr/local/google/gitc/"),
+            None,
+        )
+
 
 class SetGitTrace2ParentSid(RepoWrapperTestCase):
     """Check SetGitTrace2ParentSid behavior."""
@@ -126,7 +198,7 @@ class RunCommand(RepoWrapperTestCase):
         self.wrapper.run_command(["true"], check=False)
         self.wrapper.run_command(["true"], check=True)
         self.wrapper.run_command(["false"], check=False)
-        with self.assertRaises(subprocess.CalledProcessError):
+        with self.assertRaises(self.wrapper.RunError):
             self.wrapper.run_command(["false"], check=True)
 
 
@@ -359,8 +431,8 @@ class VerifyRev(RepoWrapperTestCase):
 
     def test_verify_passes(self):
         """Check when we have a valid signed tag."""
-        desc_result = subprocess.CompletedProcess([], 0, "v1.0\n", "")
+        desc_result = self.wrapper.RunResult(0, "v1.0\n", "")
-        gpg_result = subprocess.CompletedProcess([], 0, "", "")
+        gpg_result = self.wrapper.RunResult(0, "", "")
         with mock.patch.object(
             self.wrapper, "run_git", side_effect=(desc_result, gpg_result)
         ):
@@ -371,8 +443,8 @@ class VerifyRev(RepoWrapperTestCase):
 
     def test_unsigned_commit(self):
         """Check we fall back to signed tag when we have an unsigned commit."""
-        desc_result = subprocess.CompletedProcess([], 0, "v1.0-10-g1234\n", "")
+        desc_result = self.wrapper.RunResult(0, "v1.0-10-g1234\n", "")
-        gpg_result = subprocess.CompletedProcess([], 0, "", "")
+        gpg_result = self.wrapper.RunResult(0, "", "")
         with mock.patch.object(
             self.wrapper, "run_git", side_effect=(desc_result, gpg_result)
         ):
@@ -383,7 +455,7 @@ class VerifyRev(RepoWrapperTestCase):
 
     def test_verify_fails(self):
         """Check we fall back to signed tag when we have an unsigned commit."""
-        desc_result = subprocess.CompletedProcess([], 0, "v1.0-10-g1234\n", "")
+        desc_result = self.wrapper.RunResult(0, "v1.0-10-g1234\n", "")
         gpg_result = Exception
         with mock.patch.object(
             self.wrapper, "run_git", side_effect=(desc_result, gpg_result)