gc: Introduce new command to remove old projects

When projects are removed from manifest, they are only removed from
worktree and not from .repo/projects and .repo/project-objects. Keeping
data under .repo can be desired if user expects deleted projects to be
restored (e.g. checking out a release branch).

Android has ongoing effort to remove many stale projects and this change
allows users to easily free-up their disk space.

Bug: b/344018971
Bug: 40013312
Change-Id: Id23c7524a88082ee6db908f9fd69dcd5d0c4f681
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/445921
Reviewed-by: Mike Frysinger <vapier@google.com>
Commit-Queue: Josip Sokcevic <sokcevic@chromium.org>
Reviewed-by: Gavin Mak <gavinmak@google.com>
Tested-by: Josip Sokcevic <sokcevic@chromium.org>

command.py

@@ -547,7 +547,3 @@ class MirrorSafeCommand:
     """Command permits itself to run within a mirror, and does not require a
     working directory.
     """
-
-
-class GitcClientCommand:
-    """Command that requires the local client to be a GITC client."""

error.py

@@ -111,10 +111,6 @@ class GitAuthError(RepoExitError):
     """Cannot talk to remote due to auth issue."""
 
 
-class GitcUnsupportedError(RepoExitError):
-    """Gitc no longer supported."""
-
-
 class UploadError(RepoError):
     """A bundle upload to Gerrit did not succeed."""
 

git_trace2_event_log_base.py

@@ -130,10 +130,10 @@ class BaseEventLog:
             "time": datetime.datetime.now(datetime.timezone.utc).isoformat(),
         }
 
-    def StartEvent(self):
+    def StartEvent(self, argv):
         """Append a 'start' event to the current log."""
         start_event = self._CreateEventDict("start")
-        start_event["argv"] = sys.argv
+        start_event["argv"] = argv
         self._log.append(start_event)
 
     def ExitEvent(self, result):
@@ -159,9 +159,11 @@ class BaseEventLog:
             name: Name of the primary command (ex: repo, git)
             subcommands: List of the sub-commands (ex: version, init, sync)
         """
-        command_event = self._CreateEventDict("command")
+        command_event = self._CreateEventDict("cmd_name")
+        name = f"{name}-"
+        name += "-".join(subcommands)
         command_event["name"] = name
-        command_event["subcommands"] = subcommands
+        command_event["hierarchy"] = name
         self._log.append(command_event)
 
     def LogConfigEvents(self, config, event_dict_name):

main.py

@@ -45,7 +45,6 @@ from command import InteractiveCommand
 from command import MirrorSafeCommand
 from editor import Editor
 from error import DownloadError
-from error import GitcUnsupportedError
 from error import InvalidProjectGroupsError
 from error import ManifestInvalidRevisionError
 from error import ManifestParseError
@@ -308,10 +307,6 @@ class _Repo:
                 outer_client=outer_client,
             )
 
-        if Wrapper().gitc_parse_clientdir(os.getcwd()):
-            logger.error("GITC is not supported.")
-            raise GitcUnsupportedError()
-
         try:
             cmd = self.commands[name](
                 repodir=self.repodir,
@@ -357,7 +352,7 @@ class _Repo:
         start = time.time()
         cmd_event = cmd.event_log.Add(name, event_log.TASK_COMMAND, start)
         cmd.event_log.SetParent(cmd_event)
-        git_trace2_event_log.StartEvent()
+        git_trace2_event_log.StartEvent(["repo", name] + argv)
         git_trace2_event_log.CommandEvent(name="repo", subcommands=[name])
 
         def execute_command_helper():

man/repo-gc.1

@@ -0,0 +1,43 @@
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
+.TH REPO "1" "December 2024" "repo gc" "Repo Manual"
+.SH NAME
+repo \- repo gc - manual page for repo gc
+.SH SYNOPSIS
+.B repo
+\fI\,gc\/\fR
+.SH DESCRIPTION
+Summary
+.PP
+Cleaning up internal repo state.
+.SH OPTIONS
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+show this help message and exit
+.TP
+\fB\-n\fR, \fB\-\-dry\-run\fR
+do everything except actually delete
+.TP
+\fB\-y\fR, \fB\-\-yes\fR
+answer yes to all safe prompts
+.SS Logging options:
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+show all output
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+only show errors
+.SS Multi\-manifest options:
+.TP
+\fB\-\-outer\-manifest\fR
+operate starting at the outermost manifest
+.TP
+\fB\-\-no\-outer\-manifest\fR
+do not operate on outer manifests
+.TP
+\fB\-\-this\-manifest\-only\fR
+only operate on this (sub)manifest
+.TP
+\fB\-\-no\-this\-manifest\-only\fR, \fB\-\-all\-manifests\fR
+operate on this manifest and its submanifests
+.PP
+Run `repo help gc` to view the detailed manual.

man/repo-manifest.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "April 2024" "repo manifest" "Repo Manual"
+.TH REPO "1" "December 2024" "repo manifest" "Repo Manual"
 .SH NAME
 repo \- repo manifest - manual page for repo manifest
 .SH SYNOPSIS
@@ -192,11 +192,13 @@ CDATA #IMPLIED>
 <!ATTLIST extend\-project remote CDATA #IMPLIED>
 <!ATTLIST extend\-project dest\-branch CDATA #IMPLIED>
 <!ATTLIST extend\-project upstream CDATA #IMPLIED>
+<!ATTLIST extend\-project base\-rev CDATA #IMPLIED>
 .IP
 <!ELEMENT remove\-project EMPTY>
 <!ATTLIST remove\-project name     CDATA #IMPLIED>
 <!ATTLIST remove\-project path     CDATA #IMPLIED>
 <!ATTLIST remove\-project optional CDATA #IMPLIED>
+<!ATTLIST remove\-project base\-rev CDATA #IMPLIED>
 .IP
 <!ELEMENT repo\-hooks EMPTY>
 <!ATTLIST repo\-hooks in\-project CDATA #REQUIRED>
@@ -495,6 +497,14 @@ project. Same syntax as the corresponding element of `project`.
 Attribute `upstream`: If specified, overrides the upstream of the original
 project. Same syntax as the corresponding element of `project`.
 .PP
+Attribute `base\-rev`: If specified, adds a check against the revision to be
+extended. Manifest parse will fail and give a list of mismatch extends if the
+revisions being extended have changed since base\-rev was set. Intended for use
+with layered manifests using hash revisions to prevent patch branches hiding
+newer upstream revisions. Also compares named refs like branches or tags but is
+misleading if branches are used as base\-rev. Same syntax as the corresponding
+element of `project`.
+.PP
 Element annotation
 .PP
 Zero or more annotation elements may be specified as children of a project or
@@ -556,6 +566,14 @@ Logic otherwise behaves like both are specified.
 Attribute `optional`: Set to true to ignore remove\-project elements with no
 matching `project` element.
 .PP
+Attribute `base\-rev`: If specified, adds a check against the revision to be
+removed. Manifest parse will fail and give a list of mismatch removes if the
+revisions being removed have changed since base\-rev was set. Intended for use
+with layered manifests using hash revisions to prevent patch branches hiding
+newer upstream revisions. Also compares named refs like branches or tags but is
+misleading if branches are used as base\-rev. Same syntax as the corresponding
+element of `project`.
+.PP
 Element repo\-hooks
 .PP
 NB: See the [practical documentation](./repo\-hooks.md) for using repo hooks.

man/repo.1

@@ -1,5 +1,5 @@
 .\" DO NOT MODIFY THIS FILE!  It was generated by help2man.
-.TH REPO "1" "April 2024" "repo" "Repo Manual"
+.TH REPO "1" "December 2024" "repo" "Repo Manual"
 .SH NAME
 repo \- repository management tool built on top of git
 .SH SYNOPSIS
@@ -79,6 +79,9 @@ Download and checkout a change
 forall
 Run a shell command in each project
 .TP
+gc
+Cleaning up internal repo state.
+.TP
 grep
 Print lines matching a pattern
 .TP

project.py

@@ -576,7 +576,6 @@ class Project:
         dest_branch=None,
         optimized_fetch=False,
         retry_fetches=0,
-        old_revision=None,
     ):
         """Init a Project object.
 
@@ -609,7 +608,6 @@ class Project:
                 only fetch from the remote if the sha1 is not present locally.
             retry_fetches: Retry remote fetches n times upon receiving transient
                 error with exponential backoff and jitter.
-            old_revision: saved git commit id for open GITC projects.
         """
         self.client = self.manifest = manifest
         self.name = name
@@ -639,7 +637,6 @@ class Project:
         self.linkfiles = []
         self.annotations = []
         self.dest_branch = dest_branch
-        self.old_revision = old_revision
 
         # This will be filled in if a project is later identified to be the
         # project containing repo hooks.

repo

@@ -124,7 +124,7 @@ if not REPO_REV:
 BUG_URL = "https://issues.gerritcodereview.com/issues/new?component=1370071"
 
 # increment this whenever we make important changes to this script
-VERSION = (2, 48)
+VERSION = (2, 50)
 
 # increment this if the MAINTAINER_KEYS block is modified
 KEYRING_VERSION = (2, 3)
@@ -215,8 +215,6 @@ repodir = ".repo"  # name of repo's private directory
 S_repo = "repo"  # special repo repository
 S_manifests = "manifests"  # special manifest repository
 REPO_MAIN = S_repo + "/main.py"  # main script
-GITC_CONFIG_FILE = "/gitc/.config"
-GITC_FS_ROOT_DIR = "/gitc/manifest-rw/"
 
 
 import collections
@@ -235,12 +233,9 @@ home_dot_repo = os.path.join(repo_config_dir, ".repoconfig")
 gpg_dir = os.path.join(home_dot_repo, "gnupg")
 
 
-def GetParser(gitc_init=False):
+def GetParser():
     """Setup the CLI parser."""
-    if gitc_init:
-        sys.exit("repo: fatal: GITC not supported.")
-    else:
-        usage = "repo init [options] [-u] url"
+    usage = "repo init [options] [-u] url"
 
     parser = optparse.OptionParser(usage=usage)
     InitParser(parser)
@@ -557,49 +552,6 @@ def run_command(cmd, **kwargs):
     return ret
 
 
-_gitc_manifest_dir = None
-
-
-def get_gitc_manifest_dir():
-    global _gitc_manifest_dir
-    if _gitc_manifest_dir is None:
-        _gitc_manifest_dir = ""
-        try:
-            with open(GITC_CONFIG_FILE) as gitc_config:
-                for line in gitc_config:
-                    match = re.match("gitc_dir=(?P<gitc_manifest_dir>.*)", line)
-                    if match:
-                        _gitc_manifest_dir = match.group("gitc_manifest_dir")
-        except OSError:
-            pass
-    return _gitc_manifest_dir
-
-
-def gitc_parse_clientdir(gitc_fs_path):
-    """Parse a path in the GITC FS and return its client name.
-
-    Args:
-        gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR.
-
-    Returns:
-        The GITC client name.
-    """
-    if gitc_fs_path == GITC_FS_ROOT_DIR:
-        return None
-    if not gitc_fs_path.startswith(GITC_FS_ROOT_DIR):
-        manifest_dir = get_gitc_manifest_dir()
-        if manifest_dir == "":
-            return None
-        if manifest_dir[-1] != "/":
-            manifest_dir += "/"
-        if gitc_fs_path == manifest_dir:
-            return None
-        if not gitc_fs_path.startswith(manifest_dir):
-            return None
-        return gitc_fs_path.split(manifest_dir)[1].split("/")[0]
-    return gitc_fs_path.split(GITC_FS_ROOT_DIR)[1].split("/")[0]
-
-
 class CloneFailure(Exception):
 
     """Indicate the remote clone of repo itself failed."""
@@ -638,9 +590,9 @@ def check_repo_rev(dst, rev, repo_verify=True, quiet=False):
     return (remote_ref, rev)
 
 
-def _Init(args, gitc_init=False):
+def _Init(args):
     """Installs repo by cloning it over the network."""
-    parser = GetParser(gitc_init=gitc_init)
+    parser = GetParser()
     opt, args = parser.parse_args(args)
     if args:
         if not opt.manifest_url:
@@ -1164,7 +1116,7 @@ class _Options:
 def _ExpandAlias(name):
     """Look up user registered aliases."""
     # We don't resolve aliases for existing subcommands.  This matches git.
-    if name in {"gitc-init", "help", "init"}:
+    if name in {"help", "init"}:
         return name, []
 
     alias = _GetRepoConfig(f"alias.{name}")
@@ -1292,10 +1244,6 @@ class Requirements:
 
 
 def _Usage():
-    gitc_usage = ""
-    if get_gitc_manifest_dir():
-        gitc_usage = "  gitc-init Initialize a GITC Client.\n"
-
     print(
         """usage: repo COMMAND [ARGS]
 
@@ -1304,9 +1252,7 @@ repo is not yet installed.  Use "repo init" to install it here.
 The most commonly used repo commands are:
 
   init      Install repo in the current working directory
-"""
-        + gitc_usage
-        + """  help      Display detailed help on a command
+  help      Display detailed help on a command
 
 For access to the full online help, install repo ("repo init").
 """
@@ -1317,8 +1263,8 @@ For access to the full online help, install repo ("repo init").
 
 def _Help(args):
     if args:
-        if args[0] in {"init", "gitc-init"}:
-            parser = GetParser(gitc_init=args[0] == "gitc-init")
+        if args[0] in {"init"}:
+            parser = GetParser()
             parser.print_help()
             sys.exit(0)
         else:
@@ -1335,10 +1281,11 @@ def _Help(args):
 
 def _Version():
     """Show version information."""
+    git_version = ParseGitVersion()
     print("<repo not installed>")
     print(f"repo launcher version {'.'.join(str(x) for x in VERSION)}")
     print(f"       (from {__file__})")
-    print(f"git {ParseGitVersion().full}")
+    print(f"git {git_version.full}" if git_version else "git not installed")
     print(f"Python {sys.version}")
     uname = platform.uname()
     print(f"OS {uname.system} {uname.release} ({uname.version})")
@@ -1371,11 +1318,11 @@ def _RunSelf(wrapper_path):
     my_main = os.path.join(my_dir, "main.py")
     my_git = os.path.join(my_dir, ".git")
 
-    if os.path.isfile(my_main) and os.path.isdir(my_git):
+    if os.path.isfile(my_main):
         for name in ["git_config.py", "project.py", "subcmds"]:
             if not os.path.exists(os.path.join(my_dir, name)):
                 return None, None
-        return my_main, my_git
+        return my_main, my_git if os.path.isdir(my_git) else None
     return None, None
 
 
@@ -1406,23 +1353,11 @@ def main(orig_args):
     # We run this early as we run some git commands ourselves.
     SetGitTrace2ParentSid()
 
-    repo_main, rel_repo_dir = None, None
-    # Don't use the local repo copy, make sure to switch to the gitc client first.
-    if cmd != "gitc-init":
-        repo_main, rel_repo_dir = _FindRepo()
+    repo_main, rel_repo_dir = _FindRepo()
 
     wrapper_path = os.path.abspath(__file__)
     my_main, my_git = _RunSelf(wrapper_path)
 
-    cwd = os.getcwd()
-    if get_gitc_manifest_dir() and cwd.startswith(get_gitc_manifest_dir()):
-        print(
-            "error: repo cannot be used in the GITC local manifest directory."
-            "\nIf you want to work on this GITC client please rerun this "
-            "command from the corresponding client under /gitc/",
-            file=sys.stderr,
-        )
-        sys.exit(1)
     if not repo_main:
         # Only expand aliases here since we'll be parsing the CLI ourselves.
         # If we had repo_main, alias expansion would happen in main.py.
@@ -1437,11 +1372,11 @@ def main(orig_args):
             _Version()
         if not cmd:
             _NotInstalled()
-        if cmd == "init" or cmd == "gitc-init":
+        if cmd == "init":
             if my_git:
                 _SetDefaultsTo(my_git)
             try:
-                _Init(args, gitc_init=(cmd == "gitc-init"))
+                _Init(args)
             except CloneFailure:
                 path = os.path.join(repodir, S_repo)
                 print(

subcmds/branches.py

@@ -167,7 +167,10 @@ is shown, then the branch appears in all projects.
             else:
                 published = " "
 
-            hdr("%c%c %-*s" % (current, published, width, name))
+            # A branch name can contain a percent sign, so we need to escape it.
+            # Escape after f-string formatting to properly account for leading
+            # spaces.
+            hdr(f"{current}{published} {name:{width}}".replace("%", "%%"))
             out.write(" |")
 
             _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only)

subcmds/gc.py

@@ -0,0 +1,127 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from typing import Set
+
+from command import Command
+import platform_utils
+from progress import Progress
+
+
+class Gc(Command):
+    COMMON = True
+    helpSummary = "Cleaning up internal repo state."
+    helpUsage = """
+%prog
+"""
+
+    def _Options(self, p):
+        p.add_option(
+            "-n",
+            "--dry-run",
+            dest="dryrun",
+            default=False,
+            action="store_true",
+            help="do everything except actually delete",
+        )
+        p.add_option(
+            "-y",
+            "--yes",
+            default=False,
+            action="store_true",
+            help="answer yes to all safe prompts",
+        )
+
+    def _find_git_to_delete(
+        self, to_keep: Set[str], start_dir: str
+    ) -> Set[str]:
+        """Searches no longer needed ".git" directories.
+
+        Scans the file system starting from `start_dir` and removes all
+        directories that end with ".git" that are not in the `to_keep` set.
+        """
+        to_delete = set()
+        for root, dirs, _ in platform_utils.walk(start_dir):
+            for directory in dirs:
+                if not directory.endswith(".git"):
+                    continue
+
+                path = os.path.join(root, directory)
+                if path not in to_keep:
+                    to_delete.add(path)
+
+        return to_delete
+
+    def Execute(self, opt, args):
+        projects = self.GetProjects(
+            args, all_manifests=not opt.this_manifest_only
+        )
+        print(f"Scanning filesystem under {self.repodir}...")
+
+        project_paths = set()
+        project_object_paths = set()
+
+        for project in projects:
+            project_paths.add(project.gitdir)
+            project_object_paths.add(project.objdir)
+
+        to_delete = self._find_git_to_delete(
+            project_paths, os.path.join(self.repodir, "projects")
+        )
+
+        to_delete.update(
+            self._find_git_to_delete(
+                project_object_paths,
+                os.path.join(self.repodir, "project-objects"),
+            )
+        )
+
+        if not to_delete:
+            print("Nothing to clean up.")
+            return
+
+        print("Identified the following projects are no longer used:")
+        print("\n".join(to_delete))
+        print("\n")
+        if not opt.yes:
+            print(
+                "If you proceed, any local commits in those projects will be "
+                "destroyed!"
+            )
+            ask = input("Proceed? [y/N] ")
+            if ask.lower() != "y":
+                return 1
+
+        pm = Progress(
+            "Deleting",
+            len(to_delete),
+            delay=False,
+            quiet=opt.quiet,
+            show_elapsed=True,
+            elide=True,
+        )
+
+        for path in to_delete:
+            if opt.dryrun:
+                print(f"\nWould have deleted ${path}")
+            else:
+                tmp_path = os.path.join(
+                    os.path.dirname(path),
+                    f"to_be_deleted_{os.path.basename(path)}",
+                )
+                platform_utils.rename(path, tmp_path)
+                platform_utils.rmtree(tmp_path)
+            pm.update(msg=path)
+        pm.end()

subcmds/sync.py

@@ -821,6 +821,16 @@ later is required to fix a server side protocol bug.
         jobs = jobs_str(len(items))
         return f"{jobs} | {elapsed_str(elapsed)} {earliest_proj}"
 
+    @classmethod
+    def InitWorker(cls):
+        # Force connect to the manager server now.
+        # This is good because workers are initialized one by one. Without this,
+        # multiple workers may connect to the manager when handling the first
+        # job at the same time. Then the connection may fail if too many
+        # connections are pending and execeeded the socket listening backlog,
+        # especially on MacOS.
+        len(cls.get_parallel_context()["sync_dict"])
+
     def _Fetch(self, projects, opt, err_event, ssh_proxy, errors):
         ret = True
 
@@ -891,7 +901,7 @@ later is required to fix a server side protocol bug.
                 objdir_project_map.setdefault(project.objdir, []).append(index)
             projects_list = list(objdir_project_map.values())
 
-            jobs = min(opt.jobs_network, len(projects_list))
+            jobs = max(1, min(opt.jobs_network, len(projects_list)))
 
             # We pass the ssh proxy settings via the class.  This allows
             # multiprocessing to pickle it up when spawning children.  We can't
@@ -913,6 +923,7 @@ later is required to fix a server side protocol bug.
                     # idle while other workers still have more than one job in
                     # their chunk queue.
                     chunksize=1,
+                    initializer=self.InitWorker,
                 )
             finally:
                 sync_event.set()
@@ -1431,7 +1442,10 @@ later is required to fix a server side protocol bug.
             for need_remove_file in need_remove_files:
                 # Try to remove the updated copyfile or linkfile.
                 # So, if the file is not exist, nothing need to do.
-                platform_utils.remove(need_remove_file, missing_ok=True)
+                platform_utils.remove(
+                    os.path.join(self.client.topdir, need_remove_file),
+                    missing_ok=True,
+                )
 
         # Create copy-link-files.json, save dest path of "copyfile" and
         # "linkfile".

subcmds/upload.py

@@ -737,8 +737,8 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
             pending = []
             for result in results:
                 project_idx, avail = result
+                project = projects[project_idx]
                 if avail is None:
-                    project = projects[project_idx]
                     logger.error(
                         'repo: error: %s: Unable to upload branch "%s". '
                         "You might be able to fix the branch by running:\n"
@@ -748,7 +748,7 @@ Gerrit Code Review:  https://www.gerritcodereview.com/
                         project.manifest.branch,
                     )
                 elif avail:
-                    pending.append(result)
+                    pending.append((project, avail))
             return pending
 
         with self.ParallelContext():

tests/fixtures/gitc_config

@@ -1 +0,0 @@
-gitc_dir=/test/usr/local/google/gitc

tests/test_git_trace2_event_log.py

@@ -150,7 +150,7 @@ class EventLogTestCase(unittest.TestCase):
         <version event>
         <start event>
         """
-        self._event_log_module.StartEvent()
+        self._event_log_module.StartEvent([])
         with tempfile.TemporaryDirectory(prefix="event_log_tests") as tempdir:
             log_path = self._event_log_module.Write(path=tempdir)
             self._log_data = self.readLog(log_path)
@@ -213,10 +213,8 @@ class EventLogTestCase(unittest.TestCase):
         <version event>
         <command event>
         """
-        name = "repo"
-        subcommands = ["init" "this"]
         self._event_log_module.CommandEvent(
-            name="repo", subcommands=subcommands
+            name="repo", subcommands=["init", "this"]
         )
         with tempfile.TemporaryDirectory(prefix="event_log_tests") as tempdir:
             log_path = self._event_log_module.Write(path=tempdir)
@@ -225,12 +223,10 @@ class EventLogTestCase(unittest.TestCase):
         self.assertEqual(len(self._log_data), 2)
         command_event = self._log_data[1]
         self.verifyCommonKeys(self._log_data[0], expected_event_name="version")
-        self.verifyCommonKeys(command_event, expected_event_name="command")
+        self.verifyCommonKeys(command_event, expected_event_name="cmd_name")
         # Check for 'command' event specific fields.
         self.assertIn("name", command_event)
-        self.assertIn("subcommands", command_event)
-        self.assertEqual(command_event["name"], name)
-        self.assertEqual(command_event["subcommands"], subcommands)
+        self.assertEqual(command_event["name"], "repo-init-this")
 
     def test_def_params_event_repo_config(self):
         """Test 'def_params' event data outputs only repo config keys.
@@ -382,17 +378,17 @@ class EventLogTestCase(unittest.TestCase):
             socket_path = os.path.join(tempdir, "server.sock")
             server_ready = threading.Condition()
             # Start "server" listening on Unix domain socket at socket_path.
+            server_thread = threading.Thread(
+                target=serverLoggingThread,
+                args=(socket_path, server_ready, received_traces),
+            )
             try:
-                server_thread = threading.Thread(
-                    target=serverLoggingThread,
-                    args=(socket_path, server_ready, received_traces),
-                )
                 server_thread.start()
 
                 with server_ready:
                     server_ready.wait(timeout=120)
 
-                self._event_log_module.StartEvent()
+                self._event_log_module.StartEvent([])
                 path = self._event_log_module.Write(
                     path=f"af_unix:{socket_path}"
                 )

tests/test_subcmds_forall.py

@@ -0,0 +1,156 @@
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unittests for the forall subcmd."""
+
+from io import StringIO
+import os
+from shutil import rmtree
+import subprocess
+import tempfile
+import unittest
+from unittest import mock
+
+import git_command
+import manifest_xml
+import project
+import subcmds
+
+
+class AllCommands(unittest.TestCase):
+    """Check registered all_commands."""
+
+    def setUp(self):
+        """Common setup."""
+        self.tempdirobj = tempfile.TemporaryDirectory(prefix="forall_tests")
+        self.tempdir = self.tempdirobj.name
+        self.repodir = os.path.join(self.tempdir, ".repo")
+        self.manifest_dir = os.path.join(self.repodir, "manifests")
+        self.manifest_file = os.path.join(
+            self.repodir, manifest_xml.MANIFEST_FILE_NAME
+        )
+        self.local_manifest_dir = os.path.join(
+            self.repodir, manifest_xml.LOCAL_MANIFESTS_DIR_NAME
+        )
+        os.mkdir(self.repodir)
+        os.mkdir(self.manifest_dir)
+
+    def tearDown(self):
+        """Common teardown."""
+        rmtree(self.tempdir, ignore_errors=True)
+
+    def initTempGitTree(self, git_dir):
+        """Create a new empty git checkout for testing."""
+
+        # Tests need to assume, that main is default branch at init,
+        # which is not supported in config until 2.28.
+        cmd = ["git", "init", "-q"]
+        if git_command.git_require((2, 28, 0)):
+            cmd += ["--initial-branch=main"]
+        else:
+            # Use template dir for init
+            templatedir = os.path.join(self.tempdirobj.name, ".test-template")
+            os.makedirs(templatedir)
+            with open(os.path.join(templatedir, "HEAD"), "w") as fp:
+                fp.write("ref: refs/heads/main\n")
+            cmd += ["--template", templatedir]
+        cmd += [git_dir]
+        subprocess.check_call(cmd)
+
+    def getXmlManifestWith8Projects(self):
+        """Create and return a setup of 8 projects with enough dummy
+        files and setup to execute forall."""
+
+        # Set up a manifest git dir for parsing to work
+        gitdir = os.path.join(self.repodir, "manifests.git")
+        os.mkdir(gitdir)
+        with open(os.path.join(gitdir, "config"), "w") as fp:
+            fp.write(
+                """[remote "origin"]
+                    url = https://localhost:0/manifest
+                    verbose = false
+                """
+            )
+
+        # Add the manifest data
+        manifest_data = """
+                <manifest>
+                    <remote name="origin" fetch="http://localhost" />
+                    <default remote="origin" revision="refs/heads/main" />
+                    <project name="project1" path="tests/path1" />
+                    <project name="project2" path="tests/path2" />
+                    <project name="project3" path="tests/path3" />
+                    <project name="project4" path="tests/path4" />
+                    <project name="project5" path="tests/path5" />
+                    <project name="project6" path="tests/path6" />
+                    <project name="project7" path="tests/path7" />
+                    <project name="project8" path="tests/path8" />
+                </manifest>
+            """
+        with open(self.manifest_file, "w", encoding="utf-8") as fp:
+            fp.write(manifest_data)
+
+        # Set up 8 empty projects to match the manifest
+        for x in range(1, 9):
+            os.makedirs(
+                os.path.join(
+                    self.repodir, "projects/tests/path" + str(x) + ".git"
+                )
+            )
+            os.makedirs(
+                os.path.join(
+                    self.repodir, "project-objects/project" + str(x) + ".git"
+                )
+            )
+            git_path = os.path.join(self.tempdir, "tests/path" + str(x))
+            self.initTempGitTree(git_path)
+
+        return manifest_xml.XmlManifest(self.repodir, self.manifest_file)
+
+    # Use mock to capture stdout from the forall run
+    @unittest.mock.patch("sys.stdout", new_callable=StringIO)
+    def test_forall_all_projects_called_once(self, mock_stdout):
+        """Test that all projects get a command run once each."""
+
+        manifest_with_8_projects = self.getXmlManifestWith8Projects()
+
+        cmd = subcmds.forall.Forall()
+        cmd.manifest = manifest_with_8_projects
+
+        # Use echo project names as the test of forall
+        opts, args = cmd.OptionParser.parse_args(["-c", "echo $REPO_PROJECT"])
+        opts.verbose = False
+
+        # Mock to not have the Execute fail on remote check
+        with mock.patch.object(
+            project.Project, "GetRevisionId", return_value="refs/heads/main"
+        ):
+            # Run the forall command
+            cmd.Execute(opts, args)
+
+            # Verify that we got every project name in the prints
+            for x in range(1, 9):
+                self.assertIn("project" + str(x), mock_stdout.getvalue())
+
+            # Split the captured output into lines to count them
+            line_count = 0
+            for line in mock_stdout.getvalue().split("\n"):
+                # A commented out print to stderr as a reminder
+                # that stdout is mocked, include sys and uncomment if needed
+                # print(line, file=sys.stderr)
+                if len(line) > 0:
+                    line_count += 1
+
+            # Verify that we didn't get more lines than expected
+            assert line_count == 8

tests/test_wrapper.py

@@ -72,84 +72,11 @@ class RepoWrapperUnitTest(RepoWrapperTestCase):
 
     def test_init_parser(self):
         """Make sure 'init' GetParser works."""
-        parser = self.wrapper.GetParser(gitc_init=False)
+        parser = self.wrapper.GetParser()
         opts, args = parser.parse_args([])
         self.assertEqual([], args)
         self.assertIsNone(opts.manifest_url)
 
-    def test_gitc_init_parser(self):
-        """Make sure 'gitc-init' GetParser raises."""
-        with self.assertRaises(SystemExit):
-            self.wrapper.GetParser(gitc_init=True)
-
-    def test_get_gitc_manifest_dir_no_gitc(self):
-        """
-        Test reading a missing gitc config file
-        """
-        self.wrapper.GITC_CONFIG_FILE = fixture("missing_gitc_config")
-        val = self.wrapper.get_gitc_manifest_dir()
-        self.assertEqual(val, "")
-
-    def test_get_gitc_manifest_dir(self):
-        """
-        Test reading the gitc config file and parsing the directory
-        """
-        self.wrapper.GITC_CONFIG_FILE = fixture("gitc_config")
-        val = self.wrapper.get_gitc_manifest_dir()
-        self.assertEqual(val, "/test/usr/local/google/gitc")
-
-    def test_gitc_parse_clientdir_no_gitc(self):
-        """
-        Test parsing the gitc clientdir without gitc running
-        """
-        self.wrapper.GITC_CONFIG_FILE = fixture("missing_gitc_config")
-        self.assertEqual(self.wrapper.gitc_parse_clientdir("/something"), None)
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/test"), "test"
-        )
-
-    def test_gitc_parse_clientdir(self):
-        """
-        Test parsing the gitc clientdir
-        """
-        self.wrapper.GITC_CONFIG_FILE = fixture("gitc_config")
-        self.assertEqual(self.wrapper.gitc_parse_clientdir("/something"), None)
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/test"), "test"
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/test/"), "test"
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/test/extra"),
-            "test",
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir(
-                "/test/usr/local/google/gitc/test"
-            ),
-            "test",
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir(
-                "/test/usr/local/google/gitc/test/"
-            ),
-            "test",
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir(
-                "/test/usr/local/google/gitc/test/extra"
-            ),
-            "test",
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir("/gitc/manifest-rw/"), None
-        )
-        self.assertEqual(
-            self.wrapper.gitc_parse_clientdir("/test/usr/local/google/gitc/"),
-            None,
-        )
-
 
 class SetGitTrace2ParentSid(RepoWrapperTestCase):
     """Check SetGitTrace2ParentSid behavior."""