diff --git a/command.py b/command.py index ef2554da..7737ec71 100644 --- a/command.py +++ b/command.py @@ -23,6 +23,11 @@ from error import NoSuchProjectError from error import InvalidProjectGroupsError +# How many jobs to run in parallel by default? This assumes the jobs are +# largely I/O bound and do not hit the network. +DEFAULT_LOCAL_JOBS = min(os.cpu_count(), 8) + + class Command(object): """Base class for any command line action in repo. """ @@ -32,6 +37,10 @@ class Command(object): manifest = None _optparse = None + # Whether this command supports running in parallel. If greater than 0, + # it is the number of parallel jobs to default to. + PARALLEL_JOBS = None + def WantPager(self, _opt): return False @@ -72,6 +81,11 @@ class Command(object): def _Options(self, p): """Initialize the option parser. """ + if self.PARALLEL_JOBS is not None: + p.add_option( + '-j', '--jobs', + type=int, default=self.PARALLEL_JOBS, + help='number of jobs to run in parallel (default: %s)' % self.PARALLEL_JOBS) def _RegisteredEnvironmentOptions(self): """Get options that can be set from environment variables. diff --git a/subcmds/branches.py b/subcmds/branches.py index 20f51693..9665e85f 100644 --- a/subcmds/branches.py +++ b/subcmds/branches.py @@ -16,7 +16,7 @@ import itertools import multiprocessing import sys from color import Coloring -from command import Command +from command import Command, DEFAULT_LOCAL_JOBS # Number of projects to submit to a single worker process at a time. # This number represents a tradeoff between the overhead of IPC and finer @@ -103,17 +103,7 @@ the branch appears in, or does not appear in. If no project list is shown, then the branch appears in all projects. """ - - def _Options(self, p): - """Add flags to CLI parser for this subcommand.""" - default_jobs = min(multiprocessing.cpu_count(), 8) - p.add_option( - '-j', - '--jobs', - type=int, - default=default_jobs, - help='Number of worker processes to spawn ' - '(default: %s)' % default_jobs) + PARALLEL_JOBS = DEFAULT_LOCAL_JOBS def Execute(self, opt, args): projects = self.GetProjects(args) diff --git a/subcmds/forall.py b/subcmds/forall.py index ef11851b..d871b3ea 100644 --- a/subcmds/forall.py +++ b/subcmds/forall.py @@ -21,7 +21,7 @@ import sys import subprocess from color import Coloring -from command import Command, MirrorSafeCommand +from command import DEFAULT_LOCAL_JOBS, Command, MirrorSafeCommand import platform_utils _CAN_COLOR = [ @@ -113,8 +113,11 @@ terminal and are not redirected. If -e is used, when a command exits unsuccessfully, '%prog' will abort without iterating through the remaining projects. """ + PARALLEL_JOBS = DEFAULT_LOCAL_JOBS def _Options(self, p): + super()._Options(p) + def cmd(option, opt_str, value, parser): setattr(parser.values, option.dest, list(parser.rargs)) while parser.rargs: @@ -148,9 +151,6 @@ without iterating through the remaining projects. g.add_option('-v', '--verbose', dest='verbose', action='store_true', help='Show command error messages') - g.add_option('-j', '--jobs', - dest='jobs', action='store', type='int', default=1, - help='number of commands to execute simultaneously') def WantPager(self, opt): return opt.project_header and opt.jobs == 1 diff --git a/subcmds/status.py b/subcmds/status.py index e293d75c..f0f2e034 100644 --- a/subcmds/status.py +++ b/subcmds/status.py @@ -17,7 +17,7 @@ import glob import multiprocessing import os -from command import PagedCommand +from command import DEFAULT_LOCAL_JOBS, PagedCommand from color import Coloring import platform_utils @@ -76,11 +76,10 @@ the following meanings: d: deleted ( in index, not in work tree ) """ + PARALLEL_JOBS = DEFAULT_LOCAL_JOBS def _Options(self, p): - p.add_option('-j', '--jobs', - dest='jobs', action='store', type='int', default=2, - help="number of projects to check simultaneously") + super()._Options(p) p.add_option('-o', '--orphans', dest='orphans', action='store_true', help="include objects in working directory outside of repo projects") diff --git a/subcmds/sync.py b/subcmds/sync.py index 5b1024df..18d2256e 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -177,12 +177,14 @@ If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or later is required to fix a server side protocol bug. """ + PARALLEL_JOBS = 1 def _Options(self, p, show_smart=True): try: - self.jobs = self.manifest.default.sync_j + self.PARALLEL_JOBS = self.manifest.default.sync_j except ManifestParseError: - self.jobs = 1 + pass + super()._Options(p) p.add_option('-f', '--force-broken', dest='force_broken', action='store_true', @@ -222,9 +224,6 @@ later is required to fix a server side protocol bug. p.add_option('-q', '--quiet', dest='output_mode', action='store_false', help='only show errors') - p.add_option('-j', '--jobs', - dest='jobs', action='store', type='int', - help="projects to fetch simultaneously (default %d)" % self.jobs) p.add_option('-m', '--manifest-name', dest='manifest_name', help='temporary manifest to use for this sync', metavar='NAME.xml')