From 49de8ef584a074982f3fd86aac8a144f48209807 Mon Sep 17 00:00:00 2001 From: Mike Frysinger Date: Fri, 9 Apr 2021 00:21:02 -0400 Subject: [PATCH] sync: add separate --jobs options for different steps The number of jobs one wants to run against the network tends to factor differently from the number of jobs one wants to run when checking out local projects. The former is constrained by your internet connection & server limits while the later is constrained by your local computer's CPU & storage I/O. People with beefier computers probably want to keep the network/server jobs bounded a bit lower than the local/checkout jobs. Change-Id: Ia27ab682c62c09d244a8a1427b1c65acf0116c1c Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/302804 Reviewed-by: Raman Tenneti Tested-by: Mike Frysinger --- subcmds/sync.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/subcmds/sync.py b/subcmds/sync.py index b41b20b7..4bcd45d5 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -174,6 +174,11 @@ later is required to fix a server side protocol bug. pass super()._Options(p) + p.add_option('--jobs-network', default=None, type=int, metavar='JOBS', + help='number of network jobs to run in parallel (defaults to --jobs)') + p.add_option('--jobs-checkout', default=None, type=int, metavar='JOBS', + help='number of local checkout jobs to run in parallel (defaults to --jobs)') + p.add_option('-f', '--force-broken', dest='force_broken', action='store_true', help='obsolete option (to be deleted in the future)') @@ -364,6 +369,7 @@ later is required to fix a server side protocol bug. def _Fetch(self, projects, opt, err_event): ret = True + jobs = opt.jobs_network if opt.jobs_network else self.jobs fetched = set() pm = Progress('Fetching', len(projects), delay=False) @@ -391,7 +397,7 @@ later is required to fix a server side protocol bug. return ret # NB: Multiprocessing is heavy, so don't spin it up for one job. - if len(projects_list) == 1 or opt.jobs == 1: + if len(projects_list) == 1 or jobs == 1: if not _ProcessResults(self._FetchProjectList(opt, x) for x in projects_list): ret = False else: @@ -409,7 +415,7 @@ later is required to fix a server side protocol bug. else: pm.update(inc=0, msg='warming up') chunksize = 4 - with multiprocessing.Pool(opt.jobs) as pool: + with multiprocessing.Pool(jobs) as pool: results = pool.imap_unordered( functools.partial(self._FetchProjectList, opt), projects_list, @@ -463,6 +469,7 @@ later is required to fix a server side protocol bug. err_results: A list of strings, paths to git repos where checkout failed. """ ret = True + jobs = opt.jobs_checkout if opt.jobs_checkout else self.jobs # Only checkout projects with worktrees. all_projects = [x for x in all_projects if x.worktree] @@ -483,11 +490,11 @@ later is required to fix a server side protocol bug. return True # NB: Multiprocessing is heavy, so don't spin it up for one job. - if len(all_projects) == 1 or opt.jobs == 1: + if len(all_projects) == 1 or jobs == 1: if not _ProcessResults(self._CheckoutOne(opt, x) for x in all_projects): ret = False else: - with multiprocessing.Pool(opt.jobs) as pool: + with multiprocessing.Pool(jobs) as pool: results = pool.imap_unordered( functools.partial(self._CheckoutOne, opt), all_projects,