From 97d2b2f7a087bfc695536ae9be962406d82152f2 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Thu, 22 Sep 2011 17:23:41 -0700 Subject: [PATCH] sync: Limit -j to file descriptors Each worker thread requires at least 3 file descriptors to run the forked 'git fetch' child to operate against the local repository. Mac OS X has the RLIMIT_NOFILE set to 256 by default, which means a sync -j128 often fails when the workers run out of pipes within the Python parent process. Change-Id: I2cdb14621b899424b079daf7969bc8c16b85b903 Signed-off-by: Shawn O. Pearce --- subcmds/sync.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/subcmds/sync.py b/subcmds/sync.py index 4689ac8b..93010c51 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -28,6 +28,14 @@ try: except ImportError: import dummy_threading as _threading +try: + import resource + def _rlimit_nofile(): + return resource.getrlimit(resource.RLIMIT_NOFILE) +except ImportError: + def _rlimit_nofile(): + return (256, 256) + from git_command import GIT from git_refs import R_HEADS from project import HEAD @@ -312,6 +320,10 @@ uncommitted changes are present' % project.relpath def Execute(self, opt, args): if opt.jobs: self.jobs = opt.jobs + if self.jobs > 1: + soft_limit, _ = _rlimit_nofile() + self.jobs = min(self.jobs, (soft_limit - 5) / 3) + if opt.network_only and opt.detach_head: print >>sys.stderr, 'error: cannot combine -n and -d' sys.exit(1)