sync: support --jobs to fetch projects simultaneously

This patch does two things for being compatibile with
those Python which are built without threading support:

1. As the Python document and Shawn suggested, import dummy_threading
   when the threading is not available.

2. Reserve the single threaded code and make it default.
   In cases the --jobs does not work properly with dummy_threading,
   we still have a safe fallback.

Change-Id: I40909ef8e9b5c22f315c0a1da9be38eed8b0a2dc
This commit is contained in:
Roy Lee 2010-05-09 04:32:08 +08:00 committed by Shawn O. Pearce
parent 6623b21e10
commit 18afd7f679

View File

@ -23,6 +23,11 @@ import sys
import time import time
import xmlrpclib import xmlrpclib
try:
import threading as _threading
except ImportError:
import dummy_threading as _threading
from git_command import GIT from git_command import GIT
from git_refs import R_HEADS from git_refs import R_HEADS
from project import HEAD from project import HEAD
@ -35,6 +40,7 @@ from project import SyncBuffer
from progress import Progress from progress import Progress
class Sync(Command, MirrorSafeCommand): class Sync(Command, MirrorSafeCommand):
jobs = 1
common = True common = True
helpSummary = "Update working tree to the latest revision" helpSummary = "Update working tree to the latest revision"
helpUsage = """ helpUsage = """
@ -104,6 +110,9 @@ later is required to fix a server side protocol bug.
p.add_option('-d','--detach', p.add_option('-d','--detach',
dest='detach_head', action='store_true', dest='detach_head', action='store_true',
help='detach projects back to manifest revision') help='detach projects back to manifest revision')
p.add_option('-j','--jobs',
dest='jobs', action='store', type='int',
help="number of projects to fetch simultaneously")
if show_smart: if show_smart:
p.add_option('-s', '--smart-sync', p.add_option('-s', '--smart-sync',
dest='smart_sync', action='store_true', dest='smart_sync', action='store_true',
@ -117,9 +126,23 @@ later is required to fix a server side protocol bug.
dest='repo_upgraded', action='store_true', dest='repo_upgraded', action='store_true',
help=SUPPRESS_HELP) help=SUPPRESS_HELP)
def _FetchHelper(self, project, lock, fetched, pm, sem):
if not project.Sync_NetworkHalf():
print >>sys.stderr, 'error: Cannot fetch %s' % project.name
sem.release()
sys.exit(1)
lock.acquire()
fetched.add(project.gitdir)
pm.update()
lock.release()
sem.release()
def _Fetch(self, projects): def _Fetch(self, projects):
fetched = set() fetched = set()
pm = Progress('Fetching projects', len(projects)) pm = Progress('Fetching projects', len(projects))
if self.jobs == 1:
for project in projects: for project in projects:
pm.update() pm.update()
if project.Sync_NetworkHalf(): if project.Sync_NetworkHalf():
@ -127,6 +150,20 @@ later is required to fix a server side protocol bug.
else: else:
print >>sys.stderr, 'error: Cannot fetch %s' % project.name print >>sys.stderr, 'error: Cannot fetch %s' % project.name
sys.exit(1) sys.exit(1)
else:
threads = set()
lock = _threading.Lock()
sem = _threading.Semaphore(self.jobs)
for project in projects:
sem.acquire()
t = _threading.Thread(target = self._FetchHelper,
args = (project, lock, fetched, pm, sem))
threads.add(t)
t.start()
for t in threads:
t.join()
pm.end() pm.end()
return fetched return fetched
@ -190,6 +227,8 @@ uncommitted changes are present' % project.relpath
return 0 return 0
def Execute(self, opt, args): def Execute(self, opt, args):
if opt.jobs:
self.jobs = opt.jobs
if opt.network_only and opt.detach_head: if opt.network_only and opt.detach_head:
print >>sys.stderr, 'error: cannot combine -n and -d' print >>sys.stderr, 'error: cannot combine -n and -d'
sys.exit(1) sys.exit(1)