mirror of
https://gerrit.googlesource.com/git-repo
synced 2024-12-21 07:16:21 +00:00
event_log: turn id generation from a generator to a func call
Running lots of sync processes in parallel can hit the failure: Fetching projects: 23% (124/523)Exception in thread Thread-201: Traceback (most recent call last): File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/src/repo/subcmds/sync.py", line 278, in _FetchProjectList success = self._FetchHelper(opt, project, *args, **kwargs) File "/usr/local/src/repo/subcmds/sync.py", line 357, in _FetchHelper start, finish, success) File "/usr/local/src/repo/event_log.py", line 104, in AddSync event = self.Add(project.relpath, task_name, start, finish, success) File "/usr/local/src/repo/event_log.py", line 74, in Add 'id': (kind, next(self._next_id)), ValueError: generator already executing It looks like, while we lock the multiprocessing value correctly, the generator that wraps the value isn't parallel safe. Since we don't have a way of doing that (as it's part of the language), turn it into a plain function call instead. Bug: https://crbug.com/gerrit/10293 Change-Id: I0db03601986ca0370a1699bab32adb03e7b2910a
This commit is contained in:
parent
12ee5446e9
commit
13f323b2c2
23
event_log.py
23
event_log.py
@ -51,7 +51,6 @@ class EventLog(object):
|
||||
def __init__(self):
|
||||
"""Initializes the event log."""
|
||||
self._log = []
|
||||
self._next_id = _EventIdGenerator()
|
||||
self._parent = None
|
||||
|
||||
def Add(self, name, task_name, start, finish=None, success=None,
|
||||
@ -71,7 +70,7 @@ class EventLog(object):
|
||||
A dictionary of the event added to the log.
|
||||
"""
|
||||
event = {
|
||||
'id': (kind, next(self._next_id)),
|
||||
'id': (kind, _NextEventId()),
|
||||
'name': name,
|
||||
'task_name': task_name,
|
||||
'start_time': start,
|
||||
@ -162,16 +161,16 @@ class EventLog(object):
|
||||
f.write('\n')
|
||||
|
||||
|
||||
def _EventIdGenerator():
|
||||
"""Returns multi-process safe iterator that generates locally unique id.
|
||||
# An integer id that is unique across this invocation of the program.
|
||||
_EVENT_ID = multiprocessing.Value('i', 1)
|
||||
|
||||
Yields:
|
||||
def _NextEventId():
|
||||
"""Helper function for grabbing the next unique id.
|
||||
|
||||
Returns:
|
||||
A unique, to this invocation of the program, integer id.
|
||||
"""
|
||||
eid = multiprocessing.Value('i', 1)
|
||||
|
||||
while True:
|
||||
with eid.get_lock():
|
||||
val = eid.value
|
||||
eid.value += 1
|
||||
yield val
|
||||
with _EVENT_ID.get_lock():
|
||||
val = _EVENT_ID.value
|
||||
_EVENT_ID.value += 1
|
||||
return val
|
||||
|
Loading…
Reference in New Issue
Block a user