Compare commits

...

5 Commits

Author SHA1 Message Date
13f323b2c2 event_log: turn id generation from a generator to a func call
Running lots of sync processes in parallel can hit the failure:
Fetching projects:  23% (124/523)Exception in thread Thread-201:
Traceback (most recent call last):
  File "/usr/lib/python2.7/threading.py", line 801, in __bootstrap_inner
    self.run()
  File "/usr/lib/python2.7/threading.py", line 754, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/usr/local/src/repo/subcmds/sync.py", line 278, in _FetchProjectList
    success = self._FetchHelper(opt, project, *args, **kwargs)
  File "/usr/local/src/repo/subcmds/sync.py", line 357, in _FetchHelper
    start, finish, success)
  File "/usr/local/src/repo/event_log.py", line 104, in AddSync
    event = self.Add(project.relpath, task_name, start, finish, success)
  File "/usr/local/src/repo/event_log.py", line 74, in Add
    'id': (kind, next(self._next_id)),
ValueError: generator already executing

It looks like, while we lock the multiprocessing value correctly, the
generator that wraps the value isn't parallel safe.  Since we don't
have a way of doing that (as it's part of the language), turn it into
a plain function call instead.

Bug: https://crbug.com/gerrit/10293
Change-Id: I0db03601986ca0370a1699bab32adb03e7b2910a
2019-01-14 16:11:08 -05:00
12ee5446e9 init: Remove -c short option for --current-branch
This option conflicts with the gitc-init -c short option.

Bug: https://crbug.com/gerrit/10200
Change-Id: I06f37564429ca0bd4c0bbea6066daae4f663c838
2018-12-20 19:55:02 +00:00
e158e3802d Merge "README: link in new bug tracker" 2018-12-20 07:21:02 +00:00
3bbbcaf99d README: link in new bug tracker
Change-Id: I043afc0b77e709919e49ce548dff47776fddaddf
2018-12-20 02:11:46 -05:00
d4b13c280b Leverage the next keyword from python 2.7
This is literally what the next keyword is for.
https://www.python.org/dev/peps/pep-3114/

Change-Id: I843755910b847737b077ff2361ba3e04409db0f0
2018-12-19 11:06:35 -08:00
5 changed files with 24 additions and 24 deletions

View File

@ -5,6 +5,6 @@
<pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
<path>/git-repo</path>
</pydev_pathproperty>
<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.6</pydev_property>
<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
</pydev_project>

View File

@ -7,7 +7,7 @@ easier to work with Git. The repo command is an executable Python script
that you can put anywhere in your path.
* Homepage: https://gerrit.googlesource.com/git-repo/
* Bug reports: https://code.google.com/p/git-repo/issues/
* Bug reports: https://bugs.chromium.org/p/gerrit/issues/list?q=component:repo
* Source: https://gerrit.googlesource.com/git-repo/
* Overview: https://source.android.com/source/developing.html
* Docs: https://source.android.com/source/using-repo.html

View File

@ -18,8 +18,6 @@ from __future__ import print_function
import json
import multiprocessing
from pyversion import is_python3
TASK_COMMAND = 'command'
TASK_SYNC_NETWORK = 'sync-network'
TASK_SYNC_LOCAL = 'sync-local'
@ -53,7 +51,6 @@ class EventLog(object):
def __init__(self):
"""Initializes the event log."""
self._log = []
self._next_id = _EventIdGenerator()
self._parent = None
def Add(self, name, task_name, start, finish=None, success=None,
@ -73,7 +70,7 @@ class EventLog(object):
A dictionary of the event added to the log.
"""
event = {
'id': (kind, self._next_id.__next__() if is_python3() else self._next_id.next()),
'id': (kind, _NextEventId()),
'name': name,
'task_name': task_name,
'start_time': start,
@ -164,16 +161,16 @@ class EventLog(object):
f.write('\n')
def _EventIdGenerator():
"""Returns multi-process safe iterator that generates locally unique id.
# An integer id that is unique across this invocation of the program.
_EVENT_ID = multiprocessing.Value('i', 1)
Yields:
def _NextEventId():
"""Helper function for grabbing the next unique id.
Returns:
A unique, to this invocation of the program, integer id.
"""
eid = multiprocessing.Value('i', 1)
while True:
with eid.get_lock():
val = eid.value
eid.value += 1
yield val
with _EVENT_ID.get_lock():
val = _EVENT_ID.value
_EVENT_ID.value += 1
return val

17
repo
View File

@ -23,7 +23,7 @@ REPO_REV = 'stable'
# limitations under the License.
# increment this whenever we make important changes to this script
VERSION = (1, 24)
VERSION = (1, 25)
# increment this if the MAINTAINER_KEYS block is modified
KEYRING_VERSION = (1, 2)
@ -113,7 +113,7 @@ repodir = '.repo' # name of repo's private directory
S_repo = 'repo' # special repo repository
S_manifests = 'manifests' # special manifest repository
REPO_MAIN = S_repo + '/main.py' # main script
MIN_PYTHON_VERSION = (2, 6) # minimum supported python version
MIN_PYTHON_VERSION = (2, 7) # minimum supported python version
GITC_CONFIG_FILE = '/gitc/.config'
GITC_FS_ROOT_DIR = '/gitc/manifest-rw/'
@ -152,9 +152,12 @@ def _print(*objects, **kwargs):
# Python version check
ver = sys.version_info
if (ver[0], ver[1]) < MIN_PYTHON_VERSION:
_print('error: Python version %s unsupported.\n'
'Please use Python 2.6 - 2.7 instead.'
% sys.version.split(' ')[0], file=sys.stderr)
_print('error: Python version {} unsupported.\n'
'Please use Python {}.{} instead.'.format(
sys.version.split(' ')[0],
MIN_PYTHON_VERSION[0],
MIN_PYTHON_VERSION[1],
), file=sys.stderr)
sys.exit(1)
home_dot_repo = os.path.expanduser('~/.repoconfig')
@ -180,7 +183,7 @@ group.add_option('-b', '--manifest-branch',
group.add_option('-m', '--manifest-name',
dest='manifest_name',
help='initial manifest file', metavar='NAME.xml')
group.add_option('-c', '--current-branch',
group.add_option('--current-branch',
dest='current_branch_only', action='store_true',
help='fetch only current manifest branch from server')
group.add_option('--mirror',
@ -909,6 +912,6 @@ def main(orig_args):
if __name__ == '__main__':
if ver[0] == 3:
_print('warning: Python 3 support is currently experimental. YMMV.\n'
'Please use Python 2.6 - 2.7 instead.',
'Please use Python 2.7 instead.',
file=sys.stderr)
main(sys.argv[1:])

View File

@ -95,7 +95,7 @@ to update the working directory files.
g.add_option('-b', '--manifest-branch',
dest='manifest_branch',
help='manifest branch or revision', metavar='REVISION')
g.add_option('-c', '--current-branch',
g.add_option('--current-branch',
dest='current_branch_only', action='store_true',
help='fetch only current manifest branch from server')
g.add_option('-m', '--manifest-name',