mirror of
https://gerrit.googlesource.com/git-repo
synced 2024-12-21 07:16:21 +00:00
b2fa30a2b8
This avoids GIL limitations with using threads for parallel processing. This reworks the fetch logic to return results for processing in the main thread instead of leaving every thread to do its own processing. We have to tweak the chunking logic a little here because multiprocessing favors batching over returning immediate results when using a larger value for chunksize. When a single job can be quite slow, this tradeoff is not good UX. Bug: https://crbug.com/gerrit/12389 Change-Id: I0f0512d15ad7332d1eb28aff52c29d378acc9e1d Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/298642 Reviewed-by: Chris Mcdonald <cjmcdonald@google.com> Tested-by: Mike Frysinger <vapier@google.com>
124 lines
3.4 KiB
Python
124 lines
3.4 KiB
Python
# Copyright (C) 2009 The Android Open Source Project
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import os
|
|
import sys
|
|
from time import time
|
|
from repo_trace import IsTrace
|
|
|
|
_NOT_TTY = not os.isatty(2)
|
|
|
|
# This will erase all content in the current line (wherever the cursor is).
|
|
# It does not move the cursor, so this is usually followed by \r to move to
|
|
# column 0.
|
|
CSI_ERASE_LINE = '\x1b[2K'
|
|
|
|
|
|
def duration_str(total):
|
|
"""A less noisy timedelta.__str__.
|
|
|
|
The default timedelta stringification contains a lot of leading zeros and
|
|
uses microsecond resolution. This makes for noisy output.
|
|
"""
|
|
hours, rem = divmod(total, 3600)
|
|
mins, secs = divmod(rem, 60)
|
|
ret = '%.3fs' % (secs,)
|
|
if mins:
|
|
ret = '%im%s' % (mins, ret)
|
|
if hours:
|
|
ret = '%ih%s' % (hours, ret)
|
|
return ret
|
|
|
|
|
|
class Progress(object):
|
|
def __init__(self, title, total=0, units='', print_newline=False, delay=True):
|
|
self._title = title
|
|
self._total = total
|
|
self._done = 0
|
|
self._start = time()
|
|
self._show = not delay
|
|
self._units = units
|
|
self._print_newline = print_newline
|
|
# Only show the active jobs section if we run more than one in parallel.
|
|
self._show_jobs = False
|
|
self._active = 0
|
|
|
|
def start(self, name):
|
|
self._active += 1
|
|
if not self._show_jobs:
|
|
self._show_jobs = self._active > 1
|
|
self.update(inc=0, msg='started ' + name)
|
|
|
|
def finish(self, name):
|
|
self.update(msg='finished ' + name)
|
|
self._active -= 1
|
|
|
|
def update(self, inc=1, msg=''):
|
|
self._done += inc
|
|
|
|
if _NOT_TTY or IsTrace():
|
|
return
|
|
|
|
if not self._show:
|
|
if 0.5 <= time() - self._start:
|
|
self._show = True
|
|
else:
|
|
return
|
|
|
|
if self._total <= 0:
|
|
sys.stderr.write('%s\r%s: %d,' % (
|
|
CSI_ERASE_LINE,
|
|
self._title,
|
|
self._done))
|
|
sys.stderr.flush()
|
|
else:
|
|
p = (100 * self._done) / self._total
|
|
if self._show_jobs:
|
|
jobs = '[%d job%s] ' % (self._active, 's' if self._active > 1 else '')
|
|
else:
|
|
jobs = ''
|
|
sys.stderr.write('%s\r%s: %2d%% %s(%d%s/%d%s)%s%s%s' % (
|
|
CSI_ERASE_LINE,
|
|
self._title,
|
|
p,
|
|
jobs,
|
|
self._done, self._units,
|
|
self._total, self._units,
|
|
' ' if msg else '', msg,
|
|
'\n' if self._print_newline else ''))
|
|
sys.stderr.flush()
|
|
|
|
def end(self):
|
|
if _NOT_TTY or IsTrace() or not self._show:
|
|
return
|
|
|
|
duration = duration_str(time() - self._start)
|
|
if self._total <= 0:
|
|
sys.stderr.write('%s\r%s: %d, done in %s\n' % (
|
|
CSI_ERASE_LINE,
|
|
self._title,
|
|
self._done,
|
|
duration))
|
|
sys.stderr.flush()
|
|
else:
|
|
p = (100 * self._done) / self._total
|
|
sys.stderr.write('%s\r%s: %3d%% (%d%s/%d%s), done in %s\n' % (
|
|
CSI_ERASE_LINE,
|
|
self._title,
|
|
p,
|
|
self._done, self._units,
|
|
self._total, self._units,
|
|
duration))
|
|
sys.stderr.flush()
|