android/git-repo
1
Fork 0
mirror of https://gerrit.googlesource.com/git-repo synced 2024-12-21 07:16:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

gitc_utils: rewrite to use multiprocessing

Browse Source 
This is the only code in the tree that uses GitCommand asynchronously.
Rewrite it to use multiprocessing.Pool as it makes the code a little
bit easier to understand and simpler.

Change-Id: I3ed3b037f24aa1e9dfe8eec9ec21815cdda7678a
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/297143
Tested-by: Mike Frysinger <vapier@google.com>
Reviewed-by: Michael Mortensen <mmortensen@google.com>
This commit is contained in:
Mike Frysinger 2021-02-16 15:01:39 -05:00
parent 84230009ee
commit 8c1e9e62a3
1 changed files with 29 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
gitc_utils.py
View File

@ -13,6 +13,7 @@
# limitations under the License. # limitations under the License.
import os import os
import multiprocessing
import platform import platform
import re import re
import sys import sys
@ -35,6 +36,15 @@ def parse_clientdir(gitc_fs_path):
return wrapper.Wrapper().gitc_parse_clientdir(gitc_fs_path) return wrapper.Wrapper().gitc_parse_clientdir(gitc_fs_path)
def _get_project_revision(args):
"""Worker for _set_project_revisions to lookup one project remote."""
(i, url, expr) = args
gitcmd = git_command.GitCommand(
None, ['ls-remote', url, expr], capture_stdout=True, cwd='/tmp')
rc = gitcmd.Wait()
return (i, rc, gitcmd.stdout.split('\t', 1)[0])
def _set_project_revisions(projects): def _set_project_revisions(projects):
"""Sets the revisionExpr for a list of projects. """Sets the revisionExpr for a list of projects.
@ -47,22 +57,24 @@ def _set_project_revisions(projects):
""" """
# Retrieve the commit id for each project based off of it's current # Retrieve the commit id for each project based off of it's current
# revisionExpr and it is not already a commit id. # revisionExpr and it is not already a commit id.
project_gitcmds = [( with multiprocessing.Pool(NUM_BATCH_RETRIEVE_REVISIONID) as pool:
project, git_command.GitCommand(None, results_iter = pool.imap_unordered(
['ls-remote', _get_project_revision,
project.remote.url, ((i, project.remote.url, project.revisionExpr)
project.revisionExpr], for i, project in enumerate(projects)
capture_stdout=True, cwd='/tmp')) if not git_config.IsId(project.revisionExpr)),
for project in projects if not git_config.IsId(project.revisionExpr)] chunksize=8)
for proj, gitcmd in project_gitcmds: for (i, rc, revisionExpr) in results_iter:
if gitcmd.Wait(): project = projects[i]
print('FATAL: Failed to retrieve revisionExpr for %s' % proj) if rc:
print('FATAL: Failed to retrieve revisionExpr for %s' % project.name)
pool.terminate()
sys.exit(1) sys.exit(1)
revisionExpr = gitcmd.stdout.split('\t')[0]
if not revisionExpr: if not revisionExpr:
pool.terminate()
raise ManifestParseError('Invalid SHA-1 revision project %s (%s)' % raise ManifestParseError('Invalid SHA-1 revision project %s (%s)' %
(proj.remote.url, proj.revisionExpr)) (project.remote.url, project.revisionExpr))
proj.revisionExpr = revisionExpr project.revisionExpr = revisionExpr
def _manifest_groups(manifest): def _manifest_groups(manifest):
@ -123,11 +135,7 @@ def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
else: else:
proj.revisionExpr = gitc_proj.revisionExpr proj.revisionExpr = gitc_proj.revisionExpr
index = 0 _set_project_revisions(projects)
while index < len(projects):
_set_project_revisions(
projects[index:(index + NUM_BATCH_RETRIEVE_REVISIONID)])
index += NUM_BATCH_RETRIEVE_REVISIONID
if gitc_manifest is not None: if gitc_manifest is not None:
for path, proj in gitc_manifest.paths.items(): for path, proj in gitc_manifest.paths.items():