From 784e16f3aa941ca3564d823cc686017a161621a1 Mon Sep 17 00:00:00 2001 From: Raman Tenneti Date: Fri, 11 Jun 2021 17:29:45 -0700 Subject: [PATCH] superproject: Don't exit if superproject tag doesn't exist in manifest. Don't exit if there are missing commit ids in superproject. This change implements the following suggestion from delphij@: "we should note the event (so we know that --use-superproject but there were some errors, e.g. manifest didn't specify commit id for some reason, or if there is no superproject but --use-superproject is used), print out a message telling the use that this is not support, but continue as if --no-use-superproject was specified?" Changes: superproject: + Added git_trace2_event_log as an argument to the constructor. + Sync method returns SyncResult a NamedTuple of ++ success - True if sync of superproject is successful, or False. ++ fatal - True if caller should exit, Or False. + UpdateProjectsRevisionId returns UpdateProjectsResult a NamedTuple of ++ manifest_path - path name of the overriding manifest file instead of None ++ fatal - True if caller should exit, Or False + _GetAllProjectsCommitIds returns CommitIdsResult a NamedTuple of ++ commit_ids - a dictionary with the projects/commit ids on success, otherwise None ++ fatal - True if caller should exit, Or False + Added _SkipUpdatingProjectRevisionId a helper function to see if a project's revision id needs to be updated or not. This function is used to exclude projects from local manifest file. + Added the following error events into git_trace2_event_log ++ If superproject is missing in a manifest ++ If there are missing commit ids for projects. command.py: + Deleted unused import - platform + Added git_trace2_event_log as a member so all subcmds can log error events. main.py: + Initialized git_trace2_event_log as a member of command object. init.py: + Deleted unused import - optparse init.py: + Called sys.exit only if Sync returns exit=True sync.py: + Called sys.exit only if Superproject's UpdateProjectsRevisionId returns exit=True + Reloaded the manifest only if manifest path is returned by UpdateProjectsRevisionId. If not, fall back to the old way of doing repo sync. test_git_superproject: + Added code to verify error events are being logged. + Added a test for no superproject tag + Added test for UpdateProjectsRevisionId not updating the revision id with the commit ids. Tested the code with the following commands. + Positive test case with aosp-master. $ repo_dev init -u persistent-https://android.git.corp.google.com/platform/manifest -b master --use-superproject NOTICE: --use-superproject is in beta; report any issues to the address described in `repo version` .../android/aosp/.repo/exp-superproject/925043f706ba64db713e9bf3b55987e2-superproject.git: Initial setup for superproject completed. Your identity is: Raman Tenneti If you want to change this, please re-run 'repo init' with --config-name repo has been initialized in .../android/aosp $ repo_dev sync -j40 --use-superproject remote: Total 12 (delta 4), reused 12 (delta 4) NOTICE: --use-superproject is in beta; report any issues to the address described in `repo version` .../android/aosp/.repo/exp-superproject/925043f706ba64db713e9bf3b55987e2-superproject.git: Initial setup for superproject completed. ... repo sync has finished successfully. + Negative test case without superproject tag. $ repo_dev sync -j40 --use-superproject NOTICE: --use-superproject is in beta; report any issues to the address described in `repo version` repo error: superproject tag is not defined in manifest: .../android/aosp/.repo/manifest.xml error: Cannot get project commit ids from manifest error: Update of revsionId from superproject has failed. Please resync with --no-use-superproject option ... Checking out: 100% (1022/1022), done in 3.589s repo sync has finished successfully. + Test for missing commit_id for a project. $ repo_dev sync -j40 --use-superproject NOTICE: --use-superproject is in beta; report any issues to the address described in `repo version` .../android/aosp/.repo/exp-superproject/925043f706ba64db713e9bf3b55987e2-superproject.git: Initial setup for superproject completed. error: please file a bug using go/repo-bug to report missing commit_ids for: ['build/blueprint'] error: Update of revsionId from superproject has failed. Please resync with --no-use-superproject option ... Checking out: 100% (1022/1022), done in 3.364s repo sync has finished successfully. $ ./run_tests -v ... ...== 164 passed in 2.87s ==... Bug: [google internal] b/189371541 Change-Id: I5ea49f87e8fa41be590fc0c914573e16c8cdfcfa Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/309162 Tested-by: Raman Tenneti Reviewed-by: Mike Frysinger --- command.py | 5 +- git_superproject.py | 118 +++++++++++++++++++++-------- main.py | 3 +- subcmds/init.py | 8 +- subcmds/sync.py | 18 +++-- tests/test_git_superproject.py | 134 ++++++++++++++++++++++++++++----- 6 files changed, 225 insertions(+), 61 deletions(-) diff --git a/command.py b/command.py index dc765db0..4087cab5 100644 --- a/command.py +++ b/command.py @@ -15,7 +15,6 @@ import multiprocessing import os import optparse -import platform import re import sys @@ -58,11 +57,13 @@ class Command(object): # it is the number of parallel jobs to default to. PARALLEL_JOBS = None - def __init__(self, repodir=None, client=None, manifest=None, gitc_manifest=None): + def __init__(self, repodir=None, client=None, manifest=None, gitc_manifest=None, + git_event_log=None): self.repodir = repodir self.client = client self.manifest = manifest self.gitc_manifest = gitc_manifest + self.git_event_log = git_event_log # Cache for the OptionParser property. self._optparse = None diff --git a/git_superproject.py b/git_superproject.py index 3c4144dd..8f1e04d6 100644 --- a/git_superproject.py +++ b/git_superproject.py @@ -19,12 +19,13 @@ https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects Examples: superproject = Superproject() - project_commit_ids = superproject.UpdateProjectsRevisionId(projects) + UpdateProjectsResult = superproject.UpdateProjectsRevisionId(projects) """ import hashlib import os import sys +from typing import NamedTuple from git_command import git_require, GitCommand from git_refs import R_HEADS @@ -34,6 +35,33 @@ _SUPERPROJECT_GIT_NAME = 'superproject.git' _SUPERPROJECT_MANIFEST_NAME = 'superproject_override.xml' +class SyncResult(NamedTuple): + """Return the status of sync and whether caller should exit.""" + + # Whether the superproject sync was successful. + success: bool + # Whether the caller should exit. + fatal: bool + + +class CommitIdsResult(NamedTuple): + """Return the commit ids and whether caller should exit.""" + + # A dictionary with the projects/commit ids on success, otherwise None. + commit_ids: dict + # Whether the caller should exit. + fatal: bool + + +class UpdateProjectsResult(NamedTuple): + """Return the overriding manifest file and whether caller should exit.""" + + # Path name of the overriding manfiest file if successful, otherwise None. + manifest_path: str + # Whether the caller should exit. + fatal: bool + + class Superproject(object): """Get commit ids from superproject. @@ -41,19 +69,21 @@ class Superproject(object): lookup of commit ids for all projects. It contains _project_commit_ids which is a dictionary with project/commit id entries. """ - def __init__(self, manifest, repodir, superproject_dir='exp-superproject', - quiet=False): + def __init__(self, manifest, repodir, git_event_log, + superproject_dir='exp-superproject', quiet=False): """Initializes superproject. Args: manifest: A Manifest object that is to be written to a file. repodir: Path to the .repo/ dir for holding all internal checkout state. It must be in the top directory of the repo client checkout. + git_event_log: A git trace2 event log to log events. superproject_dir: Relative path under |repodir| to checkout superproject. quiet: If True then only print the progress messages. """ self._project_commit_ids = None self._manifest = manifest + self._git_event_log = git_event_log self._quiet = quiet self._branch = self._GetBranch() self._repodir = os.path.abspath(repodir) @@ -172,44 +202,48 @@ class Superproject(object): """Gets a local copy of a superproject for the manifest. Returns: - True if sync of superproject is successful, or False. + SyncResult """ print('NOTICE: --use-superproject is in beta; report any issues to the ' 'address described in `repo version`', file=sys.stderr) if not self._manifest.superproject: - print('error: superproject tag is not defined in manifest', - file=sys.stderr) - return False + msg = (f'repo error: superproject tag is not defined in manifest: ' + f'{self._manifest.manifestFile}') + print(msg, file=sys.stderr) + self._git_event_log.ErrorEvent(msg, '') + return SyncResult(False, False) + should_exit = True url = self._manifest.superproject['remote'].url if not url: print('error: superproject URL is not defined in manifest', file=sys.stderr) - return False + return SyncResult(False, should_exit) if not self._Init(): - return False + return SyncResult(False, should_exit) if not self._Fetch(url): - return False + return SyncResult(False, should_exit) if not self._quiet: print('%s: Initial setup for superproject completed.' % self._work_git) - return True + return SyncResult(True, False) def _GetAllProjectsCommitIds(self): """Get commit ids for all projects from superproject and save them in _project_commit_ids. Returns: - A dictionary with the projects/commit ids on success, otherwise None. + CommitIdsResult """ - if not self.Sync(): - return None + sync_result = self.Sync() + if not sync_result.success: + return CommitIdsResult(None, sync_result.fatal) data = self._LsTree() if not data: print('error: git ls-tree failed to return data for superproject', file=sys.stderr) - return None + return CommitIdsResult(None, True) # Parse lines like the following to select lines starting with '160000' and # build a dictionary with project path (last element) and its commit id (3rd element). @@ -225,7 +259,7 @@ class Superproject(object): commit_ids[ls_data[3]] = ls_data[2] self._project_commit_ids = commit_ids - return commit_ids + return CommitIdsResult(commit_ids, False) def _WriteManfiestFile(self): """Writes manifest to a file. @@ -250,6 +284,23 @@ class Superproject(object): return None return manifest_path + def _SkipUpdatingProjectRevisionId(self, project): + """Checks if a project's revision id needs to be updated or not. + + Revision id for projects from local manifest will not be updated. + + Args: + project: project whose revision id is being updated. + + Returns: + True if a project's revision id should not be updated, or False, + """ + path = project.relpath + if not path: + return True + # Skip the project if it comes from the local manifest. + return any(s.startswith(LOCAL_MANIFEST_GROUP_PREFIX) for s in project.groups) + def UpdateProjectsRevisionId(self, projects): """Update revisionId of every project in projects with the commit id. @@ -257,30 +308,35 @@ class Superproject(object): projects: List of projects whose revisionId needs to be updated. Returns: - manifest_path: Path name of the overriding manfiest file instead of None. + UpdateProjectsResult """ - commit_ids = self._GetAllProjectsCommitIds() + commit_ids_result = self._GetAllProjectsCommitIds() + commit_ids = commit_ids_result.commit_ids if not commit_ids: print('error: Cannot get project commit ids from manifest', file=sys.stderr) - return None + return UpdateProjectsResult(None, commit_ids_result.fatal) projects_missing_commit_ids = [] for project in projects: + if self._SkipUpdatingProjectRevisionId(project): + continue path = project.relpath - if not path: - continue - # Skip the project if it comes from local manifest. - if any(s.startswith(LOCAL_MANIFEST_GROUP_PREFIX) for s in project.groups): - continue commit_id = commit_ids.get(path) - if commit_id: - project.SetRevisionId(commit_id) - else: + if not commit_id: projects_missing_commit_ids.append(path) + + # If superproject doesn't have a commit id for a project, then report an + # error event and continue as if do not use superproject is specified. if projects_missing_commit_ids: - print('error: please file a bug using %s to report missing commit_ids for: %s' % - (self._manifest.contactinfo.bugurl, projects_missing_commit_ids), file=sys.stderr) - return None + msg = (f'error: please file a bug using {self._manifest.contactinfo.bugurl} ' + f'to report missing commit_ids for: {projects_missing_commit_ids}') + print(msg, file=sys.stderr) + self._git_event_log.ErrorEvent(msg, '') + return UpdateProjectsResult(None, False) + + for project in projects: + if not self._SkipUpdatingProjectRevisionId(project): + project.SetRevisionId(commit_ids.get(project.relpath)) manifest_path = self._WriteManfiestFile() - return manifest_path + return UpdateProjectsResult(manifest_path, False) diff --git a/main.py b/main.py index 32ad0ff6..f6631f5f 100755 --- a/main.py +++ b/main.py @@ -208,7 +208,8 @@ class _Repo(object): repodir=self.repodir, client=repo_client, manifest=repo_client.manifest, - gitc_manifest=gitc_manifest) + gitc_manifest=gitc_manifest, + git_event_log=git_trace2_event_log) except KeyError: print("repo: '%s' is not a repo command. See 'repo help'." % name, file=sys.stderr) diff --git a/subcmds/init.py b/subcmds/init.py index 750facba..536e367c 100644 --- a/subcmds/init.py +++ b/subcmds/init.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import optparse import os import platform import re @@ -97,10 +96,13 @@ to update the working directory files. """ superproject = git_superproject.Superproject(self.manifest, self.repodir, + self.git_event_log, quiet=opt.quiet) - if not superproject.Sync(): + sync_result = superproject.Sync() + if not sync_result.success: print('error: git update of superproject failed', file=sys.stderr) - sys.exit(1) + if sync_result.fatal: + sys.exit(1) def _SyncManifest(self, opt): m = self.manifest.manifestProject diff --git a/subcmds/sync.py b/subcmds/sync.py index b15d9477..8d89cf72 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -302,21 +302,25 @@ later is required to fix a server side protocol bug. load_local_manifests: Whether to load local manifests. Returns: - Returns path to the overriding manifest file. + Returns path to the overriding manifest file instead of None. """ superproject = git_superproject.Superproject(self.manifest, self.repodir, + self.git_event_log, quiet=opt.quiet) all_projects = self.GetProjects(args, missing_ok=True, submodules_ok=opt.fetch_submodules) - manifest_path = superproject.UpdateProjectsRevisionId(all_projects) - if not manifest_path: + update_result = superproject.UpdateProjectsRevisionId(all_projects) + manifest_path = update_result.manifest_path + if manifest_path: + self._ReloadManifest(manifest_path, load_local_manifests) + else: print('error: Update of revsionId from superproject has failed. ' 'Please resync with --no-use-superproject option', file=sys.stderr) - sys.exit(1) - self._ReloadManifest(manifest_path, load_local_manifests) + if update_result.fatal: + sys.exit(1) return manifest_path def _FetchProjectList(self, opt, projects): @@ -961,7 +965,9 @@ later is required to fix a server side protocol bug. load_local_manifests = not self.manifest.HasLocalManifests if self._UseSuperproject(opt): - manifest_name = self._UpdateProjectsRevisionId(opt, args, load_local_manifests) + new_manifest_name = self._UpdateProjectsRevisionId(opt, args, load_local_manifests) + if not new_manifest_name: + manifest_name = new_manifest_name if self.gitc_manifest: gitc_manifest_projects = self.GetProjects(args, diff --git a/tests/test_git_superproject.py b/tests/test_git_superproject.py index ba61a3d1..d612f4e7 100644 --- a/tests/test_git_superproject.py +++ b/tests/test_git_superproject.py @@ -14,6 +14,7 @@ """Unittests for the git_superproject.py module.""" +import json import os import platform import tempfile @@ -21,6 +22,7 @@ import unittest from unittest import mock import git_superproject +import git_trace2_event_log import manifest_xml import platform_utils from test_manifest_xml import sort_attributes @@ -29,6 +31,11 @@ from test_manifest_xml import sort_attributes class SuperprojectTestCase(unittest.TestCase): """TestCase for the Superproject module.""" + PARENT_SID_KEY = 'GIT_TRACE2_PARENT_SID' + PARENT_SID_VALUE = 'parent_sid' + SELF_SID_REGEX = r'repo-\d+T\d+Z-.*' + FULL_SID_REGEX = r'^%s/%s' % (PARENT_SID_VALUE, SELF_SID_REGEX) + def setUp(self): """Set up superproject every time.""" self.tempdir = tempfile.mkdtemp(prefix='repo_tests') @@ -38,6 +45,13 @@ class SuperprojectTestCase(unittest.TestCase): os.mkdir(self.repodir) self.platform = platform.system().lower() + # By default we initialize with the expected case where + # repo launches us (so GIT_TRACE2_PARENT_SID is set). + env = { + self.PARENT_SID_KEY: self.PARENT_SID_VALUE, + } + self.git_event_log = git_trace2_event_log.EventLog(env=env) + # The manifest parsing really wants a git repo currently. gitdir = os.path.join(self.repodir, 'manifests.git') os.mkdir(gitdir) @@ -54,7 +68,8 @@ class SuperprojectTestCase(unittest.TestCase): """) - self._superproject = git_superproject.Superproject(manifest, self.repodir) + self._superproject = git_superproject.Superproject(manifest, self.repodir, + self.git_event_log) def tearDown(self): """Tear down superproject every time.""" @@ -66,14 +81,56 @@ class SuperprojectTestCase(unittest.TestCase): fp.write(data) return manifest_xml.XmlManifest(self.repodir, self.manifest_file) + def verifyCommonKeys(self, log_entry, expected_event_name, full_sid=True): + """Helper function to verify common event log keys.""" + self.assertIn('event', log_entry) + self.assertIn('sid', log_entry) + self.assertIn('thread', log_entry) + self.assertIn('time', log_entry) + + # Do basic data format validation. + self.assertEqual(expected_event_name, log_entry['event']) + if full_sid: + self.assertRegex(log_entry['sid'], self.FULL_SID_REGEX) + else: + self.assertRegex(log_entry['sid'], self.SELF_SID_REGEX) + self.assertRegex(log_entry['time'], r'^\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z$') + + def readLog(self, log_path): + """Helper function to read log data into a list.""" + log_data = [] + with open(log_path, mode='rb') as f: + for line in f: + log_data.append(json.loads(line)) + return log_data + + def verifyErrorEvent(self): + """Helper to verify that error event is written.""" + + with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir: + log_path = self.git_event_log.Write(path=tempdir) + self.log_data = self.readLog(log_path) + + self.assertEqual(len(self.log_data), 2) + error_event = self.log_data[1] + self.verifyCommonKeys(self.log_data[0], expected_event_name='version') + self.verifyCommonKeys(error_event, expected_event_name='error') + # Check for 'error' event specific fields. + self.assertIn('msg', error_event) + self.assertIn('fmt', error_event) + def test_superproject_get_superproject_no_superproject(self): """Test with no url.""" manifest = self.getXmlManifest(""" """) - superproject = git_superproject.Superproject(manifest, self.repodir) - self.assertFalse(superproject.Sync()) + superproject = git_superproject.Superproject(manifest, self.repodir, self.git_event_log) + # Test that exit condition is false when there is no superproject tag. + sync_result = superproject.Sync() + self.assertFalse(sync_result.success) + self.assertFalse(sync_result.fatal) + self.verifyErrorEvent() def test_superproject_get_superproject_invalid_url(self): """Test with an invalid url.""" @@ -84,8 +141,10 @@ class SuperprojectTestCase(unittest.TestCase): """) - superproject = git_superproject.Superproject(manifest, self.repodir) - self.assertFalse(superproject.Sync()) + superproject = git_superproject.Superproject(manifest, self.repodir, self.git_event_log) + sync_result = superproject.Sync() + self.assertFalse(sync_result.success) + self.assertTrue(sync_result.fatal) def test_superproject_get_superproject_invalid_branch(self): """Test with an invalid branch.""" @@ -96,21 +155,28 @@ class SuperprojectTestCase(unittest.TestCase): """) - superproject = git_superproject.Superproject(manifest, self.repodir) + self._superproject = git_superproject.Superproject(manifest, self.repodir, + self.git_event_log) with mock.patch.object(self._superproject, '_GetBranch', return_value='junk'): - self.assertFalse(superproject.Sync()) + sync_result = self._superproject.Sync() + self.assertFalse(sync_result.success) + self.assertTrue(sync_result.fatal) def test_superproject_get_superproject_mock_init(self): """Test with _Init failing.""" with mock.patch.object(self._superproject, '_Init', return_value=False): - self.assertFalse(self._superproject.Sync()) + sync_result = self._superproject.Sync() + self.assertFalse(sync_result.success) + self.assertTrue(sync_result.fatal) def test_superproject_get_superproject_mock_fetch(self): """Test with _Fetch failing.""" with mock.patch.object(self._superproject, '_Init', return_value=True): os.mkdir(self._superproject._superproject_path) with mock.patch.object(self._superproject, '_Fetch', return_value=False): - self.assertFalse(self._superproject.Sync()) + sync_result = self._superproject.Sync() + self.assertFalse(sync_result.success) + self.assertTrue(sync_result.fatal) def test_superproject_get_all_project_commit_ids_mock_ls_tree(self): """Test with LsTree being a mock.""" @@ -122,12 +188,13 @@ class SuperprojectTestCase(unittest.TestCase): with mock.patch.object(self._superproject, '_Init', return_value=True): with mock.patch.object(self._superproject, '_Fetch', return_value=True): with mock.patch.object(self._superproject, '_LsTree', return_value=data): - commit_ids = self._superproject._GetAllProjectsCommitIds() - self.assertEqual(commit_ids, { + commit_ids_result = self._superproject._GetAllProjectsCommitIds() + self.assertEqual(commit_ids_result.commit_ids, { 'art': '2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea', 'bootable/recovery': 'e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06', 'build/bazel': 'ade9b7a0d874e25fff4bf2552488825c6f111928' }) + self.assertFalse(commit_ids_result.fatal) def test_superproject_write_manifest_file(self): """Test with writing manifest to a file after setting revisionId.""" @@ -163,9 +230,10 @@ class SuperprojectTestCase(unittest.TestCase): return_value=data): # Create temporary directory so that it can write the file. os.mkdir(self._superproject._superproject_path) - manifest_path = self._superproject.UpdateProjectsRevisionId(projects) - self.assertIsNotNone(manifest_path) - with open(manifest_path, 'r') as fp: + update_result = self._superproject.UpdateProjectsRevisionId(projects) + self.assertIsNotNone(update_result.manifest_path) + self.assertFalse(update_result.fatal) + with open(update_result.manifest_path, 'r') as fp: manifest_xml_data = fp.read() self.assertEqual( sort_attributes(manifest_xml_data), @@ -178,6 +246,34 @@ class SuperprojectTestCase(unittest.TestCase): '' '') + def test_superproject_update_project_revision_id_no_superproject_tag(self): + """Test update of commit ids of a manifest without superproject tag.""" + manifest = self.getXmlManifest(""" + + + + + +""") + self.maxDiff = None + self._superproject = git_superproject.Superproject(manifest, self.repodir, + self.git_event_log) + self.assertEqual(len(self._superproject._manifest.projects), 1) + projects = self._superproject._manifest.projects + project = projects[0] + project.SetRevisionId('ABCDEF') + update_result = self._superproject.UpdateProjectsRevisionId(projects) + self.assertIsNone(update_result.manifest_path) + self.assertFalse(update_result.fatal) + self.verifyErrorEvent() + self.assertEqual( + sort_attributes(manifest.ToXml().toxml()), + '' + '' + '' + '' + '') + def test_superproject_update_project_revision_id_from_local_manifest_group(self): """Test update of commit ids of a manifest that have local manifest no superproject group.""" local_group = manifest_xml.LOCAL_MANIFEST_GROUP_PREFIX + ':local' @@ -194,7 +290,8 @@ class SuperprojectTestCase(unittest.TestCase): " /> """) self.maxDiff = None - self._superproject = git_superproject.Superproject(manifest, self.repodir) + self._superproject = git_superproject.Superproject(manifest, self.repodir, + self.git_event_log) self.assertEqual(len(self._superproject._manifest.projects), 2) projects = self._superproject._manifest.projects data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00' @@ -206,9 +303,10 @@ class SuperprojectTestCase(unittest.TestCase): return_value=data): # Create temporary directory so that it can write the file. os.mkdir(self._superproject._superproject_path) - manifest_path = self._superproject.UpdateProjectsRevisionId(projects) - self.assertIsNotNone(manifest_path) - with open(manifest_path, 'r') as fp: + update_result = self._superproject.UpdateProjectsRevisionId(projects) + self.assertIsNotNone(update_result.manifest_path) + self.assertFalse(update_result.fatal) + with open(update_result.manifest_path, 'r') as fp: manifest_xml_data = fp.read() # Verify platform/vendor/x's project revision hasn't changed. self.assertEqual(