diff --git a/command.py b/command.py
index dc765db0..4087cab5 100644
--- a/command.py
+++ b/command.py
@@ -15,7 +15,6 @@
import multiprocessing
import os
import optparse
-import platform
import re
import sys
@@ -58,11 +57,13 @@ class Command(object):
# it is the number of parallel jobs to default to.
PARALLEL_JOBS = None
- def __init__(self, repodir=None, client=None, manifest=None, gitc_manifest=None):
+ def __init__(self, repodir=None, client=None, manifest=None, gitc_manifest=None,
+ git_event_log=None):
self.repodir = repodir
self.client = client
self.manifest = manifest
self.gitc_manifest = gitc_manifest
+ self.git_event_log = git_event_log
# Cache for the OptionParser property.
self._optparse = None
diff --git a/git_superproject.py b/git_superproject.py
index 3c4144dd..8f1e04d6 100644
--- a/git_superproject.py
+++ b/git_superproject.py
@@ -19,12 +19,13 @@ https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects
Examples:
superproject = Superproject()
- project_commit_ids = superproject.UpdateProjectsRevisionId(projects)
+ UpdateProjectsResult = superproject.UpdateProjectsRevisionId(projects)
"""
import hashlib
import os
import sys
+from typing import NamedTuple
from git_command import git_require, GitCommand
from git_refs import R_HEADS
@@ -34,6 +35,33 @@ _SUPERPROJECT_GIT_NAME = 'superproject.git'
_SUPERPROJECT_MANIFEST_NAME = 'superproject_override.xml'
+class SyncResult(NamedTuple):
+ """Return the status of sync and whether caller should exit."""
+
+ # Whether the superproject sync was successful.
+ success: bool
+ # Whether the caller should exit.
+ fatal: bool
+
+
+class CommitIdsResult(NamedTuple):
+ """Return the commit ids and whether caller should exit."""
+
+ # A dictionary with the projects/commit ids on success, otherwise None.
+ commit_ids: dict
+ # Whether the caller should exit.
+ fatal: bool
+
+
+class UpdateProjectsResult(NamedTuple):
+ """Return the overriding manifest file and whether caller should exit."""
+
+ # Path name of the overriding manfiest file if successful, otherwise None.
+ manifest_path: str
+ # Whether the caller should exit.
+ fatal: bool
+
+
class Superproject(object):
"""Get commit ids from superproject.
@@ -41,19 +69,21 @@ class Superproject(object):
lookup of commit ids for all projects. It contains _project_commit_ids which
is a dictionary with project/commit id entries.
"""
- def __init__(self, manifest, repodir, superproject_dir='exp-superproject',
- quiet=False):
+ def __init__(self, manifest, repodir, git_event_log,
+ superproject_dir='exp-superproject', quiet=False):
"""Initializes superproject.
Args:
manifest: A Manifest object that is to be written to a file.
repodir: Path to the .repo/ dir for holding all internal checkout state.
It must be in the top directory of the repo client checkout.
+ git_event_log: A git trace2 event log to log events.
superproject_dir: Relative path under |repodir| to checkout superproject.
quiet: If True then only print the progress messages.
"""
self._project_commit_ids = None
self._manifest = manifest
+ self._git_event_log = git_event_log
self._quiet = quiet
self._branch = self._GetBranch()
self._repodir = os.path.abspath(repodir)
@@ -172,44 +202,48 @@ class Superproject(object):
"""Gets a local copy of a superproject for the manifest.
Returns:
- True if sync of superproject is successful, or False.
+ SyncResult
"""
print('NOTICE: --use-superproject is in beta; report any issues to the '
'address described in `repo version`', file=sys.stderr)
if not self._manifest.superproject:
- print('error: superproject tag is not defined in manifest',
- file=sys.stderr)
- return False
+ msg = (f'repo error: superproject tag is not defined in manifest: '
+ f'{self._manifest.manifestFile}')
+ print(msg, file=sys.stderr)
+ self._git_event_log.ErrorEvent(msg, '')
+ return SyncResult(False, False)
+ should_exit = True
url = self._manifest.superproject['remote'].url
if not url:
print('error: superproject URL is not defined in manifest',
file=sys.stderr)
- return False
+ return SyncResult(False, should_exit)
if not self._Init():
- return False
+ return SyncResult(False, should_exit)
if not self._Fetch(url):
- return False
+ return SyncResult(False, should_exit)
if not self._quiet:
print('%s: Initial setup for superproject completed.' % self._work_git)
- return True
+ return SyncResult(True, False)
def _GetAllProjectsCommitIds(self):
"""Get commit ids for all projects from superproject and save them in _project_commit_ids.
Returns:
- A dictionary with the projects/commit ids on success, otherwise None.
+ CommitIdsResult
"""
- if not self.Sync():
- return None
+ sync_result = self.Sync()
+ if not sync_result.success:
+ return CommitIdsResult(None, sync_result.fatal)
data = self._LsTree()
if not data:
print('error: git ls-tree failed to return data for superproject',
file=sys.stderr)
- return None
+ return CommitIdsResult(None, True)
# Parse lines like the following to select lines starting with '160000' and
# build a dictionary with project path (last element) and its commit id (3rd element).
@@ -225,7 +259,7 @@ class Superproject(object):
commit_ids[ls_data[3]] = ls_data[2]
self._project_commit_ids = commit_ids
- return commit_ids
+ return CommitIdsResult(commit_ids, False)
def _WriteManfiestFile(self):
"""Writes manifest to a file.
@@ -250,6 +284,23 @@ class Superproject(object):
return None
return manifest_path
+ def _SkipUpdatingProjectRevisionId(self, project):
+ """Checks if a project's revision id needs to be updated or not.
+
+ Revision id for projects from local manifest will not be updated.
+
+ Args:
+ project: project whose revision id is being updated.
+
+ Returns:
+ True if a project's revision id should not be updated, or False,
+ """
+ path = project.relpath
+ if not path:
+ return True
+ # Skip the project if it comes from the local manifest.
+ return any(s.startswith(LOCAL_MANIFEST_GROUP_PREFIX) for s in project.groups)
+
def UpdateProjectsRevisionId(self, projects):
"""Update revisionId of every project in projects with the commit id.
@@ -257,30 +308,35 @@ class Superproject(object):
projects: List of projects whose revisionId needs to be updated.
Returns:
- manifest_path: Path name of the overriding manfiest file instead of None.
+ UpdateProjectsResult
"""
- commit_ids = self._GetAllProjectsCommitIds()
+ commit_ids_result = self._GetAllProjectsCommitIds()
+ commit_ids = commit_ids_result.commit_ids
if not commit_ids:
print('error: Cannot get project commit ids from manifest', file=sys.stderr)
- return None
+ return UpdateProjectsResult(None, commit_ids_result.fatal)
projects_missing_commit_ids = []
for project in projects:
+ if self._SkipUpdatingProjectRevisionId(project):
+ continue
path = project.relpath
- if not path:
- continue
- # Skip the project if it comes from local manifest.
- if any(s.startswith(LOCAL_MANIFEST_GROUP_PREFIX) for s in project.groups):
- continue
commit_id = commit_ids.get(path)
- if commit_id:
- project.SetRevisionId(commit_id)
- else:
+ if not commit_id:
projects_missing_commit_ids.append(path)
+
+ # If superproject doesn't have a commit id for a project, then report an
+ # error event and continue as if do not use superproject is specified.
if projects_missing_commit_ids:
- print('error: please file a bug using %s to report missing commit_ids for: %s' %
- (self._manifest.contactinfo.bugurl, projects_missing_commit_ids), file=sys.stderr)
- return None
+ msg = (f'error: please file a bug using {self._manifest.contactinfo.bugurl} '
+ f'to report missing commit_ids for: {projects_missing_commit_ids}')
+ print(msg, file=sys.stderr)
+ self._git_event_log.ErrorEvent(msg, '')
+ return UpdateProjectsResult(None, False)
+
+ for project in projects:
+ if not self._SkipUpdatingProjectRevisionId(project):
+ project.SetRevisionId(commit_ids.get(project.relpath))
manifest_path = self._WriteManfiestFile()
- return manifest_path
+ return UpdateProjectsResult(manifest_path, False)
diff --git a/main.py b/main.py
index 32ad0ff6..f6631f5f 100755
--- a/main.py
+++ b/main.py
@@ -208,7 +208,8 @@ class _Repo(object):
repodir=self.repodir,
client=repo_client,
manifest=repo_client.manifest,
- gitc_manifest=gitc_manifest)
+ gitc_manifest=gitc_manifest,
+ git_event_log=git_trace2_event_log)
except KeyError:
print("repo: '%s' is not a repo command. See 'repo help'." % name,
file=sys.stderr)
diff --git a/subcmds/init.py b/subcmds/init.py
index 750facba..536e367c 100644
--- a/subcmds/init.py
+++ b/subcmds/init.py
@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import optparse
import os
import platform
import re
@@ -97,10 +96,13 @@ to update the working directory files.
"""
superproject = git_superproject.Superproject(self.manifest,
self.repodir,
+ self.git_event_log,
quiet=opt.quiet)
- if not superproject.Sync():
+ sync_result = superproject.Sync()
+ if not sync_result.success:
print('error: git update of superproject failed', file=sys.stderr)
- sys.exit(1)
+ if sync_result.fatal:
+ sys.exit(1)
def _SyncManifest(self, opt):
m = self.manifest.manifestProject
diff --git a/subcmds/sync.py b/subcmds/sync.py
index b15d9477..8d89cf72 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -302,21 +302,25 @@ later is required to fix a server side protocol bug.
load_local_manifests: Whether to load local manifests.
Returns:
- Returns path to the overriding manifest file.
+ Returns path to the overriding manifest file instead of None.
"""
superproject = git_superproject.Superproject(self.manifest,
self.repodir,
+ self.git_event_log,
quiet=opt.quiet)
all_projects = self.GetProjects(args,
missing_ok=True,
submodules_ok=opt.fetch_submodules)
- manifest_path = superproject.UpdateProjectsRevisionId(all_projects)
- if not manifest_path:
+ update_result = superproject.UpdateProjectsRevisionId(all_projects)
+ manifest_path = update_result.manifest_path
+ if manifest_path:
+ self._ReloadManifest(manifest_path, load_local_manifests)
+ else:
print('error: Update of revsionId from superproject has failed. '
'Please resync with --no-use-superproject option',
file=sys.stderr)
- sys.exit(1)
- self._ReloadManifest(manifest_path, load_local_manifests)
+ if update_result.fatal:
+ sys.exit(1)
return manifest_path
def _FetchProjectList(self, opt, projects):
@@ -961,7 +965,9 @@ later is required to fix a server side protocol bug.
load_local_manifests = not self.manifest.HasLocalManifests
if self._UseSuperproject(opt):
- manifest_name = self._UpdateProjectsRevisionId(opt, args, load_local_manifests)
+ new_manifest_name = self._UpdateProjectsRevisionId(opt, args, load_local_manifests)
+ if not new_manifest_name:
+ manifest_name = new_manifest_name
if self.gitc_manifest:
gitc_manifest_projects = self.GetProjects(args,
diff --git a/tests/test_git_superproject.py b/tests/test_git_superproject.py
index ba61a3d1..d612f4e7 100644
--- a/tests/test_git_superproject.py
+++ b/tests/test_git_superproject.py
@@ -14,6 +14,7 @@
"""Unittests for the git_superproject.py module."""
+import json
import os
import platform
import tempfile
@@ -21,6 +22,7 @@ import unittest
from unittest import mock
import git_superproject
+import git_trace2_event_log
import manifest_xml
import platform_utils
from test_manifest_xml import sort_attributes
@@ -29,6 +31,11 @@ from test_manifest_xml import sort_attributes
class SuperprojectTestCase(unittest.TestCase):
"""TestCase for the Superproject module."""
+ PARENT_SID_KEY = 'GIT_TRACE2_PARENT_SID'
+ PARENT_SID_VALUE = 'parent_sid'
+ SELF_SID_REGEX = r'repo-\d+T\d+Z-.*'
+ FULL_SID_REGEX = r'^%s/%s' % (PARENT_SID_VALUE, SELF_SID_REGEX)
+
def setUp(self):
"""Set up superproject every time."""
self.tempdir = tempfile.mkdtemp(prefix='repo_tests')
@@ -38,6 +45,13 @@ class SuperprojectTestCase(unittest.TestCase):
os.mkdir(self.repodir)
self.platform = platform.system().lower()
+ # By default we initialize with the expected case where
+ # repo launches us (so GIT_TRACE2_PARENT_SID is set).
+ env = {
+ self.PARENT_SID_KEY: self.PARENT_SID_VALUE,
+ }
+ self.git_event_log = git_trace2_event_log.EventLog(env=env)
+
# The manifest parsing really wants a git repo currently.
gitdir = os.path.join(self.repodir, 'manifests.git')
os.mkdir(gitdir)
@@ -54,7 +68,8 @@ class SuperprojectTestCase(unittest.TestCase):
""")
- self._superproject = git_superproject.Superproject(manifest, self.repodir)
+ self._superproject = git_superproject.Superproject(manifest, self.repodir,
+ self.git_event_log)
def tearDown(self):
"""Tear down superproject every time."""
@@ -66,14 +81,56 @@ class SuperprojectTestCase(unittest.TestCase):
fp.write(data)
return manifest_xml.XmlManifest(self.repodir, self.manifest_file)
+ def verifyCommonKeys(self, log_entry, expected_event_name, full_sid=True):
+ """Helper function to verify common event log keys."""
+ self.assertIn('event', log_entry)
+ self.assertIn('sid', log_entry)
+ self.assertIn('thread', log_entry)
+ self.assertIn('time', log_entry)
+
+ # Do basic data format validation.
+ self.assertEqual(expected_event_name, log_entry['event'])
+ if full_sid:
+ self.assertRegex(log_entry['sid'], self.FULL_SID_REGEX)
+ else:
+ self.assertRegex(log_entry['sid'], self.SELF_SID_REGEX)
+ self.assertRegex(log_entry['time'], r'^\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z$')
+
+ def readLog(self, log_path):
+ """Helper function to read log data into a list."""
+ log_data = []
+ with open(log_path, mode='rb') as f:
+ for line in f:
+ log_data.append(json.loads(line))
+ return log_data
+
+ def verifyErrorEvent(self):
+ """Helper to verify that error event is written."""
+
+ with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir:
+ log_path = self.git_event_log.Write(path=tempdir)
+ self.log_data = self.readLog(log_path)
+
+ self.assertEqual(len(self.log_data), 2)
+ error_event = self.log_data[1]
+ self.verifyCommonKeys(self.log_data[0], expected_event_name='version')
+ self.verifyCommonKeys(error_event, expected_event_name='error')
+ # Check for 'error' event specific fields.
+ self.assertIn('msg', error_event)
+ self.assertIn('fmt', error_event)
+
def test_superproject_get_superproject_no_superproject(self):
"""Test with no url."""
manifest = self.getXmlManifest("""
""")
- superproject = git_superproject.Superproject(manifest, self.repodir)
- self.assertFalse(superproject.Sync())
+ superproject = git_superproject.Superproject(manifest, self.repodir, self.git_event_log)
+ # Test that exit condition is false when there is no superproject tag.
+ sync_result = superproject.Sync()
+ self.assertFalse(sync_result.success)
+ self.assertFalse(sync_result.fatal)
+ self.verifyErrorEvent()
def test_superproject_get_superproject_invalid_url(self):
"""Test with an invalid url."""
@@ -84,8 +141,10 @@ class SuperprojectTestCase(unittest.TestCase):
""")
- superproject = git_superproject.Superproject(manifest, self.repodir)
- self.assertFalse(superproject.Sync())
+ superproject = git_superproject.Superproject(manifest, self.repodir, self.git_event_log)
+ sync_result = superproject.Sync()
+ self.assertFalse(sync_result.success)
+ self.assertTrue(sync_result.fatal)
def test_superproject_get_superproject_invalid_branch(self):
"""Test with an invalid branch."""
@@ -96,21 +155,28 @@ class SuperprojectTestCase(unittest.TestCase):
""")
- superproject = git_superproject.Superproject(manifest, self.repodir)
+ self._superproject = git_superproject.Superproject(manifest, self.repodir,
+ self.git_event_log)
with mock.patch.object(self._superproject, '_GetBranch', return_value='junk'):
- self.assertFalse(superproject.Sync())
+ sync_result = self._superproject.Sync()
+ self.assertFalse(sync_result.success)
+ self.assertTrue(sync_result.fatal)
def test_superproject_get_superproject_mock_init(self):
"""Test with _Init failing."""
with mock.patch.object(self._superproject, '_Init', return_value=False):
- self.assertFalse(self._superproject.Sync())
+ sync_result = self._superproject.Sync()
+ self.assertFalse(sync_result.success)
+ self.assertTrue(sync_result.fatal)
def test_superproject_get_superproject_mock_fetch(self):
"""Test with _Fetch failing."""
with mock.patch.object(self._superproject, '_Init', return_value=True):
os.mkdir(self._superproject._superproject_path)
with mock.patch.object(self._superproject, '_Fetch', return_value=False):
- self.assertFalse(self._superproject.Sync())
+ sync_result = self._superproject.Sync()
+ self.assertFalse(sync_result.success)
+ self.assertTrue(sync_result.fatal)
def test_superproject_get_all_project_commit_ids_mock_ls_tree(self):
"""Test with LsTree being a mock."""
@@ -122,12 +188,13 @@ class SuperprojectTestCase(unittest.TestCase):
with mock.patch.object(self._superproject, '_Init', return_value=True):
with mock.patch.object(self._superproject, '_Fetch', return_value=True):
with mock.patch.object(self._superproject, '_LsTree', return_value=data):
- commit_ids = self._superproject._GetAllProjectsCommitIds()
- self.assertEqual(commit_ids, {
+ commit_ids_result = self._superproject._GetAllProjectsCommitIds()
+ self.assertEqual(commit_ids_result.commit_ids, {
'art': '2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea',
'bootable/recovery': 'e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06',
'build/bazel': 'ade9b7a0d874e25fff4bf2552488825c6f111928'
})
+ self.assertFalse(commit_ids_result.fatal)
def test_superproject_write_manifest_file(self):
"""Test with writing manifest to a file after setting revisionId."""
@@ -163,9 +230,10 @@ class SuperprojectTestCase(unittest.TestCase):
return_value=data):
# Create temporary directory so that it can write the file.
os.mkdir(self._superproject._superproject_path)
- manifest_path = self._superproject.UpdateProjectsRevisionId(projects)
- self.assertIsNotNone(manifest_path)
- with open(manifest_path, 'r') as fp:
+ update_result = self._superproject.UpdateProjectsRevisionId(projects)
+ self.assertIsNotNone(update_result.manifest_path)
+ self.assertFalse(update_result.fatal)
+ with open(update_result.manifest_path, 'r') as fp:
manifest_xml_data = fp.read()
self.assertEqual(
sort_attributes(manifest_xml_data),
@@ -178,6 +246,34 @@ class SuperprojectTestCase(unittest.TestCase):
''
'')
+ def test_superproject_update_project_revision_id_no_superproject_tag(self):
+ """Test update of commit ids of a manifest without superproject tag."""
+ manifest = self.getXmlManifest("""
+
+
+
+
+
+""")
+ self.maxDiff = None
+ self._superproject = git_superproject.Superproject(manifest, self.repodir,
+ self.git_event_log)
+ self.assertEqual(len(self._superproject._manifest.projects), 1)
+ projects = self._superproject._manifest.projects
+ project = projects[0]
+ project.SetRevisionId('ABCDEF')
+ update_result = self._superproject.UpdateProjectsRevisionId(projects)
+ self.assertIsNone(update_result.manifest_path)
+ self.assertFalse(update_result.fatal)
+ self.verifyErrorEvent()
+ self.assertEqual(
+ sort_attributes(manifest.ToXml().toxml()),
+ ''
+ ''
+ ''
+ ''
+ '')
+
def test_superproject_update_project_revision_id_from_local_manifest_group(self):
"""Test update of commit ids of a manifest that have local manifest no superproject group."""
local_group = manifest_xml.LOCAL_MANIFEST_GROUP_PREFIX + ':local'
@@ -194,7 +290,8 @@ class SuperprojectTestCase(unittest.TestCase):
" />
""")
self.maxDiff = None
- self._superproject = git_superproject.Superproject(manifest, self.repodir)
+ self._superproject = git_superproject.Superproject(manifest, self.repodir,
+ self.git_event_log)
self.assertEqual(len(self._superproject._manifest.projects), 2)
projects = self._superproject._manifest.projects
data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00'
@@ -206,9 +303,10 @@ class SuperprojectTestCase(unittest.TestCase):
return_value=data):
# Create temporary directory so that it can write the file.
os.mkdir(self._superproject._superproject_path)
- manifest_path = self._superproject.UpdateProjectsRevisionId(projects)
- self.assertIsNotNone(manifest_path)
- with open(manifest_path, 'r') as fp:
+ update_result = self._superproject.UpdateProjectsRevisionId(projects)
+ self.assertIsNotNone(update_result.manifest_path)
+ self.assertFalse(update_result.fatal)
+ with open(update_result.manifest_path, 'r') as fp:
manifest_xml_data = fp.read()
# Verify platform/vendor/x's project revision hasn't changed.
self.assertEqual(